1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530enum FixStep {
532 Resource,
533 Recorded { report_lost: bool },
534}
535
536struct FixClaim {
553 path: PathBuf,
554}
555
556impl FixClaim {
557 fn acquire(dir: &Path) -> Result<Self> {
558 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
559 let path = dir.join("fix.lock");
560 let retry = crate::lock_file::Retry::new(None);
561 match Self::create(&path, retry) {
562 Ok(claim) => Ok(claim),
563 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
564 if Self::reclaim_if_dead(&path) {
565 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
566 } else {
567 bail!(
568 "another repair or `magi fix` is already running for this run ({} exists)",
569 path.display()
570 )
571 }
572 }
573 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
574 }
575 }
576
577 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
578 let mut f = retry.create(path)?;
579 use std::io::Write as _;
580 writeln!(f, "{}", std::process::id())?;
582 Ok(Self {
583 path: path.to_owned(),
584 })
585 }
586
587 fn reclaim_if_dead(path: &Path) -> bool {
591 let dead = std::fs::read_to_string(path)
592 .ok()
593 .and_then(|body| body.trim().parse::<u32>().ok())
594 .is_some_and(|pid| !crate::proc::pid_alive(pid));
595 dead && std::fs::remove_file(path).is_ok()
596 }
597}
598
599impl Drop for FixClaim {
600 fn drop(&mut self) {
601 let _ = std::fs::remove_file(&self.path);
602 }
603}
604
605struct DriverExecution<'a> {
608 runner: &'a mut Runner,
609 driver: crate::driver::DriverGuard,
610}
611
612impl Drop for DriverExecution<'_> {
613 fn drop(&mut self) {
614 self.driver.release_local(&mut self.runner.state);
615 }
616}
617
618impl Runner {
619 pub async fn start(
621 repo: &Path,
622 instruction: String,
623 config: Config,
624 origin: Origin,
625 ) -> Result<Self> {
626 Self::start_naming(repo, instruction, "", config, origin).await
627 }
628
629 pub async fn start_naming(
632 repo: &Path,
633 instruction: String,
634 _also_scan: &str,
635 config: Config,
636 origin: Origin,
637 ) -> Result<Self> {
638 let repo = git::toplevel(repo).await?;
639 let missing = agent::missing_programs(&config.agents);
640 if !missing.is_empty() {
641 bail!(
642 "these agent programs are not on PATH: {}. Fix the roster in \
643 magi.toml or install them.",
644 missing.join(", ")
645 );
646 }
647 let base_branch =
648 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
649 .await?;
650 let fetched_base = resolve_base(
651 &repo,
652 &base_branch,
653 &config.merge.remote,
654 config.coordination.as_ref(),
655 )
656 .await?;
657 let base_commit = fetched_base.commit.clone();
658 let roles = config.resolve_roles()?;
659 let max_parallel = config.graph.max_parallel.max(1);
660 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
661 state.fetched_base = Some(fetched_base);
665 state.origin = Some(origin);
666 state.reserve()?;
667 state.event("start", format!("run {} created", state.id));
668 state.save()?;
669 Ok(Self {
670 state,
671 roles,
672 sem: Arc::new(Semaphore::new(max_parallel)),
673 pause: Pause::new(),
674 interrupt: Pause::new(),
675 })
676 }
677
678 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
692 Self::review_taking_over(repo, branch, config, None, origin).await
693 }
694
695 pub async fn review_taking_over(
702 repo: &Path,
703 branch: &str,
704 config: Config,
705 takeover: Option<crate::handover::Takeover>,
706 origin: Origin,
707 ) -> Result<Self> {
708 let repo = git::toplevel(repo).await?;
709 let missing = agent::missing_programs(&config.agents);
710 if !missing.is_empty() {
711 bail!(
712 "these agent programs are not on PATH: {}. Fix the roster in \
713 magi.toml or install them.",
714 missing.join(", ")
715 );
716 }
717 let base_branch =
718 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
719 .await?;
720 if base_branch == branch {
721 bail!("`{branch}` is the base branch; there is nothing to review against");
722 }
723 let fetched_base = resolve_base(
724 &repo,
725 &base_branch,
726 &config.merge.remote,
727 config.coordination.as_ref(),
728 )
729 .await?;
730 let base_commit = fetched_base.commit.clone();
731
732 let roles = config.resolve_roles()?;
733 let max_parallel = config.graph.max_parallel.max(1);
734 let mut state = RunState::new(
735 repo.clone(),
736 base_branch,
737 base_commit.clone(),
738 String::new(),
739 config,
740 );
741 state.fetched_base = Some(fetched_base);
742 state.origin = Some(origin);
743 state.reserve()?;
744
745 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
752 earlier: Vec::new(),
753 home: crate::run::home(),
754 choice: None,
755 task_instruction: None,
756 });
757 let wait = repository_wait(&state, None);
758 let coordination_config = state.config.coordination.clone();
759 let coordination =
760 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
761 .await?;
762 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
763 if let Some(released) = &released {
764 state.event(
765 "release",
766 format!(
767 "took `{branch}` over from run {}: its worktree was released: {}",
768 crate::run::short_of(&released.old_id),
769 released.audit
770 ),
771 );
772 }
773 if let Some(choice) = takeover.choice.as_ref()
777 && let Err(e) =
778 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
779 .await
780 {
781 if let Some(released) = &released {
782 released.restore(&repo, branch).await;
783 }
784 return Err(e.context("applying the owner's answer about the diverged branch"));
785 }
786 for id in takeover.earlier.iter().rev() {
792 if let Ok(old) = RunState::load_under(id, &takeover.home)
793 && let Some(head) = old.pr_head.clone()
794 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
795 {
796 state.pr_head = Some(head);
797 break;
798 }
799 }
800 drop(coordination);
801 let opened = Self::open_review(
802 &repo,
803 branch,
804 state,
805 roles,
806 max_parallel,
807 base_commit,
808 &takeover,
809 )
810 .await;
811 if opened.is_err()
812 && let Some(released) = &released
813 {
814 let _coordination = crate::repo_lock::SeatLease::operation_with(
815 &repo,
816 coordination_config.as_ref(),
817 &wait,
818 )
819 .await?;
820 released.restore(&repo, branch).await;
821 }
822 opened
823 }
824
825 pub async fn recover_pr(
827 repo: &Path,
828 task: &str,
829 selection: crate::pr_recovery::Selection,
830 mut config: Config,
831 ) -> Result<Self> {
832 selection.configure(&mut config)?;
833 selection.check_task(task, &selection.instruction)?;
834 let base_branch =
835 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
836 .await?;
837 let fetched_base = resolve_base(
838 repo,
839 &base_branch,
840 &config.merge.remote,
841 config.coordination.as_ref(),
842 )
843 .await?;
844 let base_commit = fetched_base.commit.clone();
845 selection
846 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
847 .await?;
848 Box::pin(selection.verify_source(repo)).await?;
849 if let Some(proof) = &selection.source_proof {
850 anyhow::ensure!(
851 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
852 "source rebase base is outside the fetched landing base"
853 );
854 }
855 let roles = config.resolve_roles()?;
856 let max_parallel = config.graph.max_parallel.max(1);
857 let mut state = RunState::new(
858 repo.to_path_buf(),
859 base_branch,
860 base_commit.clone(),
861 selection.instruction.clone(),
862 config,
863 );
864 state.fetched_base = Some(fetched_base);
865 state.origin = Some(crate::run::Origin::queue(task));
866 state.pr_head = Some(selection.branch.clone());
867 state.pr_recovery = Some(selection.clone());
868 state.reserve()?;
869 if let Err(e) = crate::pr_head::check_predecessors(&state, &selection.branch).await {
870 state.discard_reserved();
872 return Err(e);
873 }
874 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
875 }
876
877 async fn open_selected(
878 repo: &Path,
879 state: RunState,
880 roles: ResolvedRoles,
881 max_parallel: usize,
882 base_commit: String,
883 selection: crate::pr_recovery::Selection,
884 ) -> Result<Self> {
885 let mut state = state;
886 let branch = state.branch_for('A');
887 let wait = repository_wait(&state, None);
888 let prepared = async {
889 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
890 git::git(repo, &["branch", &branch, selection.source()]).await?;
891 drop(coordination);
892 anyhow::Ok(())
893 }
894 .await;
895 if let Err(e) = prepared {
896 state.discard_reserved();
899 return Err(e);
900 }
901 state.event(
902 "recovery",
903 format!(
904 "selected implementation {} for {} (public lease {})",
905 selection.source(),
906 selection.url,
907 selection.head
908 ),
909 );
910 let takeover = crate::handover::Takeover {
911 earlier: Vec::new(),
912 home: crate::run::home(),
913 choice: None,
914 task_instruction: Some(selection.instruction.clone()),
915 };
916 if let Err(e) = state.save() {
917 state.discard_reserved();
918 return Err(e);
919 }
920 let id = state.id.clone();
921 let opened = Self::open_review(
922 repo,
923 &branch,
924 state,
925 roles,
926 max_parallel,
927 base_commit,
928 &takeover,
929 )
930 .await;
931 match opened {
932 Ok(runner) => Ok(runner),
933 Err(e) => {
934 if let Ok(mut partial) = RunState::load(&id) {
935 partial.status = RunStatus::Blocked;
936 partial.driver_exited = true;
937 partial.event(
938 "recovery",
939 format!("startup failed; retained source and candidate {branch}: {e:#}"),
940 );
941 partial.save()?;
942 }
943 Err(crate::pr_recovery::StartupFailed {
944 run: id,
945 cause: format!("{e:#}"),
946 }
947 .into())
948 }
949 }
950 }
951
952 async fn inherit(
957 repo: &Path,
958 branch: &str,
959 base_commit: &str,
960 takeover: &crate::handover::Takeover,
961 ) -> Option<crate::recovery::RecoveryRecord> {
962 use crate::recovery::{Relation, assess};
963 let newest = takeover.earlier.last()?;
964 let mut chosen = None;
965 for id in takeover.earlier.iter().rev() {
966 if let Ok(old) = RunState::load_under(id, &takeover.home)
967 && !old.reviews.is_empty()
968 && (old.winner().is_some_and(|w| w.branch == branch)
969 || old.pr_head.as_deref() == Some(branch))
970 {
971 chosen = Some(old);
972 break;
973 }
974 }
975 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
976 .await
977 .unwrap_or_default();
978 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
979 Some(last) if last.head == tip => Relation::Same,
980 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
981 _ => Relation::Diverged,
982 };
983 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
984 Some(assess(from, chosen.as_ref(), base_commit, relation))
985 }
986
987 async fn open_review(
990 repo: &Path,
991 branch: &str,
992 mut state: RunState,
993 roles: ResolvedRoles,
994 max_parallel: usize,
995 base_commit: String,
996 takeover: &crate::handover::Takeover,
997 ) -> Result<Self> {
998 let wait = repository_wait(&state, None);
999 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
1000 if state.pr_recovery.is_none() {
1001 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1002 }
1003 let start = review_base(
1006 repo,
1007 &state.config.merge.remote,
1008 &state.base_branch,
1009 &base_commit,
1010 branch,
1011 )
1012 .await;
1013 let log = git::log_oneline(repo, &start, branch)
1014 .await
1015 .unwrap_or_default();
1016 let log_text = if log.trim().is_empty() {
1017 "(no commit messages)"
1018 } else {
1019 log.trim()
1020 };
1021 let mut instruction = format!(
1022 "Review the work already on branch `{branch}`. There is no task \
1023 statement: what the change claims to do is whatever its commits \
1024 say.\n\n{log_text}"
1025 );
1026 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1030 if let Some(task) = takeover
1031 .task_instruction
1032 .as_deref()
1033 .filter(|t| !t.trim().is_empty())
1034 && rec.reviewed_head.is_some()
1035 {
1036 instruction = format!(
1037 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1038 task.trim()
1039 );
1040 rec.criteria_inherited = true;
1041 }
1042 state.event("recovery", rec.summary());
1043 state.recovery = Some(rec);
1044 }
1045 if let Some(selected) = &state.pr_recovery {
1046 instruction = selected.instruction.clone();
1047 }
1048 state.task_instruction = takeover.task_instruction.clone();
1049 state.instruction = instruction;
1050 state.reviewed_commits = Some(
1051 git::subjects(repo, &start, branch)
1052 .await
1053 .unwrap_or_default(),
1054 );
1055
1056 for id in &takeover.earlier {
1059 if let Ok(old) = RunState::load_under(id, &takeover.home)
1060 && old.repo == state.repo
1061 && (old.pr_head.as_deref() == Some(branch)
1062 || old.winner().is_some_and(|w| w.branch == branch))
1063 {
1064 for hold in old.branch_owner_holds {
1065 if !state.branch_owner_holds.contains(&hold) {
1066 state.event(
1067 "approval-policy",
1068 format!(
1069 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1070 hold.control.key(),
1071 hold.file,
1072 hold.head
1073 ),
1074 );
1075 state.branch_owner_holds.push(hold);
1076 }
1077 }
1078 }
1079 }
1080 let reviewed_head = git::rev_parse(repo, branch).await?;
1081 state.observe_branch_owner_holds(&reviewed_head).await;
1082
1083 let worktree = state.worktree_root().join("under-review");
1086 if let Some(parent) = worktree.parent() {
1087 tokio::fs::create_dir_all(parent).await.ok();
1088 }
1089 let path = worktree.to_string_lossy().to_string();
1090 git::git(repo, &["worktree", "add", &path, branch])
1091 .await
1092 .with_context(|| {
1093 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1094 })?;
1095
1096 let setup = match ensure_setup_config(&mut state, repo).await {
1097 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1098 Err(e) => Err(e),
1099 };
1100 if let Err(e) = setup {
1101 git::worktree_remove_clean(repo, &worktree).await.ok();
1102 return Err(e);
1103 }
1104 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1105 .await
1106 .unwrap_or(0);
1107 if commits == 0 {
1108 git::worktree_remove_clean(repo, &worktree).await.ok();
1109 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1110 }
1111 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1112 .await
1113 .map(|f| f.len())
1114 .unwrap_or(0);
1115 if files == 0
1116 && let (Ok(head_tree), Ok(base_tree)) = (
1117 git::tree_of(&worktree, "HEAD").await,
1118 git::tree_of(&worktree, &base_commit).await,
1119 )
1120 && head_tree == base_tree
1121 {
1122 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1123 git::worktree_remove_clean(repo, &worktree).await.ok();
1124 bail!(
1125 "`{branch}` at {} has a tree identical to base {}; this usually means \
1126 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1127 against `{}/{branch}`) rather than an empty change",
1128 short(&head),
1129 short(&base_commit),
1130 state.config.merge.remote
1131 );
1132 }
1133 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1134 .await
1135 .unwrap_or_default();
1136
1137 state.candidates.push(Candidate {
1138 index: 0,
1139 label: 'A',
1140 agent: EXISTING_BRANCH.to_owned(),
1143 branch: branch.to_owned(),
1144 worktree,
1145 summary: String::new(),
1146 stat,
1147 files,
1148 commits,
1149 empty: false,
1150 failed: None,
1151 verified_noop: None,
1152 duration_ms: 0,
1153 folded: false,
1154 });
1155 state.tally = Some(Tally {
1156 first_choice: BTreeMap::from([('A', 0)]),
1157 borda: BTreeMap::new(),
1158 winner: 'A',
1159 rankings: 0,
1160 unanimous_initial: false,
1161 deliberated: false,
1162 changed_votes: 0,
1163 unanimous_final: false,
1164 tie_break: None,
1165 judges: 0,
1169 present: 0,
1170 quorum: 0,
1171 met_quorum: true,
1172 uncontested: Some("review-only run: nothing competed".to_owned()),
1173 });
1174 state.status = RunStatus::Reviewing;
1175 state.prep_complete = Some(true);
1176 state.event(
1177 "start",
1178 format!(
1179 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1180 state.id
1181 ),
1182 );
1183 state.save()?;
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub fn resume(id: &str) -> Result<Self> {
1195 let state = RunState::load(id)?;
1196 crate::pr_head::ensure_owner(&state)?;
1197 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1198 bail!(
1199 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1200 );
1201 }
1202 if let Some(to) = &state.released_to {
1203 bail!(
1204 "run {} cannot be resumed: its worktree was released to run {}",
1205 state.short(),
1206 crate::run::short_of(to)
1207 );
1208 }
1209 let roles = state.config.resolve_roles()?;
1210 if let Some(id) = &state.config.graph.selected_implementer
1211 && state.candidates.iter().any(|c| c.agent != *id)
1212 {
1213 bail!(
1214 "replacement implementer pin does not match the saved candidates; task must stay held"
1215 );
1216 }
1217 let max_parallel = state.config.graph.max_parallel.max(1);
1218 Ok(Self {
1219 state,
1220 roles,
1221 sem: Arc::new(Semaphore::new(max_parallel)),
1222 pause: Pause::new(),
1223 interrupt: Pause::new(),
1224 })
1225 }
1226
1227 pub async fn execute(&mut self) -> Result<()> {
1234 if let Some(selection) = &self.state.pr_recovery
1235 && (selection.source_proof.is_some() || selection.source_run.is_some())
1236 {
1237 Box::pin(selection.verify_source(&self.state.repo)).await?;
1238 }
1239 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1242 || self.state.land_approval.is_some()
1243 || self.state.merge_decision.is_some()
1244 || !self.state.merge_repairs.is_empty()
1245 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1246 {
1247 let claim = FixClaim::acquire(&self.state.dir())?;
1248 let disk = RunState::load(&self.state.id)?;
1249 let checkpoint = |s: &RunState| {
1250 (
1251 s.merge_repairs.len(),
1252 s.merge_repairs
1253 .last()
1254 .map(|r| (r.question.clone(), r.phase)),
1255 s.land_approval
1256 .as_ref()
1257 .map(|a| (a.question.clone(), a.head.clone())),
1258 )
1259 };
1260 if checkpoint(&disk) != checkpoint(&self.state) {
1261 bail!(
1262 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1263 );
1264 }
1265 Some(claim)
1266 } else {
1267 None
1268 };
1269 let mut execution = self.claim_driver().await?;
1270 let runner = &mut *execution.runner;
1271 let wait = runner.repository_wait();
1272 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1275 let result = match result {
1276 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1277 record_repository_block(&mut runner.state, &e);
1278 runner.park_here()?;
1279 Ok(())
1280 }
1281 other => other,
1282 };
1283 execution.driver.finish(&mut runner.state).await?;
1284 let ended = if result.is_err() {
1285 Some({
1286 let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1287 if let Err(error) = &result {
1288 notice.message = format!(
1289 "Run {} stopped with an error: {}",
1290 runner.state.id,
1291 error.to_string().lines().next().unwrap_or("unknown error")
1292 );
1293 }
1294 notice
1295 })
1296 } else {
1297 crate::notices::run_ended(&runner.state)
1298 };
1299 if let Some(notice) = ended {
1300 crate::notices::raise(notice);
1301 }
1302 result
1303 }
1304
1305 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1306 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1307 }
1308
1309 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1310 let wait = self.repository_wait();
1311 coordinate_operation(&mut self.state, cwd, &wait).await
1312 }
1313
1314 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1316 let owner = match crate::pr_head::owner_lock_wait(
1317 &crate::run::home(),
1318 &self.state.id,
1319 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1320 )
1321 .await
1322 {
1323 Ok(owner) => owner,
1324 Err(error) => {
1325 let detail = format!("worktree maintenance or ownership update: {error:#}");
1326 self.state.resource_block = Some(crate::run::ResourceBlock {
1330 cause: crate::run::ResourceBlockCause::Other,
1331 node: "start".into(),
1332 kind: "worktree maintenance".into(),
1333 detail: detail.clone(),
1334 at: Timestamp::now(),
1335 });
1336 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1337 }
1338 };
1339 let disk = RunState::load(&self.state.id)?;
1340 crate::pr_head::ensure_owner(&disk)?;
1341 let claimed =
1342 crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1343 anyhow::ensure!(
1348 disk.liveness(claimed) != crate::run::Liveness::Live,
1349 "run already has a live driver; wait for its execution to end"
1350 );
1351 if let Some(to) = &disk.released_to {
1352 bail!(
1353 "run {} cannot continue: its worktree was released to run {}",
1354 self.state.short(),
1355 crate::run::short_of(to)
1356 );
1357 }
1358 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1359 let mut execution = DriverExecution {
1360 runner: self,
1361 driver,
1362 };
1363 let _owner = owner;
1366 let state = &mut execution.runner.state;
1367 state.parked = false;
1372 state.resource_wait = None;
1373 state.resource_block = None;
1374 state.clear_active();
1381 let pid = std::process::id();
1396 state.driver_pid = Some(pid);
1397 state.driver_started_at = crate::proc::process_started_at(pid);
1398 state.driver_exited = false;
1399 state.driver_claimed = Some(true);
1400 state.begin_pass();
1401 execution.driver.bind(state);
1402 state.save()?;
1403 execution.driver.record(state)?;
1404 Ok(execution)
1405 }
1406
1407 async fn execute_graph(&mut self) -> Result<()> {
1408 self.state.configure_recovery_resume()?;
1409 self.persist_candidates().await?;
1412 if self
1413 .state
1414 .merge_decision
1415 .as_ref()
1416 .is_some_and(|decision| !decision.closed)
1417 && self.state.status != RunStatus::Landing
1418 {
1419 if crate::merge_decision::resumable_hold(&self.state) {
1420 self.state.merge = Some(MergeOutcome {
1421 mode: MergeMode::Pr,
1422 ok: false,
1423 detail: self
1424 .state
1425 .merge_decision
1426 .as_ref()
1427 .unwrap()
1428 .target
1429 .pr
1430 .clone(),
1431 empty: false,
1432 });
1433 self.run_land().await?;
1434 self.settle_questions();
1435 return Ok(());
1436 }
1437 bail!(
1438 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1439 );
1440 }
1441 if self.state.config.graph.selected_implementer.is_some()
1442 && self.state.status == RunStatus::Blocked
1443 && self.state.tally.is_none()
1444 && self.state.candidate_preservation.is_empty()
1445 && self.state.interrupted_implementations.is_empty()
1446 {
1447 bail!(
1448 "selected implementation is held; explicitly release the owning task to retry it"
1449 );
1450 }
1451
1452 if self.state.merge_repairs.last().is_some_and(|r| {
1453 matches!(
1454 r.phase,
1455 crate::run::MergeRepairPhase::Requested
1456 | crate::run::MergeRepairPhase::Fixing
1457 | crate::run::MergeRepairPhase::Reviewing
1458 )
1459 }) {
1460 if self.repair_merge_review().await? {
1461 self.run_land().await?;
1462 }
1463 self.settle_questions();
1464 return Ok(());
1465 }
1466 if self.state.pr_publication_pending {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 self.settle_questions();
1475 return Ok(());
1476 }
1477 if self.state.status == RunStatus::Stalled {
1490 if self.recover_stall().await? {
1491 self.finish_after_tally().await?;
1492 } else {
1493 self.state.save()?;
1495 }
1496 return Ok(());
1497 }
1498 if self.state.status == RunStatus::Landing
1508 || crate::handover::landing_checkpoint(&self.state)
1509 {
1510 self.state.land_sync_pending = true;
1511 self.state.save()?;
1512 self.run_land().await?;
1513 self.settle_questions();
1518 return Ok(());
1519 }
1520 if self.state.github_text.is_some()
1525 && self.state.merge.is_none()
1526 && self.state.status == RunStatus::Gating
1527 {
1528 self.merge().await?;
1529 if self.resource_halt()? {
1530 return Ok(());
1531 }
1532 self.state.save()?;
1533 return Ok(());
1534 }
1535 if let Err(e) = self.prep().await {
1536 if e.is::<crate::repo_lock::ResourceBlocked>() {
1537 return Err(e);
1538 }
1539 self.state.status = RunStatus::Blocked;
1540 self.state
1541 .prep_failure
1542 .get_or_insert_with(|| format!("{e:#}"));
1543 self.state
1544 .event("prep", format!("preparation stopped: {e:#}"));
1545 self.state.save()?;
1546 return Err(e);
1547 }
1548 if self.park_here()? {
1549 return Ok(());
1550 }
1551 self.advise().await?;
1552 if self.park_here()? {
1553 return Ok(());
1554 }
1555 self.implement().await?;
1556 if self.park_here()? {
1557 return Ok(());
1558 }
1559 if self.state.status == RunStatus::VerifiedNoop {
1563 return Ok(());
1564 }
1565 self.judge().await?;
1566 if self.park_here()? {
1567 return Ok(());
1568 }
1569 self.deliberate().await?;
1570 if self.park_here()? {
1571 return Ok(());
1572 }
1573 self.vote().await?;
1574 if self.park_here()? {
1575 return Ok(());
1576 }
1577 self.tally()?;
1578 if self.state.status == RunStatus::Stalled {
1583 self.state.save()?;
1587 return Ok(());
1588 }
1589 self.finish_after_tally().await?;
1590 Ok(())
1591 }
1592
1593 fn stop_for_resource(&mut self) -> Result<()> {
1598 if self.pause.parked() || self.interrupt.parked() {
1601 self.park_here()?;
1602 return Ok(());
1603 }
1604 if self.state.resource_block.is_some() {
1605 self.state.status = RunStatus::Blocked;
1606 }
1607 self.state.save()
1608 }
1609
1610 fn resource_halt(&mut self) -> Result<bool> {
1612 if self.state.resource_block.is_none() {
1613 return Ok(false);
1614 }
1615 self.stop_for_resource()?;
1616 Ok(true)
1617 }
1618
1619 fn park_here(&mut self) -> Result<bool> {
1626 let parking = self.pause.parked() || self.interrupt.parked();
1627 if !parking && self.state.resource_block.is_some() {
1628 self.stop_for_resource()?;
1629 return Ok(true);
1630 }
1631 if !self.pause.parked() && !self.interrupt.parked() {
1636 return Ok(false);
1637 }
1638 if self.state.parked {
1639 return Ok(true);
1640 }
1641 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1642 if let Some(block) = &mut self.state.resource_block
1643 && matches!(
1644 block.cause,
1645 crate::run::ResourceBlockCause::Cancelled { .. }
1646 )
1647 {
1648 block.cause = crate::run::ResourceBlockCause::Cancelled {
1649 park: cause.clone(),
1650 };
1651 }
1652 let why = match cause.as_ref().map(ParkCause::label) {
1653 Some(reason) => format!(
1654 "parked after `{}` ({reason}) — resume to carry on from here",
1655 self.state.status.as_str()
1656 ),
1657 None => format!(
1658 "parked after `{}` — resume to carry on from here",
1659 self.state.status.as_str()
1660 ),
1661 };
1662 self.state.event("park", why);
1663 self.state
1664 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1665 self.state.parked = true;
1666 self.state.save()?;
1667 Ok(true)
1668 }
1669
1670 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1672 if self.state.prep_failure.is_none() {
1673 return Ok(());
1674 }
1675 let failed_at = self
1676 .state
1677 .events
1678 .iter()
1679 .rev()
1680 .find(|event| event.node == "prep")
1681 .map_or(self.state.updated_at, |event| event.at);
1682 anyhow::ensure!(
1683 approval_at > failed_at,
1684 "preparation retry requires a new explicit release after the failure"
1685 );
1686 self.state.prep_failure = None;
1687 self.state.prep_complete = Some(false);
1688 self.state
1689 .event("prep-retry", "operator approved retry of preparation");
1690 Ok(())
1691 }
1692
1693 pub fn on_pause(&mut self, pause: Pause) {
1695 self.pause = pause;
1696 }
1697
1698 pub fn watch_interrupt(&mut self, pause: Pause) {
1704 self.interrupt = pause;
1705 }
1706
1707 fn settle_questions(&mut self) {
1727 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1728 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1729 }
1730 }
1731
1732 async fn finish_after_tally(&mut self) -> Result<()> {
1735 self.graft().await?;
1738 if self.resource_halt()? {
1739 return Ok(());
1740 }
1741 self.fold_losers().await?;
1742 self.sync_to_base().await?;
1747 if self.resource_halt()? {
1748 return Ok(());
1749 }
1750 if self.state.status == RunStatus::AlreadyInBase {
1751 return Ok(());
1752 }
1753 self.review_loop().await?;
1754 if self.resource_halt()? {
1755 return Ok(());
1756 }
1757 self.sync_to_base().await?;
1758 if self.resource_halt()? {
1759 return Ok(());
1760 }
1761 if self.state.status == RunStatus::AlreadyInBase {
1762 return Ok(());
1763 }
1764 self.gate().await?;
1765 self.merge().await?;
1766 if self.resource_halt()? {
1767 return Ok(());
1768 }
1769 self.state.save()?;
1770 Ok(())
1771 }
1772
1773 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1780 let result = match ensure_setup_config(&mut self.state, repo).await {
1781 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1782 Err(e) => Err(e),
1783 };
1784 if let Err(e) = result {
1785 git::worktree_remove_clean(repo, wt).await.ok();
1786 if in_prep {
1787 self.rollback_prep(repo).await;
1788 }
1789 self.state.status = RunStatus::Blocked;
1790 self.state.event(
1791 "setup",
1792 format!("worktree setup failed in {}: {e:#}", wt.display()),
1793 );
1794 self.state.save()?;
1795 return Err(e);
1796 }
1797 Ok(())
1798 }
1799
1800 async fn rollback_prep(&mut self, repo: &Path) {
1802 let root = self.state.worktree_root();
1803 for c in std::mem::take(&mut self.state.candidates) {
1804 if matches!(
1805 git::worktree_remove_clean(repo, &c.worktree).await,
1806 Ok(true)
1807 ) {
1808 git::branch_delete(repo, &c.branch).await.ok();
1809 } else {
1810 self.state.candidates.push(c);
1811 }
1812 }
1813 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1814 for j in 1..=seats {
1815 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1816 .await
1817 .ok();
1818 }
1819 for k in 1..=self.state.config.graph.advisors {
1820 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1821 .await
1822 .ok();
1823 }
1824 }
1825
1826 async fn prep(&mut self) -> Result<()> {
1827 if let Some(reason) = &self.state.prep_failure {
1828 bail!(
1829 "candidate preparation previously failed: {reason}; correct the source before retrying"
1830 );
1831 }
1832 if self
1833 .state
1834 .prep_complete
1835 .unwrap_or(!self.state.candidates.is_empty())
1836 {
1837 return Ok(());
1838 }
1839 let repo = self.state.repo.clone();
1840 let _coordination = self.coordinate(&repo).await?;
1843 let base = self.state.base_commit.clone();
1844 let plan = self.state.sources.plan(&repo, &base).await?;
1845 self.rollback_prep(&repo).await;
1846 if !self.state.candidates.is_empty() {
1847 bail!(
1848 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1849 );
1850 }
1851 self.state.prep_complete = Some(false);
1852 self.state.status = RunStatus::Prep;
1853 self.state.save()?;
1854 let repo = self.state.repo.clone();
1855 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1856 let root = self.state.worktree_root();
1857 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1858
1859 let hooks_dir = self.state.dir().join("hooks");
1862 if self.state.config.blind.commit_msg_hook {
1863 std::fs::create_dir_all(&hooks_dir)
1864 .with_context(|| format!("create {}", hooks_dir.display()))?;
1865 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1866 let path = hooks_dir.join("commit-msg");
1867 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1868 make_executable(&path)?;
1869 git::acquire_worktree_config(&repo).await?;
1877 self.state.enabled_worktree_config = true;
1878 }
1879
1880 ensure_setup_config(&mut self.state, &repo).await?;
1883
1884 for (index, (spec, label)) in self
1885 .roles
1886 .implementers
1887 .clone()
1888 .into_iter()
1889 .zip(labels)
1890 .enumerate()
1891 {
1892 let branch = self.state.branch_for(label);
1893 let worktree = root.join(format!("cand-{label}"));
1894 self.state.event(
1898 "prep-intent",
1899 serde_json::json!({
1900 "branch": branch,
1901 "worktree": worktree,
1902 "start": start,
1903 "picks": plan.picks,
1904 })
1905 .to_string(),
1906 );
1907 self.state.save()?;
1908 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1909 if self.state.config.blind.commit_msg_hook {
1910 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1911 }
1912 git::local_exclude(&worktree, "/.magi/").await?;
1913 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1914 git::worktree_remove_clean(&repo, &worktree).await.ok();
1917 git::branch_delete(&repo, &branch).await.ok();
1918 self.rollback_prep(&repo).await;
1919 self.state.status = RunStatus::Blocked;
1920 self.state
1921 .event("prep", format!("worktree setup failed: {e:#}"));
1922 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1923 self.state.save()?;
1924 return Err(e);
1925 }
1926 self.state.candidates.push(Candidate {
1927 index,
1928 label,
1929 agent: spec.id.clone(),
1930 branch,
1931 worktree,
1932 summary: String::new(),
1933 stat: String::new(),
1934 files: 0,
1935 commits: 0,
1936 empty: false,
1937 failed: None,
1938 verified_noop: None,
1939 duration_ms: 0,
1940 folded: false,
1941 });
1942 self.state.save()?;
1943 for pick in &plan.picks {
1944 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1945 {
1946 self.state.status = RunStatus::Blocked;
1947 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1948 self.state
1949 .event("prep", format!("cannot apply referenced commit: {e}"));
1950 self.state.save()?;
1951 return Err(e);
1952 }
1953 }
1954 }
1955
1956 for j in 1..=self.roles.judges.len() {
1957 let wt = root.join(format!("judge-{j}"));
1958 if !wt.exists() {
1959 git::worktree_add_detached(&repo, &wt, &base).await?;
1960 self.setup_seat_worktree(&repo, &wt, true).await?;
1961 }
1962 }
1963
1964 if self.state.config.graph.advise {
1972 for k in 1..=self.state.config.graph.advisors {
1973 let wt = root.join(format!("advisor-{k}"));
1974 if !wt.exists() {
1975 git::worktree_add_detached(&repo, &wt, &base).await?;
1976 self.setup_seat_worktree(&repo, &wt, true).await?;
1977 }
1978 }
1979 }
1980
1981 let authors: Vec<&str> = self
1986 .roles
1987 .implementers
1988 .iter()
1989 .map(|a| a.id.as_str())
1990 .collect();
1991 let overlap: Vec<String> = self
1992 .roles
1993 .judges
1994 .iter()
1995 .enumerate()
1996 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1997 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1998 .collect();
1999 if !overlap.is_empty() {
2000 let note = format!(
2001 "{} also authored a candidate; blind, but the panel is less \
2002 independent than {} distinct agents would be",
2003 overlap.join(", "),
2004 self.roles.judges.len()
2005 );
2006 self.state.event("prep", note);
2007 }
2008
2009 self.state.event(
2010 "prep",
2011 format!(
2012 "{} candidates, {} judges, base {} ({})",
2013 self.state.candidates.len(),
2014 self.roles.judges.len(),
2015 &self.state.base_commit[..7.min(self.state.base_commit.len())],
2016 self.state.base_branch
2017 ),
2018 );
2019 self.state.prep_complete = Some(true);
2020 self.state.status = RunStatus::Implementing;
2021 self.state.save()?;
2022 Ok(())
2023 }
2024
2025 async fn advise(&mut self) -> Result<()> {
2058 let implement_untouched = self
2059 .state
2060 .candidates
2061 .iter()
2062 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2063 if !self.state.config.graph.advise || self.state.advise_attempted {
2064 return Ok(());
2065 }
2066 if !implement_untouched {
2067 self.state.event(
2068 "advise",
2069 "skipping the design-deliberation stage: at least one \
2070 candidate already shows implementation progress, so this \
2071 run is past the point the stage exists to run before"
2072 .to_owned(),
2073 );
2074 self.state.advise_attempted = true;
2075 self.state.save()?;
2076 return Ok(());
2077 }
2078 let run_id = self.state.id.clone();
2079 let prompts = self.state.config.prompts.clone();
2080 let instruction = self.state.instruction.clone();
2081 let language = self.state.config.graph.language.clone();
2082 let root = self.state.worktree_root();
2083 let n = self.state.config.graph.advisors;
2084 let where_recorded = self.state.dir().join("run.json");
2085
2086 let seats = match self.state.config.advisors() {
2087 Ok(seats) if !seats.is_empty() => seats,
2088 Ok(_) => {
2089 self.state.event(
2090 "advise",
2091 format!(
2092 "[graph] advisors is 0; skipping the design-deliberation \
2093 stage and continuing without a synthesis brief (see {})",
2094 where_recorded.display()
2095 ),
2096 );
2097 self.state.advise_attempted = true;
2098 self.state.save()?;
2099 return Ok(());
2100 }
2101 Err(e) => {
2102 self.state.event(
2103 "advise",
2104 format!(
2105 "could not resolve advisor seats ({e:#}); continuing \
2106 without a design-deliberation brief (see {})",
2107 where_recorded.display()
2108 ),
2109 );
2110 self.state.advise_attempted = true;
2111 self.state.save()?;
2112 return Ok(());
2113 }
2114 };
2115
2116 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2117 let artifacts = agent::artifacts_dir(&self.state.dir());
2118 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2119
2120 let mut jobs = Vec::new();
2121 for (i, spec) in seats.iter().cloned().enumerate() {
2122 let seat_key = format!("advisor-{}", i + 1);
2123 let seat = self.seat(&seat_key, &spec.id);
2124 jobs.push(SeatJob {
2125 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2126 spec,
2127 seat,
2128 cwd: worktrees[i % worktrees.len()].clone(),
2129 timeout,
2130 allow_write: false,
2131 sessions: false,
2132 artifacts: artifacts.clone(),
2133 stem: seat_key,
2134 handover: None,
2135 });
2136 }
2137
2138 self.state.event(
2139 "advise",
2140 format!(
2141 "{} advisor seat(s) sketching a design in parallel",
2142 jobs.len()
2143 ),
2144 );
2145 let mut quota_losses = Vec::new();
2146 let cache = self.state.config.cache_dir();
2147 let ctx = WaveCtx {
2148 carry_seats: false,
2149 pauses: Some((&self.pause, &self.interrupt)),
2150 run: &run_id,
2151 node: "advise",
2152 prompts: &prompts,
2153 cache: cache.as_deref(),
2154 round: None,
2155 };
2156 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2157 let results = ask_json_wave::<Proposal>(
2158 jobs,
2159 Arc::clone(&self.sem),
2160 self.state.config.graph.retries,
2161 &advisor_roster,
2162 &ctx,
2163 &mut quota_losses,
2164 &mut self.state,
2165 &|p: &Proposal| p.validate(),
2166 )
2167 .await;
2168 if self.state.resource_block.is_some() {
2169 return self.stop_for_resource();
2170 }
2171 self.state.quota.extend(quota_losses);
2172
2173 let mut records = Vec::with_capacity(results.len());
2174 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2175 let agent_id = seat.agent.clone();
2176 self.state.seats.insert(seat.key.clone(), seat);
2177 match res {
2178 Ok((proposal, out)) => {
2179 self.state
2180 .event("advise", format!("advisor-{} proposed a design", i + 1));
2181 records.push(advise::AdvisorRecord::proposed(
2182 i + 1,
2183 agent_id,
2184 proposal,
2185 out.duration_ms,
2186 ));
2187 }
2188 Err(e) => {
2189 self.state.event(
2190 "advise",
2191 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2192 );
2193 records.push(advise::AdvisorRecord::failed(
2194 i + 1,
2195 agent_id,
2196 e.to_string(),
2197 ));
2198 }
2199 }
2200 }
2201
2202 let mut advice = advise::Advice {
2203 records,
2204 synthesis: None,
2205 };
2206 if advice.proposals().is_empty() {
2207 self.state.event(
2208 "advise",
2209 "no advisor produced a usable proposal; continuing without a \
2210 synthesis brief"
2211 .to_owned(),
2212 );
2213 } else {
2214 match self
2215 .synthesize_brief(
2216 &advice,
2217 &instruction,
2218 &language,
2219 &worktrees[0],
2220 &artifacts,
2221 &run_id,
2222 &prompts,
2223 cache.as_deref(),
2224 )
2225 .await
2226 {
2227 Ok(Some(text)) => {
2228 self.state.event(
2229 "advise",
2230 "synthesized a design brief for the implementer".to_owned(),
2231 );
2232 advice.synthesis = Some(text);
2233 }
2234 Ok(None) => {
2235 self.state.event(
2236 "advise",
2237 "the synthesis seat produced nothing usable; continuing \
2238 without a design brief"
2239 .to_owned(),
2240 );
2241 }
2242 Err(e) => {
2243 if e.is::<crate::repo_lock::ResourceBlocked>() {
2244 return Err(e);
2245 }
2246 self.state.event(
2247 "advise",
2248 format!("could not synthesize a design brief: {e:#}"),
2249 );
2250 }
2251 }
2252 }
2253 advise::apply_reflection(&mut advice);
2254
2255 self.state.advice = Some(advice);
2256 self.state.advise_attempted = true;
2257 self.state.save()?;
2258 Ok(())
2259 }
2260
2261 #[allow(clippy::too_many_arguments)]
2273 async fn synthesize_brief(
2274 &mut self,
2275 advice: &advise::Advice,
2276 instruction: &str,
2277 language: &str,
2278 cwd: &Path,
2279 artifacts: &Path,
2280 run_id: &str,
2281 prompts: &Prompts,
2282 cache: Option<&Path>,
2283 ) -> Result<Option<String>> {
2284 let chain = agent::pick_chain(
2285 &self.state.config.agents,
2286 self.state.config.roles.synthesizer.as_ref(),
2287 &agent::installed,
2288 "synthesizer",
2289 )?;
2290 let proposals = advice.proposals();
2291 let mut prompt = prompt::with_overlay(
2292 prompt::synthesize_brief(instruction, &proposals, language),
2293 prompts.overlay("advise"),
2294 );
2295 if cache.is_some() {
2296 prompt.push('\n');
2301 prompt.push_str(&prompt::build_cache_note("advise", false));
2302 }
2303 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2304 let mut last = None;
2309 for (n, spec) in chain.iter().enumerate() {
2310 if n > 0 {
2311 self.state
2312 .event("advise", format!("synthesis falling back to {}", spec.id));
2313 }
2314 let mut seat = self.seat("advise-synthesis", &spec.id);
2315 let wait = self.repository_wait();
2316 let outcome = wait
2317 .scope(agent::invoke(
2318 spec,
2319 &mut seat,
2320 &Invocation {
2321 scope: crate::agent::InvocationScope::Repository,
2322 coordination: self.state.config.coordination.as_ref(),
2323 fetched_base: None,
2324 cwd,
2325 prompt: &prompt,
2326 timeout,
2327 allow_write: false,
2328 unsandboxed: false,
2329 sessions: false,
2330 artifacts,
2331 stem: &if n == 0 {
2332 "advise-synthesis".to_owned()
2333 } else {
2334 format!("advise-synthesis-{}", spec.id)
2335 },
2336 run: run_id,
2337 node: "advise",
2338 cache_dir: None,
2339 attachments: &[],
2340 writable: &[],
2341 },
2342 ))
2343 .await;
2344 if outcome.is_ok() {
2345 self.state.seats.insert(seat.key.clone(), seat);
2346 }
2347 let advance = agent::chain_advances(&outcome);
2348 last = Some(outcome);
2349 if !advance {
2350 break;
2351 }
2352 }
2353 let out = last.expect("a chain holds at least one agent")?;
2356 if !out.usable() {
2357 return Ok(None);
2358 }
2359 let text =
2360 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2361 Ok((!text.trim().is_empty()).then_some(text))
2362 }
2363
2364 async fn implement(&mut self) -> Result<()> {
2367 self.persist_candidates().await?;
2368 let run_id = self.state.id.clone();
2373 let prompts = self.state.config.prompts.clone();
2374 let todo: Vec<usize> = self
2375 .state
2376 .candidates
2377 .iter()
2378 .enumerate()
2379 .filter(|(_, c)| {
2380 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2381 && c.failed.is_none()
2382 && !c.empty
2383 })
2384 .map(|(i, _)| i)
2385 .collect();
2386 if todo.is_empty() {
2387 return self.after_implement();
2388 }
2389 self.state.status = RunStatus::Implementing;
2390
2391 let language = self.state.config.graph.language.clone();
2392 let github_language = self.state.config.graph.github_language.clone();
2393 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2394 let sessions = self.state.config.graph.sessions;
2395 let artifacts = agent::artifacts_dir(&self.state.dir());
2396 let brief = self
2400 .state
2401 .advice
2402 .as_ref()
2403 .and_then(|a| a.synthesis.as_deref())
2404 .map(str::to_owned);
2405 let attachments = self.state.attachments.clone();
2406
2407 let mut jobs = Vec::new();
2408 for &i in &todo {
2409 let (index, label, worktree) = {
2410 let c = &self.state.candidates[i];
2411 (c.index, c.label, c.worktree.clone())
2412 };
2413 let spec = self.roles.implementers[index].clone();
2414 let seat_key = format!("impl-{label}");
2415 let seat = self.seat(&seat_key, &spec.id);
2416 let instruction = seeded_instruction(&self.state);
2417 jobs.push(SeatJob {
2418 spec,
2419 seat,
2420 prompt: prompt::implement_in(
2421 &instruction,
2422 &worktree.to_string_lossy(),
2423 &language,
2424 &github_language,
2425 brief.as_deref(),
2426 &attachments,
2427 ),
2428 cwd: worktree,
2429 timeout,
2430 allow_write: true,
2431 sessions,
2432 artifacts: artifacts.clone(),
2433 stem: format!("impl-{label}"),
2434 handover: None,
2435 });
2436 }
2437
2438 self.state.event(
2439 "implement",
2440 format!("{} candidates in parallel", jobs.len()),
2441 );
2442 let mut sent = jobs.clone();
2448 let cache = self.state.config.cache_dir();
2449 let ctx = WaveCtx {
2450 carry_seats: false,
2451 pauses: Some((&self.pause, &self.interrupt)),
2452 run: &run_id,
2453 node: "implement",
2454 prompts: &prompts,
2455 cache: cache.as_deref(),
2456 round: None,
2457 };
2458 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2459 if !wave_blocked(&results) {
2460 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2461 .await;
2462 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2463 .await?;
2464 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2465 .await;
2466 }
2467 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2468
2469 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2470 && results
2471 .iter()
2472 .any(|(_, _, out)| FailClass::of(out).is_some());
2473 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2474 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2475 if self
2479 .state
2480 .candidate_preservation
2481 .contains_key(&self.state.candidates[i].label)
2482 {
2483 self.state.seats.insert(seat.key.clone(), seat);
2484 }
2485 continue;
2486 }
2487 let seat_key = seat.key.clone();
2488 let agent = seat.agent.clone();
2497 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2498 self.state.seats.insert(seat.key.clone(), seat);
2499 let label = self.state.candidates[i].label;
2500 self.state.interrupted_implementations.remove(&label);
2501
2502 let (summary, duration, failed, verified_claim) = match out {
2503 AgentOutcome::Ok(o) => {
2504 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2505 let failed = (!o.usable()).then(|| {
2506 if o.timed_out {
2507 "agent timed out".to_owned()
2508 } else {
2509 format!(
2510 "agent exited with {}",
2511 crate::notices::exit_status(o.exit_code)
2512 )
2513 }
2514 });
2515 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2516 (text, o.duration_ms, failed, verified_claim)
2517 }
2518 AgentOutcome::Dropped(o) => {
2524 let why = o
2525 .dropped
2526 .as_ref()
2527 .map(|d| d.why.as_str())
2528 .unwrap_or("the CLI ended the stream without delivering its answer");
2529 (
2530 String::new(),
2531 o.duration_ms,
2532 Some(format!("the CLI dropped the stream ({why})")),
2533 None,
2534 )
2535 }
2536 AgentOutcome::Quota(o) => {
2537 self.state.quota.push(QuotaLoss {
2538 seat: seat_key,
2539 node: "implement".to_owned(),
2540 at: Timestamp::now(),
2541 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2542 });
2543 (
2544 String::new(),
2545 o.duration_ms,
2546 Some("rate limited (quota); produced no change".to_owned()),
2547 None,
2548 )
2549 }
2550 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2551 (String::new(), 0, Some(e), None)
2552 }
2553 };
2554
2555 let c = &mut self.state.candidates[i];
2558 if !exhausted_the_fallback_chain {
2559 c.agent = agent;
2560 }
2561 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2562 c.duration_ms = duration;
2563 c.failed = failed;
2564 c.verified_noop = verified_claim;
2565 self.state
2566 .candidate_preservation
2567 .insert(label, "awaiting persistence".to_owned());
2568 }
2569 self.state.save()?;
2570 self.persist_candidates().await?;
2571
2572 if resource_blocked {
2573 return self.stop_for_resource();
2574 }
2575 if selected_failed {
2576 self.state.status = RunStatus::Blocked;
2577 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2578 self.state.save()?;
2579 bail!(
2580 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2581 );
2582 }
2583 self.after_implement()
2584 }
2585
2586 async fn persist_candidates(&mut self) -> Result<()> {
2588 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2589 for label in labels {
2590 let i = self
2591 .state
2592 .candidates
2593 .iter()
2594 .position(|c| c.label == label)
2595 .context("pending candidate is missing")?;
2596 let c = self.state.candidates[i].clone();
2597 let interrupted = self.state.interrupted_implementations.contains(&label);
2598 let result = async {
2599 let wait = crate::repo_lock::WaitPolicy::new(
2600 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2601 || false,
2602 );
2603 let _coordination =
2604 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2605 if let Err(e) = git::snapshot_uncommitted(
2608 &c.worktree,
2609 &self
2610 .state
2611 .dir()
2612 .join("artifacts")
2613 .join(format!("candidate-{label}-recovery")),
2614 )
2615 .await
2616 {
2617 self.state.event(
2618 "preserve",
2619 format!("candidate {label} snapshot incomplete: {e:#}"),
2620 );
2621 }
2622 let rescued = git::rescue_commit(
2623 &c.worktree,
2624 &format!("magi: candidate {label} (uncommitted work)"),
2625 )
2626 .await?;
2627 self.state.note_withheld("implement", &rescued.withheld);
2628 let base = &self.state.base_commit;
2629 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2630 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2631 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2632 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2633 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2634 if c.empty && commits > 0 && !patch.trim().is_empty()
2635 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2636 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2637 || self.state.tally.is_some())
2638 {
2639 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2643 }
2644 let c = &mut self.state.candidates[i];
2645 c.commits = commits;
2646 c.stat = stat;
2647 c.files = files;
2648 let no_changes = commits == 0 || patch.trim().is_empty();
2649 c.empty = no_changes && !interrupted;
2650 if interrupted && no_changes {
2651 c.failed = None;
2652 c.verified_noop = None;
2653 }
2654 if !c.empty {
2655 c.failed = None;
2656 c.verified_noop = None;
2657 }
2658 self.state.event(
2659 "preserve",
2660 format!("candidate {label}: {files} files, {commits} commits"),
2661 );
2662 Ok::<_, anyhow::Error>(())
2663 }
2664 .await;
2665 match result {
2666 Ok(()) => {
2667 self.state.candidate_preservation.remove(&label);
2668 if !interrupted || self.state.candidates[i].files > 0 {
2669 self.state.interrupted_implementations.remove(&label);
2670 }
2671 }
2672 Err(e) => {
2673 let reason = format!("candidate {label} preservation failed: {e:#}");
2674 self.state.event("preserve", &reason);
2675 self.state.candidate_preservation.insert(label, reason);
2676 }
2677 }
2678 if self.state.candidate_preservation.is_empty()
2679 && self.state.status == RunStatus::Blocked
2680 && self.state.candidates.iter().all(|c| c.failed.is_none())
2681 {
2682 self.state.status = RunStatus::Implementing;
2686 self.state.event(
2687 "preserve",
2688 "candidate persistence recovered; implementation can continue",
2689 );
2690 }
2691 self.state.save()?;
2692 }
2693 if !self.state.candidate_preservation.is_empty() {
2694 self.state.status = RunStatus::Blocked;
2695 self.state.save()?;
2696 bail!(
2697 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2698 );
2699 }
2700 Ok(())
2701 }
2702
2703 async fn resume_undelivered(
2731 &mut self,
2732 results: &mut [(usize, SeatState, AgentOutcome)],
2733 sent: &[SeatJob],
2734 prompts: &Prompts,
2735 run_id: &str,
2736 ) {
2737 for (wi, seat, out) in results.iter_mut() {
2738 let Some(dropped) = (match &*out {
2739 AgentOutcome::Dropped(o) => o.dropped.clone(),
2740 _ => None,
2741 }) else {
2742 continue;
2743 };
2744 let Some(job) = sent.get(*wi) else { continue };
2745 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2747 self.state.event(
2748 "implement",
2749 format!(
2750 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2751 work is in the tree",
2752 seat.key, dropped.output_tokens, dropped.why
2753 ),
2754 );
2755 continue;
2756 }
2757 if !has_context(&job.spec, seat, job.sessions) {
2765 self.state.event(
2766 "implement",
2767 format!(
2768 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2769 is no session left to resume",
2770 seat.key, dropped.output_tokens, dropped.why
2771 ),
2772 );
2773 continue;
2774 }
2775 self.state.event(
2776 "implement",
2777 format!(
2778 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2779 conversation",
2780 seat.key, dropped.output_tokens, dropped.why
2781 ),
2782 );
2783 let mut retry = job.clone();
2784 retry.seat = seat.clone();
2785 retry.prompt = prompt::resume_after_drop(&dropped.why);
2786 retry.timeout = retry_budget(job.timeout, true);
2787 retry.stem = format!("{}-resume", job.stem);
2788 let cache = self.state.config.cache_dir();
2789 let ctx = WaveCtx {
2790 carry_seats: false,
2791 pauses: Some((&self.pause, &self.interrupt)),
2792 run: run_id,
2793 node: "implement",
2794 prompts,
2795 cache: cache.as_deref(),
2796 round: None,
2797 };
2798 let (resumed_seat, resumed) =
2799 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2800 *seat = resumed_seat;
2801 *out = resumed;
2802 }
2803 }
2804
2805 async fn resume_seat_handovers(
2870 &mut self,
2871 results: &mut [(usize, SeatState, AgentOutcome)],
2872 sent: &mut [SeatJob],
2873 prompts: &Prompts,
2874 run_id: &str,
2875 ) -> Result<()> {
2876 let instruction = seeded_instruction(&self.state);
2877 let language = self.state.config.graph.language.clone();
2878 let github_language = self.state.config.graph.github_language.clone();
2879 let brief = self
2880 .state
2881 .advice
2882 .as_ref()
2883 .and_then(|a| a.synthesis.as_deref())
2884 .map(str::to_owned);
2885 let attachments = self.state.attachments.clone();
2886 for (wi, seat, out) in results.iter_mut() {
2887 let others: BTreeSet<String> = sent
2890 .iter()
2891 .enumerate()
2892 .filter(|(j, _)| j != wi)
2893 .map(|(_, j)| j.spec.id.clone())
2894 .collect();
2895 let Some(job) = sent.get_mut(*wi) else {
2896 continue;
2897 };
2898 let start = self
2903 .roles
2904 .implementer_roster
2905 .iter()
2906 .position(|s| s.id == job.spec.id)
2907 .unwrap_or(0);
2908 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2909 let mut fallback_attempt = 0usize;
2910 let mut prev: Option<FailClass> = None;
2911 while let Some(cur) = FailClass::of(&*out) {
2912 if !should_hand_over(prev.as_ref(), &cur) {
2913 break;
2914 }
2915 let Some(next) =
2916 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2917 .cloned()
2918 else {
2919 break;
2920 };
2921 tried.insert(next.id.clone());
2922 fallback_attempt += 1;
2923
2924 let coordination = self.coordinate(&job.cwd).await?;
2925 if let Ok(r) = git::rescue_commit(
2926 &job.cwd,
2927 &format!(
2928 "magi: candidate {} (uncommitted work before {} fallback)",
2929 seat.key,
2930 if cur == FailClass::Quota {
2931 "quota"
2932 } else {
2933 "handover"
2934 }
2935 ),
2936 )
2937 .await
2938 {
2939 self.state.note_withheld("implement", &r.withheld);
2940 }
2941 drop(coordination);
2942
2943 record_handover(
2944 &mut self.state,
2945 "implement",
2946 &seat.key,
2947 &seat.agent,
2948 &next.id,
2949 &cur,
2950 &fail_reason(&*out),
2951 );
2952 prev = Some(cur.clone());
2953
2954 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2955 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2956 job.spec = next.clone();
2964 let mut retry = job.clone();
2965 retry.seat = new_seat;
2966 retry.prompt = prompt::implement_in(
2967 &instruction,
2968 &job.cwd.to_string_lossy(),
2969 &language,
2970 &github_language,
2971 brief.as_deref(),
2972 &attachments,
2973 );
2974 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2975 let cache = self.state.config.cache_dir();
2976 let ctx = WaveCtx {
2977 carry_seats: false,
2978 pauses: Some((&self.pause, &self.interrupt)),
2979 run: run_id,
2980 node: "implement",
2981 prompts,
2982 cache: cache.as_deref(),
2983 round: None,
2984 };
2985 let (fallback_seat, fallback_out) = run_one(
2986 retry,
2987 Arc::clone(&self.sem),
2988 &ctx,
2989 &mut self.state,
2990 fallback_attempt,
2991 )
2992 .await;
2993 *seat = fallback_seat;
2994 *out = fallback_out;
2995 }
2996 }
2997 Ok(())
2998 }
2999
3000 async fn resume_unconfirmed_commands(
3024 &mut self,
3025 results: &mut [(usize, SeatState, AgentOutcome)],
3026 sent: &[SeatJob],
3027 prompts: &Prompts,
3028 run_id: &str,
3029 ) {
3030 for (wi, seat, out) in results.iter_mut() {
3031 let AgentOutcome::Ok(o) = &*out else {
3032 continue;
3033 };
3034 if !has_unconfirmed_command(&o.commands) {
3035 continue;
3036 }
3037 let Some(job) = sent.get(*wi) else { continue };
3038 if !has_context(&job.spec, seat, job.sessions) {
3039 self.state.event(
3040 "implement",
3041 format!(
3042 "{}: the reply named a command whose own CLI never confirmed the exit \
3043 status of, but there is no session left to resume",
3044 seat.key
3045 ),
3046 );
3047 continue;
3048 }
3049 self.state.event(
3050 "implement",
3051 format!(
3052 "{}: the reply named a command whose own CLI never confirmed the exit \
3053 status of; resuming the conversation",
3054 seat.key
3055 ),
3056 );
3057 let mut retry = job.clone();
3058 retry.seat = seat.clone();
3059 retry.prompt = prompt::resume_incomplete(
3060 "a command in your last reply had no confirmed exit status",
3061 );
3062 retry.timeout = retry_budget(job.timeout, true);
3063 retry.stem = format!("{}-confirm", job.stem);
3064 let cache = self.state.config.cache_dir();
3065 let ctx = WaveCtx {
3066 carry_seats: false,
3067 pauses: Some((&self.pause, &self.interrupt)),
3068 run: run_id,
3069 node: "implement",
3070 prompts,
3071 cache: cache.as_deref(),
3072 round: None,
3073 };
3074 let (resumed_seat, resumed) =
3075 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3076 *seat = resumed_seat;
3077 *out = resumed;
3078 }
3079 }
3080
3081 async fn continue_fix_report(
3102 &mut self,
3103 mut seat: SeatState,
3104 parse_err: String,
3105 job: &SeatJob,
3106 prompts: &Prompts,
3107 run_id: &str,
3108 round: usize,
3109 ) -> (
3110 SeatState,
3111 Option<FixReport>,
3112 Option<String>,
3113 ContinuationRecord,
3114 ) {
3115 let mut last_err = parse_err;
3116 let mut cumulative_wait_ms = 0u64;
3117 let mut attempts = 0usize;
3118 loop {
3119 if !has_context(&job.spec, &seat, job.sessions) {
3120 self.state.event(
3121 "fix",
3122 format!(
3123 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3124 session left to resume into"
3125 ),
3126 );
3127 let outcome = if attempts == 0 {
3128 ContinuationOutcome::NoSession
3129 } else {
3130 ContinuationOutcome::Exhausted
3131 };
3132 return (
3133 seat,
3134 None,
3135 Some(format!("unparsable fix report: {last_err}")),
3136 ContinuationRecord {
3137 attempts,
3138 cumulative_wait_ms,
3139 outcome,
3140 },
3141 );
3142 }
3143 if attempts >= MAX_FIX_CONTINUATIONS {
3144 self.state.event(
3145 "fix",
3146 format!(
3147 "round {round}: fixer's reply still had no adoption report after \
3148 {attempts} continuation(s) ({last_err}); giving up"
3149 ),
3150 );
3151 return (
3152 seat,
3153 None,
3154 Some(format!(
3155 "unparsable fix report after {attempts} continuation(s): {last_err}"
3156 )),
3157 ContinuationRecord {
3158 attempts,
3159 cumulative_wait_ms,
3160 outcome: ContinuationOutcome::Exhausted,
3161 },
3162 );
3163 }
3164 attempts += 1;
3165 self.state.event(
3166 "fix",
3167 format!(
3168 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3169 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3170 ),
3171 );
3172 let mut retry = job.clone();
3173 retry.seat = seat.clone();
3174 retry.prompt = prompt::resume_incomplete(&last_err);
3175 retry.timeout = retry_budget(job.timeout, true);
3176 retry.stem = format!("{}-continue{attempts}", job.stem);
3177 let cache = self.state.config.cache_dir();
3178 let ctx = WaveCtx {
3179 carry_seats: false,
3180 pauses: Some((&self.pause, &self.interrupt)),
3181 run: run_id,
3182 node: "fix",
3183 prompts,
3184 cache: cache.as_deref(),
3185 round: Some(round),
3186 };
3187 let (resumed_seat, resumed_out) = run_one(
3188 retry,
3189 Arc::clone(&self.sem),
3190 &ctx,
3191 &mut self.state,
3192 attempts,
3193 )
3194 .await;
3195 seat = resumed_seat;
3196 match resumed_out {
3197 AgentOutcome::Ok(o) => {
3198 cumulative_wait_ms += o.duration_ms;
3199 match verdict::extract_json::<FixReport>(&o.text) {
3200 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3201 self.state.event(
3202 "fix",
3203 format!(
3204 "round {round}: fixer's adoption report recovered after \
3205 {attempts} continuation(s)"
3206 ),
3207 );
3208 return (
3209 seat,
3210 Some(report),
3211 None,
3212 ContinuationRecord {
3213 attempts,
3214 cumulative_wait_ms,
3215 outcome: ContinuationOutcome::Resumed,
3216 },
3217 );
3218 }
3219 Ok(_) => {
3227 last_err = "the reply parsed, but it reported a command whose own CLI \
3228 never confirmed an exit status"
3229 .to_owned();
3230 }
3231 Err(e) => last_err = e.to_string(),
3232 }
3233 }
3234 AgentOutcome::Quota(o) => {
3235 cumulative_wait_ms += o.duration_ms;
3236 self.state.quota.push(QuotaLoss {
3237 seat: seat.key.clone(),
3238 node: "fix".to_owned(),
3239 at: Timestamp::now(),
3240 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3241 });
3242 self.state.event(
3243 "fix",
3244 format!(
3245 "round {round}: continuation rate limited (quota); not retrying now"
3246 ),
3247 );
3248 return (
3249 seat,
3250 None,
3251 Some("rate limited (quota) while recovering the fix report".to_owned()),
3252 ContinuationRecord {
3253 attempts,
3254 cumulative_wait_ms,
3255 outcome: ContinuationOutcome::QuotaLost,
3256 },
3257 );
3258 }
3259 AgentOutcome::Dropped(o) => {
3260 cumulative_wait_ms += o.duration_ms;
3261 let why = o
3262 .dropped
3263 .as_ref()
3264 .map(|d| d.why.as_str())
3265 .unwrap_or("the CLI ended the stream without delivering its answer");
3266 last_err = format!("the CLI dropped the stream ({why})");
3267 }
3268 AgentOutcome::Failed(e) => last_err = e,
3269 AgentOutcome::ResourceBlocked(e) => {
3272 return (
3273 seat,
3274 None,
3275 Some(format!("the continuation was not started: {e}")),
3276 ContinuationRecord {
3277 attempts: attempts.saturating_sub(1),
3278 cumulative_wait_ms,
3279 outcome: ContinuationOutcome::NoSession,
3280 },
3281 );
3282 }
3283 }
3284 }
3285 }
3286
3287 fn after_implement(&mut self) -> Result<()> {
3288 if !self.state.candidate_preservation.is_empty() {
3289 bail!("candidate preservation incomplete; judging is unsafe");
3290 }
3291 if self.state.leaks.is_empty() {
3293 let cfg = self.state.config.blind.clone();
3294 let mut leaks = Vec::new();
3295 for c in &self.state.candidates {
3296 let Some(patch) =
3297 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3298 else {
3299 continue;
3300 };
3301 leaks.extend(blind::scan(
3302 &format!("candidate {} patch", c.label),
3303 &patch,
3304 &cfg.vendor_tokens,
3305 ));
3306 }
3307 if !leaks.is_empty() {
3308 let summary = leaks
3309 .iter()
3310 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3311 .collect::<Vec<_>>()
3312 .join(", ");
3313 match cfg.on_leak {
3314 LeakPolicy::Fail => {
3315 self.state.status = RunStatus::Failed;
3316 self.state
3317 .event("blind", format!("vendor text in a patch: {summary}"));
3318 self.state.leaks = leaks;
3319 self.state.save()?;
3320 self.settle_questions();
3321 bail!(
3322 "blind.on_leak = \"fail\" and vendor text reached a \
3323 judged patch: {summary}"
3324 );
3325 }
3326 LeakPolicy::Redact => self.state.event(
3327 "blind",
3328 format!("redacting vendor text for judging: {summary}"),
3329 ),
3330 LeakPolicy::Warn => self.state.event(
3331 "blind",
3332 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3333 ),
3334 }
3335 self.state.leaks = leaks;
3336 }
3337 }
3338
3339 if self.state.viable().is_empty() {
3340 if self.state.all_candidates_verified_noop() {
3341 self.state.status = RunStatus::VerifiedNoop;
3352 self.state.save()?;
3353 self.settle_questions();
3354 return Ok(());
3355 }
3356 self.state.status = RunStatus::Failed;
3357 self.state.save()?;
3358 self.settle_questions();
3359 bail!("no candidate produced a change; nothing to judge");
3360 }
3361 self.state.status = RunStatus::Judging;
3362 self.state.save()?;
3363 Ok(())
3364 }
3365
3366 async fn judge(&mut self) -> Result<()> {
3369 let run_id = self.state.id.clone();
3374 let prompts = self.state.config.prompts.clone();
3375 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3376 return Ok(());
3377 }
3378 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3379 if viable.len() == 1 {
3380 self.state.judge_skipped = true;
3387 self.state.event(
3388 "judge",
3389 format!(
3390 "only candidate {} produced a change; judging skipped",
3391 viable[0].label
3392 ),
3393 );
3394 self.state.save()?;
3395 return Ok(());
3396 }
3397 self.state.status = RunStatus::Judging;
3398
3399 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3400 let language = self.state.config.graph.language.clone();
3401 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3402 let sessions = self.state.config.graph.sessions;
3403 let artifacts = agent::artifacts_dir(&self.state.dir());
3404 let root = self.state.worktree_root();
3405 let base_short = short(&self.state.base_commit);
3406
3407 let mut jobs = Vec::new();
3408 let mut orders = Vec::new();
3409 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3411 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3412 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3413 let seat_key = format!("judge-{}", j + 1);
3414 let seat = self.seat(&seat_key, &spec.id);
3415 jobs.push(SeatJob {
3416 prompt: prompt::judge(
3417 &self.state.instruction,
3418 &views,
3419 self.roles.judges.len(),
3420 &base_short,
3421 &language,
3422 ),
3423 spec,
3424 seat,
3425 cwd: root.join(format!("judge-{}", j + 1)),
3426 timeout,
3427 allow_write: false,
3428 sessions,
3429 artifacts: artifacts.clone(),
3430 stem: format!("judge-{}", j + 1),
3431 handover: None,
3432 });
3433 }
3434
3435 self.state.event(
3436 "judge",
3437 format!(
3438 "{} judges ranking {} candidates blind",
3439 jobs.len(),
3440 viable.len()
3441 ),
3442 );
3443 let labels_for_check = labels.clone();
3444 let mut quota_losses = Vec::new();
3445 let cache = self.state.config.cache_dir();
3446 let ctx = WaveCtx {
3447 carry_seats: false,
3448 pauses: Some((&self.pause, &self.interrupt)),
3449 run: &run_id,
3450 node: "judge",
3451 prompts: &prompts,
3452 cache: cache.as_deref(),
3453 round: None,
3454 };
3455 let results = ask_json_wave::<Ranking>(
3456 jobs,
3457 Arc::clone(&self.sem),
3458 self.state.config.graph.retries,
3459 &self.roles.judge_roster,
3460 &ctx,
3461 &mut quota_losses,
3462 &mut self.state,
3463 &move |r: &Ranking| r.validate(&labels_for_check),
3464 )
3465 .await;
3466 if self.state.resource_block.is_some() {
3467 return self.stop_for_resource();
3468 }
3469 self.state.quota.extend(quota_losses);
3470
3471 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3472 let agent_id = seat.agent.clone();
3473 self.state.seats.insert(seat.key.clone(), seat);
3474 let mut record = Judgement {
3475 judge: j + 1,
3476 seat: format!("judge-{}", j + 1),
3477 agent: agent_id,
3478 ranking: Vec::new(),
3479 reasons: BTreeMap::new(),
3480 confidence: None,
3481 strengths: BTreeMap::new(),
3482 order: orders[j].clone(),
3483 failed: None,
3484 duration_ms: 0,
3485 };
3486 match res {
3487 Ok((ranking, out)) => {
3488 record.ranking = ranking.normalized();
3489 record.reasons = ranking.reasons;
3490 record.strengths = ranking.strengths;
3491 record.confidence = ranking.confidence;
3492 record.duration_ms = out.duration_ms;
3493 self.state.event(
3494 "judge",
3495 format!(
3496 "judge {} ranked {}",
3497 j + 1,
3498 record.ranking.iter().collect::<String>()
3499 ),
3500 );
3501 }
3502 Err(e) => {
3503 record.failed = Some(e.to_string());
3504 self.state
3505 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3506 }
3507 }
3508 self.state.judgements.push(record);
3509 self.state.save()?;
3510 }
3511 Ok(())
3512 }
3513
3514 async fn deliberate(&mut self) -> Result<()> {
3517 let run_id = self.state.id.clone();
3522 let prompts = self.state.config.prompts.clone();
3523 if !self.state.deliberation.is_empty() {
3524 return Ok(());
3525 }
3526 let tops: Vec<char> = self
3527 .state
3528 .judgements
3529 .iter()
3530 .filter_map(|j| j.ranking.first().copied())
3531 .collect();
3532 let rounds = self.state.config.graph.deliberate_rounds;
3533 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3534 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3535 self.state.event(
3536 "deliberate",
3537 format!("judges agreed on {} outright; no deliberation", tops[0]),
3538 );
3539 }
3540 self.state.status = RunStatus::Voting;
3541 self.state.save()?;
3542 return Ok(());
3543 }
3544
3545 self.state.status = RunStatus::Deliberating;
3546 self.state.event(
3547 "deliberate",
3548 format!(
3549 "split: first choices were {} — opening {rounds} round(s)",
3550 tops.iter().collect::<String>()
3551 ),
3552 );
3553
3554 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3555 let language = self.state.config.graph.language.clone();
3556 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3557 let sessions = self.state.config.graph.sessions;
3558 let artifacts = agent::artifacts_dir(&self.state.dir());
3559 let root = self.state.worktree_root();
3560 let base_short = short(&self.state.base_commit);
3561
3562 for round in 1..=rounds {
3566 let mut turns: Vec<DeliberationTurn> = Vec::new();
3567 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3568 if self.state.judgements[j].failed.is_some() {
3569 continue;
3570 }
3571 let seat_key = format!("judge-{}", j + 1);
3572 let spec = self.occupant(&seat_key, spec);
3573 let mut seat = self.seat(&seat_key, &spec.id);
3574 let transcript = self.transcript(&turns, j);
3575 let build = |context: Option<&str>| {
3576 prompt::deliberate(
3577 &self.state.instruction,
3578 context,
3579 &transcript,
3580 round,
3581 rounds,
3582 &language,
3583 )
3584 };
3585 let block = self.candidate_block(&viable, &base_short);
3586 let full = build(Some(&block));
3587 let text = if has_context(&spec, &seat, sessions) {
3588 build(None)
3589 } else {
3590 full.clone()
3591 };
3592 let job = SeatJob {
3593 spec,
3594 seat: seat.clone(),
3595 prompt: text,
3596 cwd: root.join(format!("judge-{}", j + 1)),
3597 timeout,
3598 allow_write: false,
3599 sessions,
3600 artifacts: artifacts.clone(),
3601 stem: format!("delib-{round}-judge-{}", j + 1),
3602 handover: Some(full),
3603 };
3604 let cache = self.state.config.cache_dir();
3605 let ctx = WaveCtx {
3606 carry_seats: false,
3607 pauses: Some((&self.pause, &self.interrupt)),
3608 run: &run_id,
3609 node: "deliberate",
3610 prompts: &prompts,
3611 cache: cache.as_deref(),
3612 round: None,
3613 };
3614 let mut losses = Vec::new();
3618 let mut results = ask_wave_with::<String>(
3619 vec![job],
3620 Arc::clone(&self.sem),
3621 0,
3622 &self.roles.judge_roster,
3623 &ctx,
3624 &mut losses,
3625 &mut self.state,
3626 &|text: &str| {
3627 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3628 },
3629 )
3630 .await;
3631 self.state.quota.extend(losses);
3632 let (updated, res, _) = results.pop().expect("one job in, one result out");
3633 seat = updated;
3634 let agent_id = seat.agent.clone();
3635 self.state.seats.insert(seat.key.clone(), seat);
3636 let body = match res {
3637 Ok((body, _)) => body,
3638 Err(e) => {
3641 self.state
3642 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3643 continue;
3644 }
3645 };
3646 let tentative = verdict::extract_json::<Position>(&body)
3647 .ok()
3648 .and_then(|p| p.tentative)
3649 .and_then(|s| s.trim().chars().next())
3650 .map(|c| c.to_ascii_uppercase());
3651 self.state.event(
3652 "deliberate",
3653 format!(
3654 "round {round}: judge {} now favours {}",
3655 j + 1,
3656 tentative.map_or("—".to_owned(), |c| c.to_string())
3657 ),
3658 );
3659 turns.push(DeliberationTurn {
3660 judge: j + 1,
3661 agent: agent_id,
3662 body: blind::sanitize_prose(&body, &self.state.config.blind),
3663 tentative,
3664 });
3665 }
3666 self.state
3667 .deliberation
3668 .push(DeliberationRound { round, turns });
3669 self.state.save()?;
3670 }
3671
3672 self.state.status = RunStatus::Voting;
3673 self.state.save()?;
3674 Ok(())
3675 }
3676
3677 async fn vote(&mut self) -> Result<()> {
3680 let run_id = self.state.id.clone();
3685 let prompts = self.state.config.prompts.clone();
3686 if !self.state.votes.is_empty() {
3687 return Ok(());
3688 }
3689 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3690 if viable.len() == 1 {
3691 return Ok(());
3692 }
3693 self.state.status = RunStatus::Voting;
3694
3695 let language = self.state.config.graph.language.clone();
3696 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3697 let sessions = self.state.config.graph.sessions;
3698 let artifacts = agent::artifacts_dir(&self.state.dir());
3699 let root = self.state.worktree_root();
3700 let base_short = short(&self.state.base_commit);
3701 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3702
3703 let mut jobs = Vec::new();
3704 let mut seats_at = Vec::new();
3705 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3706 if self
3707 .state
3708 .judgements
3709 .get(j)
3710 .is_some_and(|r| r.failed.is_some())
3711 {
3712 continue;
3713 }
3714 let seat_key = format!("judge-{}", j + 1);
3715 let spec = self.occupant(&seat_key, spec);
3716 let seat = self.seat(&seat_key, &spec.id);
3717 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3718 let text = if has_context(&spec, &seat, sessions) {
3719 prompt::final_vote(&viable, &language)
3720 } else {
3721 full.clone()
3722 };
3723 jobs.push(SeatJob {
3724 spec,
3725 seat,
3726 prompt: text,
3727 cwd: root.join(format!("judge-{}", j + 1)),
3728 timeout,
3729 allow_write: false,
3730 sessions,
3731 artifacts: artifacts.clone(),
3732 stem: format!("vote-judge-{}", j + 1),
3733 handover: Some(full),
3734 });
3735 seats_at.push(j);
3736 }
3737
3738 self.state.event(
3739 "vote",
3740 format!(
3741 "collecting {} final votes one by one, privately",
3742 jobs.len()
3743 ),
3744 );
3745 let allowed = viable.clone();
3746 let mut quota_losses = Vec::new();
3747 let cache = self.state.config.cache_dir();
3748 let ctx = WaveCtx {
3749 carry_seats: false,
3750 pauses: Some((&self.pause, &self.interrupt)),
3751 run: &run_id,
3752 node: "vote",
3753 prompts: &prompts,
3754 cache: cache.as_deref(),
3755 round: None,
3756 };
3757 let results = ask_json_wave::<FinalVote>(
3758 jobs,
3759 Arc::clone(&self.sem),
3760 self.state.config.graph.retries,
3761 &self.roles.judge_roster,
3762 &ctx,
3763 &mut quota_losses,
3764 &mut self.state,
3765 &move |v: &FinalVote| match v.label() {
3766 Some(c) if allowed.contains(&c) => Ok(()),
3767 other => bail!("vote {other:?} is not one of {allowed:?}"),
3768 },
3769 )
3770 .await;
3771 if self.state.resource_block.is_some() {
3772 return self.stop_for_resource();
3773 }
3774 self.state.quota.extend(quota_losses);
3775
3776 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3777 let agent_id = seat.agent.clone();
3778 self.state.seats.insert(seat.key.clone(), seat);
3779 let initial = self
3780 .state
3781 .judgements
3782 .get(j)
3783 .and_then(|r| r.ranking.first().copied());
3784 let mut record = VoteRecord {
3785 judge: j + 1,
3786 agent: agent_id,
3787 vote: None,
3788 reason: String::new(),
3789 changed: false,
3790 };
3791 match res {
3792 Ok((v, _)) => {
3793 record.vote = v.label();
3794 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3795 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3796 self.state.event(
3797 "vote",
3798 format!(
3799 "judge {} voted {}{}",
3800 j + 1,
3801 record.vote.unwrap_or('?'),
3802 if record.changed { " (changed)" } else { "" }
3803 ),
3804 );
3805 }
3806 Err(e) => {
3807 self.state
3808 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3809 }
3810 }
3811 self.state.votes.push(record);
3812 self.state.save()?;
3813 }
3814 Ok(())
3815 }
3816
3817 fn tally(&mut self) -> Result<()> {
3820 if self.state.tally.is_some() {
3821 return Ok(());
3822 }
3823 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3824 let tops: Vec<char> = self
3825 .state
3826 .judgements
3827 .iter()
3828 .filter_map(|j| j.ranking.first().copied())
3829 .collect();
3830 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3831
3832 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3835 let mut cast: Vec<char> = Vec::new();
3836 for (i, j) in self.state.judgements.iter().enumerate() {
3837 let vote = self
3838 .state
3839 .votes
3840 .iter()
3841 .find(|v| v.judge == i + 1)
3842 .and_then(|v| v.vote)
3843 .or_else(|| j.ranking.first().copied());
3844 if let Some(v) = vote {
3845 *first_choice.entry(v).or_insert(0) += 1;
3846 cast.push(v);
3847 }
3848 }
3849
3850 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3851 for j in &self.state.judgements {
3852 let n = j.ranking.len();
3853 for (pos, label) in j.ranking.iter().enumerate() {
3854 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3855 }
3856 }
3857
3858 let best = first_choice.values().copied().max().unwrap_or(0);
3859 let mut leaders: Vec<char> = first_choice
3860 .iter()
3861 .filter(|(_, v)| **v == best)
3862 .map(|(k, _)| *k)
3863 .collect();
3864 let mut tie_break = None;
3865 if leaders.len() > 1 {
3866 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3867 let borda_leaders: Vec<char> = leaders
3868 .iter()
3869 .copied()
3870 .filter(|l| borda[l] == top_borda)
3871 .collect();
3872 tie_break = Some(if borda_leaders.len() == 1 {
3873 format!(
3874 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3875 leaders.len()
3876 )
3877 } else {
3878 format!(
3879 "{} way tie on both first-choice votes and Borda points, broken by label order",
3880 leaders.len()
3881 )
3882 });
3883 leaders = borda_leaders;
3884 leaders.sort_unstable();
3885 }
3886 let winner = *leaders
3887 .first()
3888 .or(viable.first())
3889 .context("no candidate to declare a winner from")?;
3890
3891 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3892 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3893 let deliberated = !self.state.deliberation.is_empty();
3894
3895 let quota_seats: std::collections::BTreeSet<&str> =
3899 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3900 let mut present = 0usize;
3901 for (i, j) in self.state.judgements.iter().enumerate() {
3902 if quota_seats.contains(j.seat.as_str()) {
3903 continue;
3904 }
3905 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3906 let voted = self
3907 .state
3908 .votes
3909 .iter()
3910 .any(|v| v.judge == i + 1 && v.vote.is_some());
3911 if ranked || voted {
3912 present += 1;
3913 }
3914 }
3915 let needs_quorum = viable.len() > 1;
3921 let judges_total = if needs_quorum {
3922 self.roles.judges.len()
3923 } else {
3924 0
3925 };
3926 let quorum = if needs_quorum {
3927 judges_total / 2 + 1
3928 } else {
3929 0
3930 };
3931 let met_quorum = !needs_quorum || present >= quorum;
3932 let uncontested = (!needs_quorum).then(|| {
3933 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3934 });
3935
3936 self.state.event(
3937 "tally",
3938 match &uncontested {
3939 Some(reason) => format!("winner {winner} — {reason}"),
3940 None => format!(
3941 "winner {winner} — votes {} | initial {} | {} changed | \
3942 {present}/{judges_total} judges{}",
3943 first_choice
3944 .iter()
3945 .map(|(k, v)| format!("{k}:{v}"))
3946 .collect::<Vec<_>>()
3947 .join(" "),
3948 if unanimous_initial {
3949 "unanimous"
3950 } else {
3951 "split"
3952 },
3953 changed_votes,
3954 if met_quorum {
3955 String::new()
3956 } else {
3957 format!(" — below quorum ({quorum} required)")
3958 },
3959 ),
3960 },
3961 );
3962 if !met_quorum {
3963 self.state.event(
3964 "stall",
3965 format!(
3966 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3967 the run stops here, resumable"
3968 ),
3969 );
3970 }
3971 self.state.tally = Some(Tally {
3972 first_choice,
3973 borda,
3974 winner,
3975 rankings: tops.len(),
3976 unanimous_initial,
3977 deliberated,
3978 changed_votes,
3979 unanimous_final,
3980 tie_break,
3981 judges: judges_total,
3982 present,
3983 quorum,
3984 met_quorum,
3985 uncontested,
3986 });
3987 self.state.status = if met_quorum {
3988 RunStatus::Reviewing
3989 } else {
3990 RunStatus::Stalled
3991 };
3992 self.state.save()?;
3993 Ok(())
3994 }
3995
3996 #[allow(clippy::too_many_lines)]
4017 async fn recover_stall(&mut self) -> Result<bool> {
4018 let run_id = self.state.id.clone();
4023 let prompts = self.state.config.prompts.clone();
4024 let quota_seats: BTreeSet<&str> =
4029 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4030 let absent: Vec<String> = self
4031 .state
4032 .judgements
4033 .iter()
4034 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4035 .map(|j| j.seat.clone())
4036 .collect();
4037 if absent.is_empty() {
4038 return Ok(false);
4039 }
4040 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4041 if viable.len() <= 1 {
4042 return Ok(false);
4043 }
4044 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4045 let language = self.state.config.graph.language.clone();
4046 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4047 let sessions = self.state.config.graph.sessions;
4048 let artifacts = agent::artifacts_dir(&self.state.dir());
4049 let root = self.state.worktree_root();
4050 let base_short = short(&self.state.base_commit);
4051 let candidates: Vec<Candidate> = viable.clone();
4052
4053 let mut positions: Vec<usize> = absent
4055 .iter()
4056 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4057 .collect();
4058 if positions.is_empty() {
4059 return Ok(false);
4060 }
4061 positions.sort_unstable();
4062 positions.dedup();
4063
4064 let mut judge_jobs = Vec::new();
4066 for &j in &positions {
4067 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4068 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4069 let seat_key = format!("judge-{}", j + 1);
4070 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4071 let seat = self.seat(&seat_key, &spec.id);
4072 judge_jobs.push(SeatJob {
4073 spec,
4074 seat,
4075 prompt: prompt::judge(
4076 &self.state.instruction,
4077 &views,
4078 self.roles.judges.len(),
4079 &base_short,
4080 &language,
4081 ),
4082 cwd: root.join(seat_key),
4083 timeout,
4084 allow_write: false,
4085 sessions,
4086 artifacts: artifacts.clone(),
4087 stem: format!("judge-{}-recover", j + 1),
4088 handover: None,
4089 });
4090 }
4091
4092 let labels_for_check = labels.clone();
4093 let mut judge_losses = Vec::new();
4094 let retries = self.state.config.graph.retries;
4095 let cache = self.state.config.cache_dir();
4096 let ctx = WaveCtx {
4097 carry_seats: false,
4098 pauses: Some((&self.pause, &self.interrupt)),
4099 run: &run_id,
4100 node: "judge",
4101 prompts: &prompts,
4102 cache: cache.as_deref(),
4103 round: None,
4104 };
4105 let results = ask_json_wave::<Ranking>(
4106 judge_jobs,
4107 Arc::clone(&self.sem),
4108 retries,
4109 &self.roles.judge_roster,
4110 &ctx,
4111 &mut judge_losses,
4112 &mut self.state,
4113 &move |r: &Ranking| r.validate(&labels_for_check),
4114 )
4115 .await;
4116 if self.state.resource_block.is_some() {
4117 self.stop_for_resource()?;
4118 return Ok(false);
4119 }
4120
4121 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4123 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4124 let agent_id = seat.agent.clone();
4125 self.state.seats.insert(seat.key.clone(), seat);
4126 let record = &mut self.state.judgements[j];
4127 match res {
4128 Ok((ranking, out)) => {
4129 record.agent = agent_id;
4130 record.ranking = ranking.normalized();
4131 record.reasons = ranking.reasons;
4132 record.strengths = ranking.strengths;
4133 record.confidence = ranking.confidence;
4134 record.failed = None;
4135 record.duration_ms = out.duration_ms;
4136 recovered.insert(j);
4137 self.state.event(
4138 "recover",
4139 format!("judge {} ranked again after the limit", j + 1),
4140 );
4141 }
4142 Err(e) => {
4143 self.state
4144 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4145 }
4146 }
4147 }
4148
4149 let mut vote_jobs = Vec::new();
4151 let mut vote_pos: Vec<usize> = Vec::new();
4152 for &j in &recovered {
4153 let seat_key = format!("judge-{}", j + 1);
4154 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4155 let seat = self.seat(&seat_key, &spec.id);
4156 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4157 let text = if has_context(&spec, &seat, sessions) {
4158 prompt::final_vote(&labels, &language)
4159 } else {
4160 full.clone()
4161 };
4162 vote_jobs.push(SeatJob {
4163 spec,
4164 seat,
4165 prompt: text,
4166 cwd: root.join(seat_key),
4167 timeout,
4168 allow_write: false,
4169 sessions,
4170 artifacts: artifacts.clone(),
4171 stem: format!("vote-judge-{}-recover", j + 1),
4172 handover: Some(full),
4173 });
4174 vote_pos.push(j);
4175 }
4176 let allowed = labels.clone();
4177 let mut vote_losses = Vec::new();
4178 let vote_retries = self.state.config.graph.retries;
4179 let vote_cache = self.state.config.cache_dir();
4180 let ctx = WaveCtx {
4181 carry_seats: false,
4182 pauses: Some((&self.pause, &self.interrupt)),
4183 run: &run_id,
4184 node: "vote",
4185 prompts: &prompts,
4186 cache: vote_cache.as_deref(),
4187 round: None,
4188 };
4189 let votes = ask_json_wave::<FinalVote>(
4190 vote_jobs,
4191 Arc::clone(&self.sem),
4192 vote_retries,
4193 &self.roles.judge_roster,
4194 &ctx,
4195 &mut vote_losses,
4196 &mut self.state,
4197 &move |v: &FinalVote| match v.label() {
4198 Some(c) if allowed.contains(&c) => Ok(()),
4199 other => bail!("vote {other:?} is not one of {allowed:?}"),
4200 },
4201 )
4202 .await;
4203 if self.state.resource_block.is_some() {
4204 self.stop_for_resource()?;
4205 return Ok(false);
4206 }
4207 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4208 let agent_id = seat.agent.clone();
4209 self.state.seats.insert(seat.key.clone(), seat);
4210 match res {
4211 Ok((v, _)) => {
4212 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4213 rec.vote = v.label();
4214 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4215 } else {
4216 self.state.votes.push(VoteRecord {
4217 judge: j + 1,
4218 agent: agent_id,
4219 vote: v.label(),
4220 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4221 changed: false,
4222 });
4223 }
4224 self.state.event(
4225 "recover",
4226 format!("judge {} voted again after the limit", j + 1),
4227 );
4228 }
4229 Err(e) => {
4230 self.state
4231 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4232 }
4233 }
4234 }
4235
4236 let recovered_keys: BTreeSet<String> = recovered
4240 .iter()
4241 .map(|&j| format!("judge-{}", j + 1))
4242 .collect();
4243 self.state
4244 .quota
4245 .retain(|q| !recovered_keys.contains(&q.seat));
4246 for loss in judge_losses.into_iter().chain(vote_losses) {
4250 if recovered_keys.contains(&loss.seat) {
4251 continue;
4252 }
4253 self.state.quota.retain(|q| q.seat != loss.seat);
4254 self.state.quota.push(loss);
4255 }
4256
4257 self.state.tally = None;
4259 self.tally()?;
4260 Ok(self
4261 .state
4262 .tally
4263 .as_ref()
4264 .map(|t| t.met_quorum)
4265 .unwrap_or(false))
4266 }
4267
4268 async fn graft(&mut self) -> Result<()> {
4284 let Err(e) = self.graft_inner().await else {
4285 return Ok(());
4286 };
4287 if e.is::<crate::repo_lock::ResourceBlocked>() {
4288 return Err(e);
4289 }
4290 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4291 self.state.status = RunStatus::Blocked;
4292 self.state.event("graft", format!("blocked: {e:#}"));
4293 self.state.save()?;
4294 return Err(e);
4295 }
4296 if let Some(rec) = self.state.graft.clone()
4301 && !rec.outcome.settled()
4302 {
4303 if let Some(winner) = self.state.winner().cloned() {
4304 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4305 if re.is::<crate::repo_lock::ResourceBlocked>() {
4306 return Err(re);
4307 }
4308 self.state.status = RunStatus::Blocked;
4309 self.state.event("graft", format!("blocked: {re:#}"));
4310 self.state.save()?;
4311 return Err(re);
4312 }
4313 }
4314 if let Some(g) = self.state.graft.as_mut() {
4315 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4316 }
4317 }
4318 self.state.event("graft", format!("skipped: {e:#}"));
4319 if let Err(e) = self.state.save() {
4320 tracing::warn!("save after graft: {e:#}");
4321 }
4322 Ok(())
4323 }
4324
4325 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4328 let _coordination = self.coordinate(wt).await?;
4329 crate::recovery::restore_repair(wt, pre)
4330 .await
4331 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4332 }
4333
4334 fn graft_skipped(&mut self, why: &str) {
4336 let message = format!("skipped: {why}");
4337 if !self
4338 .state
4339 .events
4340 .iter()
4341 .any(|e| e.node == "graft" && e.message == message)
4342 {
4343 self.state.event("graft", message);
4344 }
4345 }
4346
4347 async fn graft_inner(&mut self) -> Result<()> {
4348 if let Some(rec) = self.state.graft.clone()
4351 && !rec.outcome.settled()
4352 {
4353 if self.state.reviews.is_empty()
4357 && let Some(winner) = self.state.winner().cloned()
4358 {
4359 self.graft_restore(&winner.worktree, &rec.pre).await?;
4360 }
4361 if rec.outcome == GraftOutcome::WaitingForRestore {
4362 self.state.graft = None;
4363 self.state.save()?;
4364 } else {
4365 if let Some(g) = self.state.graft.as_mut() {
4366 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4367 }
4368 self.state.event(
4369 "graft",
4370 "rolled back: the previous round was interrupted; the winner is untouched",
4371 );
4372 self.state.save()?;
4373 return Ok(());
4374 }
4375 }
4376 let plan = match graft::plan(&self.state) {
4377 Ok(plan) => plan,
4378 Err(skip) => {
4379 if skip != graft::Skip::AlreadyDone {
4380 self.graft_skipped(skip.reason());
4381 self.state.save()?;
4382 }
4383 return Ok(());
4384 }
4385 };
4386 let Some(winner) = self.state.winner().cloned() else {
4387 return Ok(());
4388 };
4389 let wt = winner.worktree.clone();
4390 if !git::is_clean(&wt).await? {
4391 self.graft_skipped("the winner's worktree has uncommitted changes");
4392 self.state.save()?;
4393 return Ok(());
4394 }
4395 let pre = git::rev_parse(&wt, "HEAD").await?;
4396 self.state.graft = Some(GraftRecord {
4399 pre: pre.clone(),
4400 post: None,
4401 lines: None,
4402 limit: None,
4403 agent: None,
4404 outcome: GraftOutcome::Running,
4405 requested: graft::requested(&plan),
4406 adoption: None,
4407 adoption_error: None,
4408 });
4409 self.state.event(
4410 "graft",
4411 format!(
4412 "porting strengths of candidate(s) {} onto {}",
4413 plan.strengths
4414 .iter()
4415 .map(|(l, _, _)| l.to_string())
4416 .collect::<Vec<_>>()
4417 .join(", "),
4418 plan.winner
4419 ),
4420 );
4421 self.state.save()?;
4422
4423 let fix_prompt = prompt::graft(
4424 &self.state.instruction,
4425 plan.winner,
4426 &plan.strengths,
4427 &self.state.config.graph.language,
4428 );
4429 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4430 let sessions = self.state.config.graph.sessions;
4431 let artifacts = agent::artifacts_dir(&self.state.dir());
4432 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4433 let (_, seat, out) = self
4434 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4435 SeatJob {
4436 prompt: fix_prompt.clone(),
4437 spec,
4438 seat,
4439 cwd: wt.clone(),
4440 timeout,
4441 allow_write: true,
4442 sessions,
4443 artifacts: artifacts.clone(),
4444 stem: "graft".to_owned(),
4445 handover: None,
4446 }
4447 })
4448 .await;
4449 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4450 self.state.graft.as_mut().expect("graft record").outcome =
4453 GraftOutcome::WaitingForRestore;
4454 return self.stop_for_resource();
4455 }
4456 let mut why: Option<String> = None;
4457 match &out {
4458 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4459 why = Some("the fixer produced nothing usable".to_owned());
4460 }
4461 AgentOutcome::Ok(_) => {}
4462 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4463 AgentOutcome::Quota(o) => {
4464 self.state.quota.push(QuotaLoss {
4465 seat: seat.key.clone(),
4466 node: "graft".to_owned(),
4467 at: Timestamp::now(),
4468 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4469 });
4470 why = Some("rate limited (quota)".to_owned());
4471 }
4472 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4473 why = Some(format!(
4474 "the fixer failed: {}",
4475 e.lines()
4476 .next()
4477 .unwrap_or("")
4478 .chars()
4479 .take(160)
4480 .collect::<String>()
4481 ));
4482 }
4483 }
4484 let agent_id = seat.agent.clone();
4485 self.state.seats.insert(seat.key.clone(), seat);
4486
4487 let mut post = None;
4488 let mut lines = None;
4489 let _coordination = self.coordinate(&wt).await?;
4490 if why.is_none() {
4491 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4492 if let Ok(r) = &rescued {
4493 self.state.note_withheld("graft", &r.withheld);
4494 }
4495 let head = git::rev_parse(&wt, "HEAD").await?;
4496 let dirty = match &rescued {
4500 Ok(r) => git::status_porcelain(&wt)
4501 .await
4502 .map(|st| {
4503 st.lines().any(|l| {
4504 !r.withheld
4505 .iter()
4506 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4507 })
4508 })
4509 .unwrap_or(true),
4510 Err(_) => true,
4511 };
4512 if dirty {
4513 why = Some("the fixer left changes that could not be committed".to_owned());
4514 } else if head == pre {
4515 why = Some("the fixer changed nothing".to_owned());
4516 } else {
4517 lines = git::changed_lines(&wt, &pre, &head).await;
4518 post = Some(head);
4519 }
4520 }
4521 let (adoption, adoption_error) = match &out {
4522 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4523 _ => (None, None),
4524 };
4525 let requested = graft::requested(&plan);
4526 let outcome = match why {
4527 None => {
4528 self.state.event(
4529 "graft",
4530 format!(
4531 "applied: {}",
4532 lines.map_or_else(
4533 || "changed lines unavailable".to_owned(),
4534 |n| format!("{n} changed lines")
4535 )
4536 ),
4537 );
4538 GraftOutcome::Applied
4539 }
4540 Some(why) => {
4541 crate::recovery::restore_repair(&wt, &pre)
4544 .await
4545 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4546 self.state.event(
4547 "graft",
4548 format!("rolled back: {why}; the winner is untouched"),
4549 );
4550 GraftOutcome::RolledBack(why)
4551 }
4552 };
4553 self.state.graft = Some(GraftRecord {
4554 pre,
4555 post,
4556 lines,
4557 limit: None,
4558 agent: Some(agent_id),
4559 outcome,
4560 requested,
4561 adoption,
4562 adoption_error,
4563 });
4564 self.state.save()?;
4565 Ok(())
4566 }
4567
4568 async fn fold_losers(&mut self) -> Result<()> {
4571 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4572 return Ok(());
4573 };
4574 if !self.state.candidate_preservation.is_empty() {
4575 bail!("candidate preservation incomplete; refusing cleanup");
4576 }
4577 let repo = self.state.repo.clone();
4578 let _coordination = self.coordinate(&repo).await?;
4579 let mut folded = Vec::new();
4580 for i in 0..self.state.candidates.len() {
4581 let c = &self.state.candidates[i];
4582 if c.label == winner || c.folded {
4583 continue;
4584 }
4585 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4586 remove_preserved_worktree(&repo, &wt).await?;
4587 if git::branch_exists(&repo, &branch).await?
4588 && !git::branch_delete(&repo, &branch).await?
4589 {
4590 bail!("candidate branch removal refused");
4591 }
4592 self.state.candidates[i].folded = true;
4593 self.state.save()?;
4594 folded.push(label.to_string());
4595 }
4596 let root = self.state.worktree_root();
4598 for j in 1..=self.roles.judges.len() {
4599 let wt = root.join(format!("judge-{j}"));
4600 if wt.exists() {
4601 remove_preserved_worktree(&repo, &wt).await?;
4602 }
4603 }
4604 if self.state.config.graph.advise {
4607 for k in 1..=self.state.config.graph.advisors {
4608 let wt = root.join(format!("advisor-{k}"));
4609 if wt.exists() {
4610 remove_preserved_worktree(&repo, &wt).await?;
4611 }
4612 }
4613 }
4614 if !folded.is_empty() {
4615 self.state
4616 .event("fold", format!("folded candidates {}", folded.join(", ")));
4617 self.state.save()?;
4618 }
4619 Ok(())
4620 }
4621
4622 async fn sync_to_base(&mut self) -> Result<()> {
4662 if self.state.status == RunStatus::AlreadyInBase {
4663 return Ok(());
4664 }
4665 let conflicted = self
4666 .state
4667 .base_sync
4668 .as_ref()
4669 .is_some_and(|s| s.conflict.is_some());
4670 let Some(winner) = self.state.winner().cloned() else {
4671 return Ok(());
4672 };
4673
4674 let repo = self.state.repo.clone();
4675 let remote = self.state.config.merge.remote.clone();
4676 let base_branch = self.state.base_branch.clone();
4677 let tracking = format!("{remote}/{base_branch}");
4678
4679 git::fetch(&repo, &remote, &base_branch).await.ok();
4680 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4685 return Ok(());
4686 };
4687
4688 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4689 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4690 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4691
4692 if (behind > 0 || conflicted || head == tip)
4701 && self
4702 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4703 .await?
4704 {
4705 return Ok(());
4706 }
4707 if conflicted {
4708 return Ok(());
4709 }
4710
4711 if behind == 0 {
4712 if let Some(from) = self
4721 .state
4722 .rebase_fixes
4723 .iter()
4724 .rev()
4725 .find_map(|r| r.from.clone())
4726 && from != head
4727 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4728 .await
4729 .is_ok_and(|o| o.ok())
4730 {
4731 git::sync_to_head(&winner.worktree).await?;
4732 }
4733 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4739 if let Some(why) = &conflict {
4740 self.state.status = RunStatus::Blocked;
4741 self.state.event("land", why.clone());
4742 }
4743 self.state.base_sync = Some(BaseSync {
4744 tip,
4745 behind: 0,
4746 attempts,
4747 conflict,
4748 already_in: None,
4749 });
4750 self.state.save()?;
4751 return Ok(());
4752 }
4753
4754 if attempts >= BASE_SYNC_ROUNDS {
4755 let why = format!(
4756 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4757 rebase(s); rebasing again would only race it",
4758 winner.branch
4759 );
4760 self.state.status = RunStatus::Blocked;
4761 self.state.base_sync = Some(BaseSync {
4762 tip,
4763 behind,
4764 attempts,
4765 conflict: Some(why.clone()),
4766 already_in: None,
4767 });
4768 self.state.event("land", why);
4769 self.state.save()?;
4770 return Ok(());
4771 }
4772
4773 self.state.event(
4774 "land",
4775 format!(
4776 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4777 winner.branch
4778 ),
4779 );
4780 self.state.save()?;
4781
4782 let branch_tracking = format!("{remote}/{}", winner.branch);
4788 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4789 let remote_tip = if self.state.task_pr_branch().is_none()
4790 && matches!(&fetched_branch, Ok(o) if o.ok())
4791 {
4792 git::rev_parse(&repo, &branch_tracking).await.ok()
4793 } else {
4794 None
4795 };
4796 if let Some(theirs) = &remote_tip
4800 && !git::is_ancestor(&repo, theirs, &head).await
4801 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4802 .await
4803 .is_ok_and(|missing| missing.is_empty())
4804 {
4805 let why = format!(
4806 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4807 them",
4808 short(theirs),
4809 winner.branch
4810 );
4811 self.state.status = RunStatus::Blocked;
4812 self.state.base_sync = Some(BaseSync {
4813 tip,
4814 behind,
4815 attempts,
4816 conflict: Some(why.clone()),
4817 already_in: None,
4818 });
4819 self.state.event("land", why);
4820 self.state.save()?;
4821 return Ok(());
4822 }
4823
4824 let scratch = self.state.dir().join("base-sync");
4825 let rebased = match crate::rebase::rebase_with_fixer(
4826 &mut self.state,
4827 &scratch,
4828 &winner.branch,
4829 &tracking,
4830 )
4831 .await
4832 {
4833 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4834 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4835 Err(e) => Err(e),
4836 };
4837 let attempts = attempts + 1;
4838 match rebased {
4839 Ok(None) => {
4840 git::sync_to_head(&winner.worktree).await?;
4844 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4845 let mut conflict = None;
4846 if let Some(pinned) = &remote_tip {
4847 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4848 match pushed {
4849 Ok(o) if o.ok() => self.state.event(
4850 "land",
4851 format!("pushed rebased {} to {remote}", winner.branch),
4852 ),
4853 Ok(o) => {
4854 conflict = Some(format!(
4855 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4856 winner.branch,
4857 short(pinned),
4858 o.stderr.chars().take(600).collect::<String>()
4859 ));
4860 }
4861 Err(e) => {
4862 conflict = Some(format!(
4863 "rebased {} locally but could not push it: {e:#}",
4864 winner.branch
4865 ));
4866 }
4867 }
4868 }
4869 if let Some(why) = &conflict {
4870 self.state.status = RunStatus::Blocked;
4871 self.state.event("land", why.clone());
4872 }
4873 self.state.base_sync = Some(BaseSync {
4874 tip: tip.clone(),
4875 behind: 0,
4876 attempts,
4877 conflict,
4878 already_in: None,
4879 });
4880 self.state
4881 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4882 }
4883 Ok(Some(conflict)) => {
4884 let why = format!(
4885 "{} conflicts with {tracking} and did not rebase: {}",
4886 winner.branch,
4887 conflict.chars().take(600).collect::<String>()
4888 );
4889 self.state.status = RunStatus::Blocked;
4890 self.state.base_sync = Some(BaseSync {
4891 tip,
4892 behind,
4893 attempts,
4894 conflict: Some(why.clone()),
4895 already_in: None,
4896 });
4897 self.state.event("land", why);
4898 }
4899 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4903 self.state.save()?;
4904 return Ok(());
4905 }
4906 Err(e) => {
4907 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4908 self.state.status = RunStatus::Blocked;
4909 self.state.base_sync = Some(BaseSync {
4910 tip,
4911 behind,
4912 attempts,
4913 conflict: Some(why.clone()),
4914 already_in: None,
4915 });
4916 self.state.event("land", why);
4917 }
4918 }
4919 self.state.save()?;
4920 Ok(())
4921 }
4922
4923 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4929 if self.state.task_pr_branch().is_some() {
4932 return None;
4933 }
4934 let repo = self.state.repo.clone();
4935 let remote = self.state.config.merge.remote.clone();
4936 let fetched = git::fetch(&repo, &remote, branch).await;
4937 if !matches!(&fetched, Ok(o) if o.ok()) {
4938 return None;
4939 }
4940 let branch_tracking = format!("{remote}/{branch}");
4941 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4942 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4943 return None;
4944 }
4945 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4946 .await
4947 .is_ok_and(|missing| missing.is_empty())
4948 {
4949 return Some(format!(
4950 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4951 them",
4952 short(&theirs)
4953 ));
4954 }
4955 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4956 Ok(o) if o.ok() => {
4957 self.state
4958 .event("land", format!("pushed rebased {branch} to {remote}"));
4959 None
4960 }
4961 Ok(o) => Some(format!(
4962 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4963 someone may have pushed): {}",
4964 short(&theirs),
4965 o.stderr.chars().take(600).collect::<String>()
4966 )),
4967 Err(e) => Some(format!(
4968 "{branch} is rebased locally but could not be pushed: {e:#}"
4969 )),
4970 }
4971 }
4972
4973 async fn settle_already_in(
4990 &mut self,
4991 branch: &str,
4992 tip: &str,
4993 head: &str,
4994 attempts: usize,
4995 behind: usize,
4996 ) -> Result<bool> {
4997 let repo = self.state.repo.clone();
4998 let remote = self.state.config.merge.remote.clone();
4999 let start = self.state.base_commit.clone();
5000 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5001 Ok(Some(e)) => e,
5002 Ok(None) => return Ok(false),
5003 Err(e) => {
5004 tracing::warn!("already-in-base check for {branch}: {e:#}");
5005 return Ok(false);
5006 }
5007 };
5008 let mut verified = vec![head.to_owned()];
5009 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5010 let fetched = git::fetch(&repo, &remote, &public_branch).await;
5011 if matches!(&fetched, Ok(o) if o.ok())
5012 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5013 && theirs != head
5014 {
5015 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5016 Ok(Some(_)) => verified.push(theirs),
5017 _ => return Ok(false),
5018 }
5019 }
5020 let base_branch = self.state.base_branch.clone();
5021 let message = format!(
5022 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
5023 land",
5024 evidence.names(),
5025 evidence.proof.as_str()
5026 );
5027 let closed =
5028 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
5029 .await;
5030 match closed {
5031 Ok(Ok(url)) => self
5032 .state
5033 .event("land", format!("closed {url}: superseded on {base_branch}")),
5034 Ok(Err(why)) => self
5035 .state
5036 .event("land", format!("did not close a pull request: {why}")),
5037 Err(e) => {
5038 let why = format!(
5039 "{branch} is already in {remote}/{base_branch}, but its pull request could \
5040 not be closed ({e:#}); resume to retry"
5041 );
5042 self.state.status = RunStatus::Blocked;
5043 self.state.base_sync = Some(BaseSync {
5044 tip: tip.to_owned(),
5045 behind,
5046 attempts,
5047 conflict: Some(why.clone()),
5048 already_in: None,
5049 });
5050 self.state.event("land", why);
5051 self.state.save()?;
5052 return Ok(true);
5053 }
5054 }
5055 self.state.status = RunStatus::AlreadyInBase;
5056 self.state.base_sync = Some(BaseSync {
5057 tip: tip.to_owned(),
5058 behind,
5059 attempts,
5060 conflict: None,
5061 already_in: Some(evidence),
5062 });
5063 self.state.event("land", message);
5064 self.state.save()?;
5065 self.settle_questions();
5066 Ok(true)
5067 }
5068
5069 fn landing_base(&self) -> String {
5079 self.state
5080 .base_sync
5081 .as_ref()
5082 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5083 }
5084
5085 pub async fn fix_selected(
5118 &mut self,
5119 ids: &[String],
5120 reason: &str,
5121 allow_stale: bool,
5122 ) -> Result<()> {
5123 let reason = reason.trim();
5124 if reason.is_empty() {
5125 bail!("a fix request needs a reason — that is the operator's own record of why");
5126 }
5127 if ids.is_empty() {
5128 bail!("no finding id given");
5129 }
5130 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5131 bail!(
5132 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5133 has already concluded — can be given a targeted fix. A run still \
5134 in progress should simply be resumed; a `merged` run's branch has \
5135 already landed, so its answer is a fresh `magi review <branch>`, \
5136 not reopening this run's own record",
5137 self.state.id,
5138 self.state.status.as_str()
5139 );
5140 }
5141 let Some(winner) = self.state.winner().cloned() else {
5142 bail!("run {} has no winning candidate to fix", self.state.id);
5143 };
5144 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5145 bail!(
5146 "branch `{}` no longer exists; this run cannot be extended",
5147 winner.branch
5148 );
5149 }
5150 let home = crate::run::home();
5151 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5152 bail!(
5153 "run {} is currently being worked on by another magi process",
5154 self.state.id
5155 );
5156 }
5157 let _claim = FixClaim::acquire(&self.state.dir())?;
5163
5164 let mut seen = BTreeSet::new();
5168 let mut findings = Vec::new();
5169 let mut missing = Vec::new();
5170 for id in ids {
5171 if !seen.insert(id.clone()) {
5172 continue;
5173 }
5174 match self.state.finding(id) {
5175 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5176 id: f.id.clone(),
5177 severity: f.severity,
5178 reviewer_vote: rec.vote,
5179 round: round.round,
5180 round_head: round.head.clone(),
5181 reviewer: rec.reviewer,
5182 agent: rec.agent.clone(),
5183 file: f.file.clone(),
5184 line: f.line,
5185 title: f.title.clone(),
5186 detail: f.detail.clone(),
5187 outcome: OperatorFixOutcome::Pending,
5188 }),
5189 None => missing.push(id.clone()),
5190 }
5191 }
5192 if !missing.is_empty() {
5193 bail!(
5194 "unknown finding id(s): {}; nothing was changed",
5195 missing.join(", ")
5196 );
5197 }
5198
5199 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5200 let stale_details: Vec<(String, String)> = findings
5201 .iter()
5202 .filter(|f| f.round_head != head_at_request)
5203 .map(|f| (f.id.clone(), f.round_head.clone()))
5204 .collect();
5205 let stale = !stale_details.is_empty();
5206 if stale && !allow_stale {
5207 bail!(
5208 "the branch has moved since some finding(s) were raised — {} — now \
5209 at {}; pass --allow-stale to fix anyway, or re-run review first",
5210 stale_details
5211 .iter()
5212 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5213 .collect::<Vec<_>>()
5214 .join(", "),
5215 short(&head_at_request)
5216 );
5217 }
5218
5219 let request = OperatorFixRequest {
5220 requested_at: Timestamp::now(),
5221 reason: reason.to_owned(),
5222 findings,
5223 head_at_request: head_at_request.clone(),
5224 allow_stale,
5225 stale,
5226 fix: None,
5227 result_head: None,
5228 follow_up_review_run: None,
5229 };
5230 self.state.event(
5231 "fix",
5232 format!(
5233 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5234 request.findings.len(),
5235 request
5236 .findings
5237 .iter()
5238 .map(|f| f.id.as_str())
5239 .collect::<Vec<_>>()
5240 .join(", "),
5241 ),
5242 );
5243 self.state.operator_fixes.push(request);
5250 self.state.save()?;
5251 let request_index = self.state.operator_fixes.len() - 1;
5252
5253 let repo = self.state.repo.clone();
5262 let coordination = self.coordinate(&repo).await?;
5263 if winner.worktree.exists() {
5264 let dirty = git::git(
5267 &winner.worktree,
5268 &["status", "--porcelain", "--untracked-files=all"],
5269 )
5270 .await?;
5271 let only_withheld = dirty.lines().all(|l| {
5272 l.strip_prefix("?? ")
5273 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5274 });
5275 if !only_withheld {
5276 bail!(
5277 "`{}` has uncommitted changes; refusing to touch it — commit or \
5278 discard them first",
5279 winner.worktree.display()
5280 );
5281 }
5282 git::worktree_remove(&self.state.repo, &winner.worktree)
5283 .await
5284 .ok();
5285 }
5286 let fix_worktree = self.state.worktree_root().join("operator-fix");
5287 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5288 git::git(
5289 &self.state.repo,
5290 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5291 )
5292 .await
5293 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5294 if !git::is_clean(&fix_worktree).await? {
5295 git::worktree_remove(&self.state.repo, &fix_worktree)
5296 .await
5297 .ok();
5298 bail!(
5299 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5300 winner.branch
5301 );
5302 }
5303 let repo_for_setup = self.state.repo.clone();
5304 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5305 Ok(()) => {
5306 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5307 }
5308 Err(e) => Err(e),
5309 };
5310 if let Err(e) = setup {
5311 git::worktree_remove(&self.state.repo, &fix_worktree)
5312 .await
5313 .ok();
5314 return Err(e);
5315 }
5316
5317 let run_id = self.state.id.clone();
5318 let prompts = self.state.config.prompts.clone();
5319 let language = self.state.config.graph.language.clone();
5320 let sessions = self.state.config.graph.sessions;
5321 let artifacts = agent::artifacts_dir(&self.state.dir());
5322 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5323 .findings
5324 .iter()
5325 .map(|f| Finding {
5326 id: f.id.clone(),
5327 severity: f.severity,
5328 file: f.file.clone(),
5329 line: f.line,
5330 title: f.title.clone(),
5331 detail: f.detail.clone(),
5332 })
5333 .collect();
5334 let fix_prompt = prompt::operator_fix_in(
5335 &self.state.instruction,
5336 &finding_list,
5337 reason,
5338 &stale_details,
5339 &head_at_request,
5340 &language,
5341 &self.state.config.graph.github_language,
5342 );
5343 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5344 drop(coordination);
5345 let (job, seat, out) = self
5346 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5347 prompt: fix_prompt.clone(),
5348 spec,
5349 seat,
5350 cwd: fix_worktree.clone(),
5351 timeout,
5352 allow_write: true,
5353 sessions,
5354 artifacts: artifacts.clone(),
5355 stem: "operator-fix".to_owned(),
5356 handover: None,
5357 })
5358 .await;
5359 let agent_id = seat.agent.clone();
5360
5361 let mut fix = FixRecord {
5362 agent: agent_id,
5363 addressed: Vec::new(),
5364 rejected: Vec::new(),
5365 notes: String::new(),
5366 committed: false,
5367 failed: None,
5368 duration_ms: 0,
5369 continuation: None,
5370 };
5371 let mut final_seat = seat.clone();
5372 match out {
5373 AgentOutcome::Ok(o) => {
5374 fix.duration_ms = o.duration_ms;
5375 let parsed = verdict::extract_json::<FixReport>(&o.text);
5376 let incomplete_reason = match &parsed {
5377 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5378 "the reply parsed, but it reported a command whose own CLI \
5379 never confirmed an exit status"
5380 .to_owned(),
5381 ),
5382 Ok(_) => None,
5383 Err(e) => Some(e.to_string()),
5384 };
5385 match incomplete_reason {
5386 None => {
5387 let report = parsed.expect("checked Ok above");
5388 fix.addressed = report.addressed;
5389 fix.rejected = report.rejected;
5390 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5391 }
5392 Some(reason) => {
5393 let (resumed_seat, resolved, failure, cont) = self
5394 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5395 .await;
5396 fix.duration_ms += cont.cumulative_wait_ms;
5397 fix.continuation = Some(cont);
5398 final_seat = resumed_seat;
5399 match resolved {
5400 Some(report) => {
5401 fix.addressed = report.addressed;
5402 fix.rejected = report.rejected;
5403 fix.notes =
5404 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5405 }
5406 None => fix.failed = failure,
5407 }
5408 }
5409 }
5410 }
5411 AgentOutcome::Dropped(o) => {
5412 fix.duration_ms = o.duration_ms;
5413 let why = o
5414 .dropped
5415 .as_ref()
5416 .map(|d| d.why.as_str())
5417 .unwrap_or("the CLI ended the stream without delivering its answer");
5418 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5419 }
5420 AgentOutcome::Quota(o) => {
5421 self.state.quota.push(QuotaLoss {
5422 seat: final_seat.key.clone(),
5423 node: "fix".to_owned(),
5424 at: Timestamp::now(),
5425 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5426 });
5427 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5428 }
5429 AgentOutcome::Failed(e) => fix.failed = Some(e),
5430 AgentOutcome::ResourceBlocked(e) => {
5431 fix.failed = Some(format!("the fixer was not started: {e}"));
5432 }
5433 }
5434 if fix.continuation.is_none() {
5435 fix.continuation = Some(ContinuationRecord::not_needed());
5436 }
5437 self.state.seats.insert(final_seat.key.clone(), final_seat);
5438
5439 let rescue_message = format!(
5440 "magi: operator-selected fix ({}) (uncommitted work)",
5441 self.state.operator_fixes[request_index]
5442 .findings
5443 .iter()
5444 .map(|f| f.id.as_str())
5445 .collect::<Vec<_>>()
5446 .join(", ")
5447 );
5448 let repo = self.state.repo.clone();
5449 let coordination = self.coordinate(&repo).await?;
5450 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5451 self.state.note_withheld("fix", &r.withheld);
5452 }
5453 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5454 fix.committed = after != head_at_request;
5455 git::worktree_remove(&self.state.repo, &fix_worktree)
5456 .await
5457 .ok();
5458 drop(coordination);
5459
5460 self.state.event(
5461 "fix",
5462 match &fix.failed {
5463 Some(reason) => format!(
5464 "operator fix: adoption report was lost ({reason}); {}",
5465 if fix.committed {
5466 "committed"
5467 } else {
5468 "NO new commit"
5469 }
5470 ),
5471 None => format!(
5472 "operator fix: {} addressed, {} rejected, {}",
5473 fix.addressed.len(),
5474 fix.rejected.len(),
5475 if fix.committed {
5476 "committed"
5477 } else {
5478 "NO new commit"
5479 }
5480 ),
5481 },
5482 );
5483
5484 for f in &mut self.state.operator_fixes[request_index].findings {
5491 f.outcome = if fix.failed.is_some() {
5492 OperatorFixOutcome::Unreported
5493 } else if fix.addressed.contains(&f.id) {
5494 OperatorFixOutcome::Addressed
5495 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5496 OperatorFixOutcome::Rejected { why: r.why.clone() }
5497 } else {
5498 OperatorFixOutcome::Unreported
5499 };
5500 }
5501
5502 let committed = fix.committed;
5503 if committed {
5504 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5505 }
5506 self.state.operator_fixes[request_index].fix = Some(fix);
5507 self.state.save()?;
5510
5511 if committed {
5512 self.state.event(
5513 "fix",
5514 format!(
5515 "operator fix committed {}; opening a follow-up review-only run",
5516 short(&after)
5517 ),
5518 );
5519 let origin =
5522 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5523 match Self::review(
5524 &self.state.repo,
5525 &winner.branch,
5526 self.state.config.clone(),
5527 origin,
5528 )
5529 .await
5530 {
5531 Ok(mut follow_up) => {
5532 follow_up.state.event(
5533 "start",
5534 format!(
5535 "requested by an operator fix on run {} for finding(s) {}",
5536 self.state.id,
5537 self.state.operator_fixes[request_index]
5538 .findings
5539 .iter()
5540 .map(|f| f.id.as_str())
5541 .collect::<Vec<_>>()
5542 .join(", "),
5543 ),
5544 );
5545 follow_up.state.save()?;
5546 let follow_up_id = follow_up.state.id.clone();
5547 let adopted = match crate::direct::adopt(
5550 &follow_up.state,
5551 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5552 ) {
5553 Ok(a) => a,
5554 Err(e) => {
5555 self.state.event(
5558 "fix",
5559 format!(
5560 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5561 ),
5562 );
5563 self.state.save()?;
5564 return Ok(());
5565 }
5566 };
5567 let executed = follow_up.execute().await;
5568 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5569 if let Some(a) = adopted {
5570 a.finish(&follow_up.state, executed);
5571 }
5572 if let Some(e) = failure {
5573 self.state.event(
5574 "fix",
5575 format!(
5576 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5577 ),
5578 );
5579 }
5580 self.state.operator_fixes[request_index].follow_up_review_run =
5581 Some(follow_up_id);
5582 }
5583 Err(e) => {
5584 self.state.event(
5585 "fix",
5586 format!("committed the fix but could not open a follow-up review: {e:#}"),
5587 );
5588 }
5589 }
5590 self.state.save()?;
5591 }
5592
5593 Ok(())
5594 }
5595
5596 async fn ask_fixer(
5616 &mut self,
5617 winner: &Candidate,
5618 node: &'static str,
5619 round: Option<usize>,
5620 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5621 ) -> (SeatJob, SeatState, AgentOutcome) {
5622 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5623 self.ask_fixer_among(attempts, node, round, None, build)
5624 .await
5625 }
5626
5627 async fn ask_fixer_among(
5632 &mut self,
5633 attempts: Vec<(AgentSpec, String)>,
5634 node: &'static str,
5635 round: Option<usize>,
5636 restore: Option<(&Path, &str)>,
5637 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5638 ) -> (SeatJob, SeatState, AgentOutcome) {
5639 let run_id = self.state.id.clone();
5640 let prompts = self.state.config.prompts.clone();
5641 let cache = self.state.config.cache_dir();
5642 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5643 let mut last = None;
5644 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5645 let seat = self.seat(&key, &spec.id);
5646 let mut job = build(spec.clone(), seat);
5647 if i > 0 {
5648 job.stem = format!("{}-{}", job.stem, spec.id);
5649 }
5650 let ctx = WaveCtx {
5651 carry_seats: false,
5652 pauses: Some((&self.pause, &self.interrupt)),
5653 run: &run_id,
5654 node,
5655 prompts: &prompts,
5656 cache: cache.as_deref(),
5657 round,
5658 };
5659 let (seat, out) =
5660 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5661 let advances = match &out {
5664 AgentOutcome::Ok(o) => agent::output_advances(o),
5665 AgentOutcome::ResourceBlocked(_) => false,
5666 _ => true,
5667 };
5668 if let Some(next) = ids.get(i + 1)
5669 && advances
5670 {
5671 self.state.seats.insert(seat.key.clone(), seat.clone());
5672 let class =
5673 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5674 let reason = match &out {
5675 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5676 other => fail_reason(other),
5677 };
5678 if let Some((wt, rev)) = restore
5682 && let Err(e) = self.graft_restore(wt, rev).await
5683 {
5684 let why =
5685 format!("could not restore the worktree before the next fixer: {e:#}");
5686 tracing::warn!("{why}");
5687 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5688 AgentOutcome::ResourceBlocked(why)
5689 } else {
5690 AgentOutcome::Failed(why)
5691 };
5692 last = Some((job, seat, out));
5693 break;
5694 }
5695 record_handover(
5696 &mut self.state,
5697 node,
5698 &seat.key,
5699 &spec.id,
5700 next,
5701 &class,
5702 &reason,
5703 );
5704 continue;
5705 }
5706 last = Some((job, seat, out));
5707 break;
5708 }
5709 last.expect("the fixer chain always has an entry")
5710 }
5711
5712 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5723 use crate::recovery::{Mode, Repair};
5724 if !self.state.reviews.is_empty() {
5725 return Ok(());
5726 }
5727 let Some(mut rec) = self.state.recovery.clone() else {
5728 return Ok(());
5729 };
5730 if rec.mode != Mode::VerifyRepair {
5731 return Ok(());
5732 }
5733 let wt = winner.worktree.clone();
5734 let mut head = git::rev_parse(&wt, "HEAD").await?;
5735 match rec.repair.clone() {
5736 Repair::Done { .. } => return Ok(()),
5737 Repair::WaitingForRestore { before } => {
5738 self.graft_restore(&wt, &before).await?;
5739 head = before;
5740 rec.repair = Repair::NotStarted;
5741 self.state.recovery = Some(rec.clone());
5742 self.state.save()?;
5743 }
5744 Repair::Started { before } => {
5745 if let Err(e) = self.graft_restore(&wt, &before).await {
5749 if e.is::<crate::repo_lock::ResourceBlocked>() {
5750 return Err(e);
5751 }
5752 return self
5753 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5754 .await;
5755 }
5756 rec.repair = Repair::Done {
5757 before: before.clone(),
5758 after: before,
5759 committed: false,
5760 failed: Some("interrupted; rolled back".to_owned()),
5761 };
5762 self.state
5763 .event("recovery", "the repair was interrupted and rolled back");
5764 self.state.recovery = Some(rec);
5765 self.state.save()?;
5766 return Ok(());
5767 }
5768 Repair::NotStarted => {}
5769 }
5770 if crate::recovery::recheck(&mut rec, &head) {
5771 self.state.event("recovery", rec.summary());
5772 self.state.recovery = Some(rec);
5773 self.state.save()?;
5774 return Ok(());
5775 }
5776 rec.repair = Repair::Started {
5777 before: head.clone(),
5778 };
5779 self.state.recovery = Some(rec.clone());
5780 self.state.save()?;
5781
5782 let language = self.state.config.graph.language.clone();
5783 let sessions = self.state.config.graph.sessions;
5784 let artifacts = agent::artifacts_dir(&self.state.dir());
5785 let fix_prompt = prompt::recovery_fix(
5786 &self.state.instruction,
5787 &rec,
5788 self.state.config.graph.review_rounds,
5789 &language,
5790 );
5791 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5792 self.state.event(
5793 "recovery",
5794 format!(
5795 "repair: the fixer starts from the failed verification on {}",
5796 short(&head)
5797 ),
5798 );
5799 let (_job, seat, out) = self
5800 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5801 prompt: fix_prompt.clone(),
5802 spec,
5803 seat,
5804 cwd: wt.clone(),
5805 timeout,
5806 allow_write: true,
5807 sessions,
5808 artifacts: artifacts.clone(),
5809 stem: "fix-recovery".to_owned(),
5810 handover: None,
5811 })
5812 .await;
5813 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5814 rec.repair = Repair::WaitingForRestore { before: head };
5815 self.state.recovery = Some(rec);
5816 return self.stop_for_resource();
5817 }
5818 let failed = match &out {
5819 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5820 AgentOutcome::Quota(o) => {
5821 self.state.quota.push(QuotaLoss {
5822 seat: seat.key.clone(),
5823 node: "fix".to_owned(),
5824 at: Timestamp::now(),
5825 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5826 });
5827 Some("rate limited (quota); the fixer could not run".to_owned())
5828 }
5829 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5830 AgentOutcome::Failed(e) => Some(e.clone()),
5831 };
5832 self.state.seats.insert(seat.key.clone(), seat);
5833 let coordination = self.coordinate(&wt).await?;
5834 let failed = match failed {
5835 Some(why) => {
5837 if let Err(e) = git::restore_to(&wt, &head).await {
5838 return self
5839 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5840 .await;
5841 }
5842 Some(why)
5843 }
5844 None => {
5845 if let Ok(r) = git::rescue_commit(
5846 &wt,
5847 "magi: recovery repair of the failed verification (uncommitted work)",
5848 )
5849 .await
5850 {
5851 self.state.note_withheld("fix", &r.withheld);
5852 }
5853 None
5854 }
5855 };
5856 let after = git::rev_parse(&wt, "HEAD").await?;
5857 drop(coordination);
5858 self.state.event(
5859 "recovery",
5860 match &failed {
5861 Some(why) => format!("repair: nothing kept ({why})"),
5862 None if after != head => {
5863 format!("repair: committed {}", short(&after))
5864 }
5865 None => "repair: the fixer made no commit".to_owned(),
5866 },
5867 );
5868 rec.repair = Repair::Done {
5869 before: head.clone(),
5870 committed: after != head,
5871 after,
5872 failed,
5873 };
5874 self.state.recovery = Some(rec);
5875 self.state.save()?;
5876 Ok(())
5877 }
5878
5879 async fn stop_recovery(
5882 &mut self,
5883 mut rec: crate::recovery::RecoveryRecord,
5884 head: &str,
5885 why: String,
5886 ) -> Result<()> {
5887 rec.repair = crate::recovery::Repair::Started {
5889 before: head.to_owned(),
5890 };
5891 self.state.recovery = Some(rec);
5892 self.state
5893 .event("recovery", format!("repair could not be undone: {why}"));
5894 self.state.status = RunStatus::Blocked;
5895 self.state.save()?;
5896 bail!("recovery repair could not be rolled back: {why}")
5897 }
5898
5899 async fn review_loop(&mut self) -> Result<()> {
5900 loop {
5901 if self
5902 .state
5903 .base_sync
5904 .as_ref()
5905 .is_some_and(|s| s.conflict.is_some())
5906 {
5907 return Ok(());
5908 }
5909 if self.catchup_repairable() {
5910 let record = self.state.reviews.last().expect("repairable round").clone();
5911 let winner = self
5912 .state
5913 .winner()
5914 .cloned()
5915 .context("catch-up repair has no winner")?;
5916 if !self.fix_review_round(&winner, record).await? {
5917 return Ok(());
5918 }
5919 }
5920 self.review_loop_pass().await?;
5921 if !self.catchup_repairable() {
5922 return Ok(());
5923 }
5924 }
5925 }
5926
5927 fn catchup_repairable(&self) -> bool {
5930 self.state.reviews.len() < self.state.config.graph.review_rounds
5931 && self.state.reviews.last().is_some_and(|r| {
5932 !r.clean
5933 && r.blocking == 0
5934 && r.fix.is_none()
5935 && r.e2e_status() == E2eStatus::Failed
5936 })
5937 }
5938
5939 async fn review_loop_pass(&mut self) -> Result<()> {
5940 if self
5945 .state
5946 .base_sync
5947 .as_ref()
5948 .is_some_and(|s| s.conflict.is_some())
5949 {
5950 return Ok(());
5951 }
5952 let run_id = self.state.id.clone();
5957 let prompts = self.state.config.prompts.clone();
5958 let Some(winner) = self.state.winner().cloned() else {
5959 return Ok(());
5960 };
5961 let max_rounds = self.state.config.graph.review_rounds;
5962 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5972 if status == RunStatus::Gating {
5975 self.record_contested_handoff();
5976 }
5977 self.state.status = status;
5978 self.state.save()?;
5979 return Ok(());
5980 }
5981 self.state.status = RunStatus::Reviewing;
5982 if self.state.reviews.last().is_some_and(|r| {
5992 r.e2e_status() == E2eStatus::ResourceBlocked
5993 || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
5994 }) {
5995 let shell = self.state.config.shell();
5996 return self
5997 .stop_unresolved_review(
5998 "the last round's own verification never resolved",
5999 &shell,
6000 &winner.worktree,
6001 )
6002 .await;
6003 }
6004
6005 self.recovery_repair(&winner).await?;
6006 if self.resource_halt()? {
6007 return Ok(());
6008 }
6009
6010 let repo = self.state.repo.clone();
6011 let root = self.state.worktree_root();
6012 let language = self.state.config.graph.language.clone();
6013 let sessions = self.state.config.graph.sessions;
6014 let artifacts = self.merge_repair_artifacts();
6015 let base = self.landing_base();
6016 let base_short = short(&base);
6017 let reviewers = self.roles.reviewers.clone();
6018 let shell = self.state.config.shell();
6019
6020 for round in (self.state.reviews.len() + 1)..=max_rounds {
6021 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6022 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6023 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
6024 let prev_verification = self
6033 .state
6034 .reviews
6035 .last()
6036 .and_then(|r| r.verification_summary(&head));
6037 let prior_section = match (&self.state.recovery, round) {
6040 (Some(rec), 1) if self.state.reviews.is_empty() => {
6041 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
6042 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
6043 Some(crate::recovery::head_of(
6044 &git::diff(&winner.worktree, reviewed, "HEAD")
6045 .await
6046 .unwrap_or_default(),
6047 crate::recovery::SINCE_DIFF_MAX,
6048 ))
6049 }
6050 _ => None,
6051 };
6052 Some(prompt::recovery_review(rec, since.as_deref()))
6053 }
6054 _ => None,
6055 };
6056
6057 let mut jobs = Vec::new();
6061 let coordination = self.coordinate(&repo).await?;
6062 for (r, spec) in reviewers.iter().cloned().enumerate() {
6063 let wt = root.join(format!("review-{}", r + 1));
6064 if wt.exists() {
6067 git::reset_detached(&wt, &head).await?;
6068 } else {
6069 git::worktree_add_detached(&repo, &wt, &head).await?;
6070 }
6071 self.setup_seat_worktree(&repo, &wt, false).await?;
6072 let seat_key = format!("review-{}", r + 1);
6073 let spec = pick_start_spec(
6076 &self.roles.reviewer_roster,
6077 spec,
6078 self.state.seat_history.get(&seat_key),
6079 );
6080 let seat = self.seat(&seat_key, &spec.id);
6081 jobs.push(SeatJob {
6082 prompt: prompt::review(&prompt::ReviewCtx {
6083 instruction: &self.state.instruction,
6084 branch: &winner.branch,
6085 base_short: &base_short,
6086 stat: &stat,
6087 patch: &patch,
6088 verification: prev_verification.as_ref(),
6089 prior: prior_section.as_deref(),
6090 reviewers: reviewers.len(),
6091 round,
6092 rounds: max_rounds,
6093 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
6096 lens: Lens::for_seat(r),
6097 language: &language,
6098 github_language: &self.state.config.graph.github_language,
6099 }),
6100 spec,
6101 seat,
6102 cwd: wt,
6103 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6104 allow_write: false,
6105 sessions,
6106 artifacts: artifacts.clone(),
6107 stem: format!("review-{round}-{}", r + 1),
6108 handover: None,
6109 });
6110 }
6111
6112 drop(coordination);
6113 self.state.event(
6114 "review",
6115 format!(
6116 "round {round}: {} reviewers on {}",
6117 jobs.len(),
6118 short(&head)
6119 ),
6120 );
6121 let mut quota_losses = Vec::new();
6122 let review_retries = self.state.config.graph.retries;
6123 let review_cache = self.state.config.cache_dir();
6124 let ctx = WaveCtx {
6125 carry_seats: true,
6126 pauses: Some((&self.pause, &self.interrupt)),
6127 run: &run_id,
6128 node: "review",
6129 prompts: &prompts,
6130 cache: review_cache.as_deref(),
6131 round: Some(round),
6132 };
6133 let results = ask_json_wave::<Review>(
6134 jobs,
6135 Arc::clone(&self.sem),
6136 review_retries,
6137 &self.roles.reviewer_roster,
6138 &ctx,
6139 &mut quota_losses,
6140 &mut self.state,
6141 &|_: &Review| Ok(()),
6142 )
6143 .await;
6144 if self.state.resource_block.is_some() {
6145 return self.stop_for_resource();
6146 }
6147 let round_quota_missing = quota_losses.len();
6151 self.state.quota.extend(quota_losses);
6152
6153 let mut records = Vec::new();
6154 let mut all_findings = Vec::new();
6155 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6156 let agent_id = seat.agent.clone();
6157 self.state.seats.insert(seat.key.clone(), seat);
6158 let mut record = ReviewRecord {
6159 reviewer: r + 1,
6160 agent: agent_id,
6161 summary: String::new(),
6162 findings: Vec::new(),
6163 vote: None,
6164 failed: None,
6165 duration_ms: 0,
6166 attempts,
6172 };
6173 match res {
6174 Ok((review, out)) => {
6175 record.summary =
6183 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6184 record.vote = Some(review.vote);
6185 record.duration_ms = out.duration_ms;
6186 for (n, mut f) in review.findings.into_iter().enumerate() {
6187 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6190 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6191 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6192 f.file = f
6198 .file
6199 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6200 all_findings.push(f.clone());
6201 record.findings.push(f);
6202 }
6203 self.state.event(
6204 "review",
6205 format!(
6206 "round {round}: reviewer {} voted {} with {} finding(s)",
6207 r + 1,
6208 review.vote.label(),
6209 record.findings.len()
6210 ),
6211 );
6212 }
6213 Err(e) => {
6214 record.failed = Some(e.to_string());
6215 self.state.event(
6216 "review",
6217 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6218 );
6219 }
6220 }
6221 records.push(record);
6222 }
6223
6224 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6231 let vote_split =
6232 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6233 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6234 if vote_split {
6235 self.state.event(
6236 "review",
6237 format!(
6238 "round {round}: votes split ({}) — one round of reconsideration",
6239 initial_votes
6240 .iter()
6241 .map(|v| v.label())
6242 .collect::<Vec<_>>()
6243 .join(", ")
6244 ),
6245 );
6246 let panel: Vec<ReviewSeatReport<'_>> = records
6249 .iter()
6250 .filter_map(|r| {
6251 r.vote.map(|vote| ReviewSeatReport {
6252 reviewer: r.reviewer,
6253 vote,
6254 summary: &r.summary,
6255 findings: &r.findings,
6256 })
6257 })
6258 .collect();
6259
6260 let mut jobs = Vec::new();
6261 let mut seats_at = Vec::new();
6262 for (r, spec) in reviewers.iter().cloned().enumerate() {
6263 if records[r].vote.is_none() {
6267 continue;
6268 }
6269 let wt = root.join(format!("review-{}", r + 1));
6270 let seat_key = format!("review-{}", r + 1);
6271 let spec = self.occupant(&seat_key, spec);
6272 let seat = self.seat(&seat_key, &spec.id);
6273 let build = |with_patch: bool| {
6281 prompt::review_reconsider(&ReviewReconsiderCtx {
6282 instruction: &self.state.instruction,
6283 reviewer: r + 1,
6284 lens: Lens::for_seat(r),
6285 panel: &panel,
6286 patch: with_patch.then_some(ReviewPatch {
6287 branch: &winner.branch,
6288 base_short: &base_short,
6289 stat: &stat,
6290 patch: &patch,
6291 }),
6292 round,
6293 rounds: max_rounds,
6294 language: &language,
6295 })
6296 };
6297 let full = build(true);
6298 let prompt = if has_context(&spec, &seat, sessions) {
6299 build(false)
6300 } else {
6301 full.clone()
6302 };
6303 jobs.push(SeatJob {
6304 prompt,
6305 spec,
6306 seat,
6307 cwd: wt,
6308 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6309 allow_write: false,
6310 sessions,
6311 artifacts: artifacts.clone(),
6312 stem: format!("review-{round}-reconsider-{}", r + 1),
6313 handover: Some(full),
6314 });
6315 seats_at.push(r);
6316 }
6317
6318 let mut recon_quota_losses = Vec::new();
6319 let recon_cache = self.state.config.cache_dir();
6320 let recon_ctx = WaveCtx {
6321 carry_seats: true,
6322 pauses: Some((&self.pause, &self.interrupt)),
6323 run: &run_id,
6324 node: "review",
6325 prompts: &prompts,
6326 cache: recon_cache.as_deref(),
6327 round: Some(round),
6328 };
6329 let recon_results = ask_json_wave::<ReviewRevote>(
6330 jobs,
6331 Arc::clone(&self.sem),
6332 review_retries,
6333 &self.roles.reviewer_roster,
6334 &recon_ctx,
6335 &mut recon_quota_losses,
6336 &mut self.state,
6337 &|_: &ReviewRevote| Ok(()),
6338 )
6339 .await;
6340 if self.state.resource_block.is_some() {
6341 return self.stop_for_resource();
6342 }
6343 self.state.quota.extend(recon_quota_losses);
6344
6345 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6346 let agent_id = seat.agent.clone();
6347 self.state.seats.insert(seat.key.clone(), seat);
6348 let mut rec = ReviewRevoteRecord {
6349 reviewer: r + 1,
6350 agent: agent_id,
6351 vote: None,
6352 reason: String::new(),
6353 failed: None,
6354 };
6355 match res {
6356 Ok((rv, _)) => {
6357 rec.vote = Some(rv.vote);
6358 rec.reason =
6359 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6360 self.state.event(
6361 "review",
6362 format!(
6363 "round {round}: reviewer {} revoted {}",
6364 r + 1,
6365 rv.vote.label()
6366 ),
6367 );
6368 }
6369 Err(e) => {
6370 rec.failed = Some(e.to_string());
6371 self.state.event(
6372 "review",
6373 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6374 );
6375 }
6376 }
6377 reconsideration.push(rec);
6378 }
6379 } else if initial_votes.len() > 1 {
6380 self.state.event(
6381 "review",
6382 format!(
6383 "round {round}: votes agreed ({}) — no reconsideration",
6384 initial_votes[0].label()
6385 ),
6386 );
6387 }
6388
6389 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6390 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6391 let defer_e2e =
6402 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6403 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6404 let reason =
6405 format!("{blocking} blocking finding(s) already required a fix this round");
6406 self.state.event(
6407 "verify",
6408 format!(
6409 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6410 {}); it will run once a round has none left",
6411 short(&head)
6412 ),
6413 );
6414 (Vec::new(), false, true, Some(reason))
6415 } else {
6416 let e2e_commands = self.state.config.verify.e2e.clone();
6417 let cache_dir = self.state.config.cache_dir();
6418 let context = format!("round {round}");
6419 let wait = self.repository_wait();
6420 let (e2e, verify_retried) = with_cache_lease(
6421 &mut self.state,
6422 cache_dir.as_deref(),
6423 "e2e",
6424 "e2e",
6425 &winner.worktree,
6426 &head,
6427 verify_timeout,
6428 &context,
6429 &wait,
6430 |state, budget| {
6431 let shell = shell.clone();
6432 let e2e_commands = e2e_commands.clone();
6433 let worktree = winner.worktree.clone();
6434 let context = context.clone();
6435 async move {
6436 run_e2e_with_retry(
6437 state,
6438 &shell,
6439 &e2e_commands,
6440 &worktree,
6441 budget,
6442 &context,
6443 )
6444 .await
6445 }
6446 },
6447 )
6448 .await?;
6449 (e2e, verify_retried, false, None)
6450 };
6451
6452 let expected = records.len();
6453 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6454 let incomplete = answered < expected;
6455 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6456 let policy = self.state.config.graph.incomplete_review;
6457 let clean = round_is_clean(
6458 blocking,
6459 e2e_ok,
6460 answered,
6461 expected,
6462 round_quota_missing,
6463 policy,
6464 );
6465
6466 let mut round_record = ReviewRound {
6467 round,
6468 head: head.clone(),
6469 verified_head: None,
6470 verified_at: None,
6471 reviews: records,
6472 e2e,
6473 verify_retried,
6474 e2e_deferred,
6475 e2e_defer_reason,
6476 fix: None,
6477 blocking,
6478 answered,
6479 expected,
6480 clean,
6481 progressed: false,
6482 vote_split,
6483 reconsideration,
6484 verdict: None,
6485 };
6486 round_record.verdict = ReviewVote::worst(
6490 round_record
6491 .final_votes()
6492 .into_iter()
6493 .map(|(_, _, vote)| vote),
6494 );
6495 if !matches!(
6506 round_record.e2e_status(),
6507 E2eStatus::Deferred | E2eStatus::NotConfigured
6508 ) {
6509 round_record.verified_head = Some(head.clone());
6510 round_record.verified_at = Some(Timestamp::now());
6511 }
6512
6513 if incomplete {
6514 let missing: Vec<String> = round_record
6515 .reviews
6516 .iter()
6517 .filter(|r| r.failed.is_some())
6518 .map(|r| format!("review-{}", r.reviewer))
6519 .collect();
6520 self.state.event(
6521 "review",
6522 format!(
6523 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6524 missing.join(", ")
6525 ),
6526 );
6527 }
6528
6529 if clean {
6530 self.state.event(
6531 "review",
6532 if incomplete && policy == IncompleteReviewPolicy::Warn {
6533 format!(
6534 "round {round}: clean (warn policy, incomplete panel) — no \
6535 blocking findings from the seats that answered, verification green"
6536 )
6537 } else if incomplete {
6538 format!(
6539 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6540 quorum) — no blocking findings from the seats that answered, \
6541 verification green",
6542 expected - answered
6543 )
6544 } else {
6545 format!("round {round}: clean — no blocking findings, verification green")
6546 },
6547 );
6548 self.state.reviews.push(round_record);
6549 self.state.status = RunStatus::Gating;
6550 self.state.save()?;
6551 return Ok(());
6552 }
6553
6554 if incomplete && blocking == 0 && e2e_ok {
6562 self.state.reviews.push(round_record);
6563 self.state.save()?;
6564 if round == max_rounds {
6565 self.state.status = RunStatus::Blocked;
6566 self.state.event(
6567 "review",
6568 format!(
6569 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6570 refusing to call it clean",
6571 expected - answered
6572 ),
6573 );
6574 return Ok(());
6575 }
6576 continue;
6577 }
6578
6579 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6591 self.state.reviews.push(round_record);
6592 return self
6593 .stop_unresolved_review(
6594 "the round's own verification could not run",
6595 &shell,
6596 &winner.worktree,
6597 )
6598 .await;
6599 }
6600
6601 if round == max_rounds {
6602 self.state.reviews.push(round_record);
6603 return self
6604 .stop_reviewing(
6605 &format!(
6606 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6607 ),
6608 &shell,
6609 &winner.worktree,
6610 )
6611 .await;
6612 }
6613
6614 if !self.fix_review_round(&winner, round_record).await? {
6615 return Ok(());
6616 }
6617 }
6618 Ok(())
6619 }
6620
6621 async fn fix_review_round(
6624 &mut self,
6625 winner: &Candidate,
6626 mut round_record: ReviewRound,
6627 ) -> Result<bool> {
6628 let round = round_record.round;
6629 let shell = self.state.config.shell();
6630 let step = self.run_fix_round(winner, &mut round_record).await?;
6631 if matches!(step, FixStep::Resource) {
6632 self.stop_for_resource()?;
6633 return Ok(false);
6634 }
6635 if self.state.reviews.len() < round {
6636 self.state.reviews.push(round_record);
6637 } else {
6638 self.state.reviews[round - 1] = round_record;
6639 }
6640 self.state.save()?;
6641
6642 if matches!(step, FixStep::Recorded { report_lost: true }) {
6655 self.stop_reviewing(
6656 "the fixer's adoption report never came back, even after resuming its \
6657 own seat; refusing to start another round against the same worktree \
6658 while that is unresolved",
6659 &shell,
6660 &winner.worktree,
6661 )
6662 .await?;
6663 return Ok(false);
6664 }
6665
6666 let streak = self
6667 .state
6668 .reviews
6669 .iter()
6670 .rev()
6671 .take_while(|r| !r.progressed)
6672 .count();
6673 if streak >= STAGNANT_LIMIT {
6674 self.stop_reviewing(
6675 &format!("the tree has not moved against base for {streak} round(s) in a row"),
6676 &shell,
6677 &winner.worktree,
6678 )
6679 .await?;
6680 return Ok(false);
6681 }
6682 Ok(true)
6683 }
6684
6685 async fn run_fix_round(
6687 &mut self,
6688 winner: &Candidate,
6689 round_record: &mut ReviewRound,
6690 ) -> Result<FixStep> {
6691 let round = round_record.round;
6692 let max_rounds = self.state.config.graph.review_rounds;
6693 let run_id = self.state.id.clone();
6694 let prompts = self.state.config.prompts.clone();
6695 let language = self.state.config.graph.language.clone();
6696 let sessions = self.state.config.graph.sessions;
6697 let artifacts = self.merge_repair_artifacts();
6698 let base = self.landing_base();
6699 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
6700 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
6701 let this_round_verification = round_record.verification_summary(&head);
6702 let all_findings: Vec<_> = round_record
6703 .reviews
6704 .iter()
6705 .flat_map(|r| r.findings.iter().cloned())
6706 .collect();
6707 let blocking_findings: Vec<_> = all_findings
6710 .iter()
6711 .filter(|f| f.severity.blocks())
6712 .cloned()
6713 .collect();
6714 let fix_prompt = prompt::fix_in(
6715 &self.state.instruction,
6716 &blocking_findings,
6717 this_round_verification.as_ref(),
6718 round,
6719 max_rounds,
6720 &language,
6721 &self.state.config.graph.github_language,
6722 );
6723 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6724 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6725 let (job, seat, out) = self
6726 .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
6727 prompt: fix_prompt.clone(),
6728 spec,
6729 seat,
6730 cwd: winner.worktree.clone(),
6731 timeout,
6732 allow_write: true,
6733 sessions,
6734 artifacts: artifacts.clone(),
6735 stem: format!("fix-{round}"),
6736 handover: None,
6737 })
6738 .await;
6739 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6742 return Ok(FixStep::Resource);
6743 }
6744 let agent_id = seat.agent.clone();
6745
6746 let mut fix = FixRecord {
6747 agent: agent_id,
6748 addressed: Vec::new(),
6749 rejected: Vec::new(),
6750 notes: String::new(),
6751 committed: false,
6752 failed: None,
6753 duration_ms: 0,
6754 continuation: None,
6755 };
6756 let mut continuation = ContinuationRecord::not_needed();
6757 let mut final_seat = seat.clone();
6758 match out {
6759 AgentOutcome::Ok(o) => {
6760 fix.duration_ms = o.duration_ms;
6761 let parsed = verdict::extract_json::<FixReport>(&o.text);
6762 let incomplete_reason = match &parsed {
6769 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6770 "the reply parsed, but it reported a command whose own CLI never \
6771 confirmed an exit status"
6772 .to_owned(),
6773 ),
6774 Ok(_) => None,
6775 Err(e) => Some(e.to_string()),
6776 };
6777 match incomplete_reason {
6778 None => {
6779 let report = parsed.expect("checked Ok above");
6780 fix.addressed = report.addressed;
6781 fix.rejected = report.rejected;
6782 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6783 }
6784 Some(reason) => {
6785 let (resumed_seat, resolved, failure, cont) = self
6786 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6787 .await;
6788 fix.duration_ms += cont.cumulative_wait_ms;
6789 continuation = cont;
6790 final_seat = resumed_seat;
6791 match resolved {
6792 Some(report) => {
6793 fix.addressed = report.addressed;
6794 fix.rejected = report.rejected;
6795 fix.notes =
6796 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6797 }
6798 None => fix.failed = failure,
6799 }
6800 }
6801 }
6802 }
6803 AgentOutcome::Dropped(o) => {
6805 fix.duration_ms = o.duration_ms;
6806 let why = o
6807 .dropped
6808 .as_ref()
6809 .map(|d| d.why.as_str())
6810 .unwrap_or("the CLI ended the stream without delivering its answer");
6811 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6812 }
6813 AgentOutcome::Quota(o) => {
6814 self.state.quota.push(QuotaLoss {
6815 seat: final_seat.key.clone(),
6816 node: "fix".to_owned(),
6817 at: Timestamp::now(),
6818 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6819 });
6820 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6821 }
6822 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6823 }
6824 fix.continuation = Some(continuation);
6825 self.state.seats.insert(final_seat.key.clone(), final_seat);
6826 let coordination = self.coordinate(&winner.worktree).await?;
6827 if let Ok(r) = git::rescue_commit(
6828 &winner.worktree,
6829 &format!("magi: review round {round} fixes (uncommitted work)"),
6830 )
6831 .await
6832 {
6833 self.state.note_withheld("fix", &r.withheld);
6834 }
6835 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6836 fix.committed = after != before;
6837 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6845 drop(coordination);
6846 let progressed = diff_after != patch;
6847 let commit_note = if fix.committed {
6848 "committed"
6849 } else {
6850 "NO new commit"
6851 };
6852 let tree_note = if progressed {
6853 "changed vs base"
6854 } else {
6855 "unchanged vs base"
6856 };
6857 self.state.event(
6858 "fix",
6859 match &fix.failed {
6860 Some(reason) => {
6866 format!(
6867 "round {round}: fixer's adoption report was lost ({reason}); \
6868 {commit_note}, tree {tree_note}"
6869 )
6870 }
6871 None => format!(
6872 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6873 {tree_note}{}",
6874 fix.addressed.len(),
6875 fix.rejected.len(),
6876 if continuation.outcome == ContinuationOutcome::Resumed {
6877 format!(
6878 " (adoption report recovered after {} continuation(s))",
6879 continuation.attempts
6880 )
6881 } else {
6882 String::new()
6883 },
6884 ),
6885 },
6886 );
6887 round_record.fix = Some(fix);
6888 round_record.progressed = progressed;
6889 Ok(FixStep::Recorded {
6890 report_lost: matches!(
6891 continuation.outcome,
6892 ContinuationOutcome::Exhausted
6893 | ContinuationOutcome::QuotaLost
6894 | ContinuationOutcome::NoSession
6895 ),
6896 })
6897 }
6898
6899 fn record_contested_handoff(&mut self) {
6933 if self.state.contested_handoff.is_some() {
6934 return;
6935 }
6936 let Some(contested) = self
6937 .state
6938 .reviews
6939 .last()
6940 .and_then(ReviewRound::contested_handoff)
6941 else {
6942 return;
6943 };
6944 self.state.event(
6945 "review",
6946 format!(
6947 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6948 wait for the owner's approval",
6949 contested.findings.len(),
6950 contested.rejecters.len()
6951 ),
6952 );
6953 self.state.contested_handoff = Some(contested);
6954 }
6955
6956 async fn stop_unresolved_review(
6957 &mut self,
6958 why: &str,
6959 shell: &[String],
6960 worktree: &Path,
6961 ) -> Result<()> {
6962 self.stop_reviewing_inner(why, shell, worktree, true).await
6963 }
6964
6965 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6966 self.stop_reviewing_inner(why, shell, worktree, false).await
6967 }
6968
6969 async fn stop_reviewing_inner(
6970 &mut self,
6971 why: &str,
6972 shell: &[String],
6973 worktree: &Path,
6974 repairable: bool,
6975 ) -> Result<()> {
6976 let round_idx = self.state.reviews.len() - 1;
6977 let needs_catchup_run = matches!(
6985 self.state.reviews[round_idx].e2e_status(),
6986 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6987 );
6988 if needs_catchup_run {
6989 let round = self.state.reviews[round_idx].round;
6990 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6991 let commands = self.state.config.verify.e2e.clone();
6992 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6993 let cache_dir = self.state.config.cache_dir();
6994 let context = format!(
6995 "round {round}: verification unresolved, catching up before the final decision"
6996 );
6997 let wait = self.repository_wait();
6998 let (outcomes, verify_retried) = with_cache_lease(
6999 &mut self.state,
7000 cache_dir.as_deref(),
7001 "e2e",
7002 "e2e",
7003 worktree,
7004 &attempted_head,
7005 timeout,
7006 &context,
7007 &wait,
7008 |state, budget| {
7009 let shell = shell.to_vec();
7010 let commands = commands.clone();
7011 let context = context.clone();
7012 async move {
7013 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
7014 .await
7015 }
7016 },
7017 )
7018 .await?;
7019 let last = &mut self.state.reviews[round_idx];
7020 last.e2e = outcomes;
7021 last.verify_retried = verify_retried;
7022 last.verified_head = Some(attempted_head);
7029 last.verified_at = Some(Timestamp::now());
7030 if verify_inconclusive(&last.e2e) {
7031 self.state.save()?;
7038 return Ok(());
7039 }
7040 last.e2e_deferred = false;
7041 }
7042 let last = &self.state.reviews[round_idx];
7043 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
7044
7045 match last.e2e_status() {
7046 E2eStatus::Failed => {
7047 let red: Vec<String> = last
7048 .e2e
7049 .iter()
7050 .filter(|o| !o.ok())
7051 .map(|o| {
7052 format!(
7053 "`{}` -> {:?}\n{}",
7054 o.command,
7055 o.code,
7056 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7057 )
7058 })
7059 .collect();
7060 let can_repair = last.blocking == 0 && last.fix.is_none();
7061 self.state.reviews[round_idx].clean = false;
7062 self.state
7063 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
7064 self.state.status = RunStatus::Blocked;
7065 if repairable
7066 && can_repair
7067 && self.state.reviews.len() < self.state.config.graph.review_rounds
7068 {
7069 self.state.status = RunStatus::Reviewing;
7070 }
7071 }
7072 E2eStatus::ResourceBlocked => {
7077 self.state.event(
7078 "review",
7079 format!(
7080 "{why}; e2e could not run (resource unavailable); not \
7081 deciding yet"
7082 ),
7083 );
7084 }
7085 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
7086 self.state.event(
7087 "review",
7088 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
7089 );
7090 self.record_contested_handoff();
7091 self.state.status = RunStatus::Gating;
7092 }
7093 }
7094 self.state.save()?;
7095 Ok(())
7096 }
7097
7098 async fn gate(&mut self) -> Result<()> {
7101 if self.state.status == RunStatus::Failed
7113 || self
7114 .state
7115 .base_sync
7116 .as_ref()
7117 .is_some_and(|s| s.conflict.is_some())
7118 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
7119 != Some(RunStatus::Gating)
7120 {
7121 return Ok(());
7122 }
7123 if self.state.gate_ran {
7124 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
7135 self.state.status = RunStatus::Blocked;
7136 self.state.save()?;
7137 }
7138 return Ok(());
7139 }
7140 let Some(winner) = self.state.winner().cloned() else {
7141 return Ok(());
7142 };
7143 self.state.status = RunStatus::Gating;
7144 let mut outcomes = self.run_gate(&winner).await?;
7145 loop {
7146 if verify_inconclusive(&outcomes) {
7157 self.state.save()?;
7158 return Ok(());
7159 }
7160 if outcomes.iter().all(CommandOutcome::ok) {
7161 break;
7162 }
7163 match self.gate_fix_round(&winner, &outcomes).await? {
7164 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
7165 GateFix::Stop => break,
7166 GateFix::Defer => {
7167 self.state.save()?;
7168 return Ok(());
7169 }
7170 }
7171 }
7172 let passed = outcomes.iter().all(CommandOutcome::ok);
7173 self.state.gate = outcomes;
7174 self.state.gate_ran = true;
7175 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7176 if !passed {
7177 self.state.status = RunStatus::Blocked;
7178 let spent = self.state.gate_fixes.len();
7179 self.state.event(
7180 "gate",
7181 if spent == 0 {
7182 "gate failed; not merging".to_owned()
7183 } else {
7184 format!("gate failed after {spent} gate-fix round(s); not merging")
7185 },
7186 );
7187 }
7188 self.state.save()?;
7189 Ok(())
7190 }
7191
7192 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7202 let commands = state.config.verify.pre_gate.clone();
7203 if commands.is_empty() {
7204 return;
7205 }
7206 let shell = state.config.shell();
7207 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7208 let (outcomes, _) = run_commands(
7209 state,
7210 "pre_gate",
7211 "pre_gate",
7212 0,
7213 &shell,
7214 &commands,
7215 &winner.worktree,
7216 timeout,
7217 )
7218 .await;
7219 for o in &outcomes {
7220 if !o.ok() {
7221 tracing::warn!(
7222 "pre_gate `{}` failed ({:?}); the gate decides",
7223 o.command,
7224 o.code
7225 );
7226 }
7227 state.event(
7228 "pre_gate",
7229 format!(
7230 "`{}` -> {}",
7231 o.command,
7232 if o.ok() {
7233 "pass".to_owned()
7234 } else {
7235 format!(
7236 "FAIL ({:?})\n{}",
7237 o.code,
7238 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7239 )
7240 }
7241 ),
7242 );
7243 }
7244 state.pre_gate = outcomes;
7245 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7246 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7247 Ok(head) => {
7248 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7249 state.pre_gate_commit = Some(head);
7250 }
7251 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7252 },
7253 Ok(false) => {}
7254 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7255 }
7256 if let Err(e) = state.save() {
7257 tracing::warn!("could not persist the pre_gate record: {e:#}");
7258 }
7259 }
7260
7261 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7264 let wait = self.repository_wait();
7265 run_landing_gate(&mut self.state, winner, &wait).await
7266 }
7267
7268 async fn gate_fix_round(
7281 &mut self,
7282 winner: &Candidate,
7283 outcomes: &[CommandOutcome],
7284 ) -> Result<GateFix> {
7285 let cap = self.state.config.graph.gate_fix_rounds;
7286 let spent = self.state.gate_fixes.len();
7287 if spent >= cap {
7288 if cap > 0 {
7289 self.state.event(
7290 "gate",
7291 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7292 );
7293 }
7294 return Ok(GateFix::Stop);
7295 }
7296 if !gate_fixable(outcomes) {
7297 self.state.event(
7298 "gate",
7299 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7300 command or similar); not spending a fix round on it",
7301 );
7302 return Ok(GateFix::Stop);
7303 }
7304 let min_free = self.state.config.disk.min_free_bytes;
7305 if min_free > 0 {
7306 match crate::disk::free_bytes(&winner.worktree) {
7307 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7308 Ok(free) => {
7309 self.state.event(
7310 "gate",
7311 format!(
7312 "only {free} bytes free ({min_free} required by `[disk] \
7313 min_free_bytes`); not spending a fix round on a failure the disk \
7314 may explain"
7315 ),
7316 );
7317 return Ok(GateFix::Stop);
7318 }
7319 Err(e) => {
7320 self.state.event(
7321 "gate",
7322 format!("free disk space could not be measured ({e:#}); no fix round"),
7323 );
7324 return Ok(GateFix::Stop);
7325 }
7326 }
7327 }
7328
7329 let attempt = spent + 1;
7330 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7331 let base = self.landing_base();
7332 let fix_prompt = prompt::gate_fix_in(
7333 &self.state.instruction,
7334 &failed,
7335 attempt,
7336 cap,
7337 &self.state.config.graph.language,
7338 &self.state.config.graph.github_language,
7339 );
7340 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7341 let sessions = self.state.config.graph.sessions;
7342 let artifacts = self.merge_repair_artifacts();
7343 self.state.event(
7344 "gate",
7345 format!("gate failed; gate-fix round {attempt} of {cap}"),
7346 );
7347 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7348 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7349 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7352 head: before.clone(),
7353 passed: false,
7354 outcomes: Vec::new(),
7355 });
7356 self.state.save()?;
7357 let (_, seat, out) = self
7358 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7359 prompt: fix_prompt.clone(),
7360 spec,
7361 seat,
7362 cwd: winner.worktree.clone(),
7363 timeout,
7364 allow_write: true,
7365 sessions,
7366 artifacts: artifacts.clone(),
7367 stem: format!("gate-fix-{attempt}"),
7368 handover: None,
7369 })
7370 .await;
7371 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7374 return Ok(GateFix::Defer);
7375 }
7376 let mut record = GateFixRecord {
7377 agent: seat.agent.clone(),
7378 failed,
7379 notes: String::new(),
7380 committed: false,
7381 error: None,
7382 };
7383 match out {
7384 AgentOutcome::Ok(o) => {
7385 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7388 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7389 }
7390 }
7391 AgentOutcome::Dropped(_) => {
7392 record.error = Some("the CLI dropped the stream".to_owned());
7393 }
7394 AgentOutcome::Quota(o) => {
7395 self.state.quota.push(QuotaLoss {
7396 seat: seat.key.clone(),
7397 node: "gate-fix".to_owned(),
7398 at: Timestamp::now(),
7399 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7400 });
7401 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7402 }
7403 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7404 }
7405 self.state.seats.insert(seat.key.clone(), seat);
7406 let coordination = self.coordinate(&winner.worktree).await?;
7407 if let Ok(r) = git::rescue_commit(
7408 &winner.worktree,
7409 &format!("magi: gate fix {attempt} (uncommitted work)"),
7410 )
7411 .await
7412 {
7413 self.state.note_withheld("gate-fix", &r.withheld);
7414 }
7415 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7416 record.committed = after != before;
7417 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7418 let note = record.error.clone();
7419 self.state.gate_fixes.push(record);
7420 self.state.save()?;
7421 if !changed {
7422 self.state.event(
7423 "gate",
7424 match note {
7425 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7426 None => format!("gate-fix round {attempt}: the tree did not change"),
7427 },
7428 );
7429 return Ok(GateFix::Stop);
7430 }
7431 self.state.event(
7432 "gate",
7433 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7434 );
7435
7436 drop(coordination);
7437 let wait = self.repository_wait();
7438 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7439 }
7440
7441 async fn guarded_pr_message(
7448 &mut self,
7449 winner: &Candidate,
7450 facts: Option<&BranchFacts>,
7451 posting: bool,
7452 ) -> Result<Option<PrMessage>> {
7453 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7454 let head = match final_head.as_deref() {
7455 Some(h) => crate::pr_outcome::FinalHead::At(h),
7456 None => crate::pr_outcome::FinalHead::Unknown,
7457 };
7458 let summarized = if posting {
7460 Some(self.summarize_change(winner, final_head.as_deref()).await)
7461 } else {
7462 None
7463 };
7464 if self.state.resource_block.is_some() {
7469 self.state.event(
7470 "pr-summary",
7471 "a shared resource was busy; nothing was published, resume to summarize again",
7472 );
7473 return Ok(None);
7474 }
7475 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7476 anyhow::bail!(
7477 "the final commit moved during public description generation; retry before publishing"
7478 );
7479 }
7480 let note = match (&summarized, &self.state.change_summary) {
7481 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7482 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7483 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7484 (None, _) => ChangeNote::NotAsked,
7485 };
7486 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7487 if !posting {
7488 let identity = crate::scrub::Identity::current();
7489 return Ok(Some(PrMessage {
7490 title: crate::scrub::scrub(&pr.title, &identity),
7491 body: crate::scrub::scrub(&pr.body, &identity),
7492 }));
7493 }
7494 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7497 if let Some(known) = self
7498 .state
7499 .github_text
7500 .clone()
7501 .filter(|g| g.fingerprint == raw_fp)
7502 {
7503 return self.settle_withheld(winner, &pr, known).await;
7504 }
7505 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7506 let mut violations =
7507 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7508 let identity = crate::scrub::Identity::current();
7509 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7510 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7511 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7512 {
7513 violations.push(crate::github_text::Violation::SensitiveData);
7514 }
7515 let has_language = violations
7519 .iter()
7520 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7521 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7522 self.state.event(
7523 "github-text",
7524 if has_language {
7525 format!("description rejected: {violations:?}; requesting one rewrite")
7526 } else {
7527 format!("description rejected: {violations:?}; asking the owner")
7528 },
7529 );
7530 let mut rewritten = false;
7531 let mut rejected_rewrite = None;
7532 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7533 {
7534 let output = text;
7535 if let Some(title) = summary_title(&output) {
7536 let raw_body = summary_without_title(&output);
7539 let clean_title = crate::scrub::scrub(&title, &identity);
7540 let mut body = crate::scrub::scrub(&raw_body, &identity);
7541 let redacted = clean_title != title || body != raw_body;
7542 let title = clean_title;
7543 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7544 if !complete {
7545 self.state.event(
7546 "github-text",
7547 "description rewrite omitted required findings",
7548 );
7549 }
7550 if !body.trim().is_empty() {
7551 body.push_str("\n\n");
7552 body.push_str(&public_pr_evidence(&self.state, head));
7553 }
7554 let marker = format!("magi:run/{}", self.state.id);
7555 if !body.trim().is_empty() && !body.contains(&marker) {
7556 body.push_str(&format!("\n\n{marker}"));
7557 }
7558 let rewrite = PrMessage {
7559 title: title.clone(),
7560 body: body.clone(),
7561 };
7562 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7563 let mut failed = crate::github_text::check_for(
7564 &self.state.config.graph,
7565 &title,
7566 &body,
7567 redecision,
7568 );
7569 if (!crate::github_text::shareable(&title)
7570 || !crate::github_text::shareable(&body))
7571 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7572 {
7573 failed.push(crate::github_text::Violation::SensitiveData);
7574 }
7575 if complete
7576 && !title.trim().is_empty()
7577 && !body.trim().is_empty()
7578 && !failed.is_empty()
7579 {
7580 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7581 title: title.clone(),
7582 body: body.clone(),
7583 failed: failed
7584 .iter()
7585 .map(|v| crate::github_text::category(*v).to_owned())
7586 .collect(),
7587 });
7588 }
7589 if complete
7590 && !title.trim().is_empty()
7591 && !raw_body.trim().is_empty()
7592 && failed.is_empty()
7593 {
7594 decision = redecision;
7595 pr = PrMessage { title, body };
7596 rewritten = true;
7597 self.state
7598 .event("github-text", "description rewrite passed");
7599 if redacted {
7600 self.state.event(
7601 "github-text",
7602 "sensitive data removed from description rewrite",
7603 );
7604 }
7605 }
7606 }
7607 }
7608 if !rewritten {
7609 self.state.event(
7610 "github-text",
7611 "description rewrite unavailable or rejected; asking the owner",
7612 );
7613 if let Some(w) =
7614 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7615 {
7616 self.retire_github_text_question("the withheld text changed");
7619 let record = crate::run::GithubTextAsk {
7620 rewrite: rejected_rewrite,
7621 fingerprint: raw_fp,
7622 title: w.title,
7623 body: w.body,
7624 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7625 question: None,
7626 asks: 0,
7627 resolved: false,
7628 chosen_title: None,
7629 choice: None,
7630 };
7631 return self.settle_withheld(winner, &pr, record).await;
7632 }
7633 }
7634 }
7635 let (title, body) =
7636 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7637 Ok(Some(PrMessage { title, body }))
7638 }
7639
7640 async fn summarize_change(
7651 &mut self,
7652 winner: &Candidate,
7653 head: Option<&str>,
7654 ) -> std::result::Result<(), String> {
7655 let Some(head) = head else {
7656 return Err("the final commit could not be read".to_owned());
7657 };
7658 let remote = self.state.config.merge.remote.clone();
7659 let tracking = format!("{remote}/{}", self.state.base_branch);
7660 if !git::rev_exists(&winner.worktree, &tracking).await {
7661 return Err(format!("the remote base `{tracking}` could not be read"));
7662 }
7663 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7664 Ok(b) if !b.is_empty() => b,
7665 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7666 };
7667 let evidence = crate::pr_outcome::summary_evidence(
7668 &self.state,
7669 crate::pr_outcome::FinalHead::At(head),
7670 );
7671 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7672 if self
7673 .state
7674 .change_summary
7675 .as_ref()
7676 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7677 {
7678 return Ok(());
7679 }
7680 if self.state.change_summary.take().is_some() {
7681 self.state.event(
7682 "pr-summary",
7683 "the final diff or evidence changed; the earlier change summary was discarded",
7684 );
7685 }
7686 if let Err(e) = write_artifact(
7688 &self.state,
7689 &format!("public-description-{fingerprint}.md"),
7690 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7691 ) {
7692 tracing::warn!("could not save public description evidence: {e:#}");
7693 }
7694 let chain = match agent::pick_chain(
7695 &self.state.config.agents,
7696 self.state.config.roles.synthesizer.as_ref(),
7697 &agent::installed,
7698 "synthesizer",
7699 ) {
7700 Ok(chain) => chain,
7701 Err(e) => {
7702 tracing::warn!("no agent for the change summary: {e:#}");
7703 self.state.event("pr-summary", "no summary agent available");
7704 return Err("no agent was available to summarize it".to_owned());
7705 }
7706 };
7707 let subjects = git::subjects(&winner.worktree, &base, head)
7708 .await
7709 .unwrap_or_default();
7710 let stat = git::diff_stat(&winner.worktree, &base, head)
7711 .await
7712 .unwrap_or_default();
7713 let stat: String = stat.chars().take(6_000).collect();
7714 let mut prompt = crate::prompt::change_summary_in(
7715 &base,
7716 head,
7717 &subjects,
7718 stat.trim(),
7719 &self.state.config.graph.github_language,
7720 );
7721 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7722 let sessions = self.state.config.graph.sessions;
7723 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7724 let prompts = self.state.config.prompts.clone();
7725 let cache = self.state.config.cache_dir();
7726 let run = self.state.id.clone();
7727 for (i, spec) in chain.into_iter().enumerate() {
7728 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7729 let github_spec = match agent::for_github_text(
7730 &spec,
7731 &self.state.config.graph.github_language,
7732 &winner.worktree,
7733 ) {
7734 Ok(spec) => spec,
7735 Err(e) => {
7736 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7737 continue;
7738 }
7739 };
7740 let job = SeatJob {
7741 spec: github_spec,
7742 seat,
7743 cwd: winner.worktree.clone(),
7744 prompt: prompt.clone(),
7745 timeout,
7746 allow_write: false,
7747 sessions,
7748 artifacts: agent::artifacts_dir(&self.state.dir()),
7749 stem: if i == 0 {
7750 "change-summary".to_owned()
7751 } else {
7752 format!("change-summary-{}", spec.id)
7753 },
7754 handover: None,
7755 };
7756 let ctx = WaveCtx {
7757 run: &run,
7758 node: "pr-summary",
7759 prompts: &prompts,
7760 cache: cache.as_deref(),
7761 round: None,
7762 carry_seats: false,
7763 pauses: Some((&self.pause, &self.interrupt)),
7764 };
7765 let (seat, outcome) =
7766 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767 self.state.seats.insert(seat.key.clone(), seat);
7768 let text = match outcome {
7769 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7770 AgentOutcome::ResourceBlocked(why) => {
7771 if self.state.resource_block.is_none() {
7774 self.state.resource_block = Some(crate::run::ResourceBlock {
7775 cause: crate::run::ResourceBlockCause::Other,
7776 node: "pr-summary".to_owned(),
7777 kind: "shared resource".to_owned(),
7778 detail: why,
7779 at: Timestamp::now(),
7780 });
7781 }
7782 return Err("a shared resource was busy".to_owned());
7783 }
7784 _ => continue,
7785 };
7786 let blind = &self.state.config.blind;
7787 let Some(title) = summary_title(&text) else {
7788 continue;
7789 };
7790 let body: String = summary_without_title(&text)
7791 .lines()
7792 .filter(|l| !l.contains("magi:run/"))
7793 .collect::<Vec<_>>()
7794 .join("\n");
7795 let body = blind::sanitize_prose(body.trim(), blind);
7796 let title = blind::sanitize_prose(&title, blind);
7797 if body.trim().is_empty() || title.trim().is_empty() {
7798 continue;
7799 }
7800 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7801 continue;
7802 }
7803 if git::rev_parse(&winner.worktree, "HEAD")
7805 .await
7806 .ok()
7807 .as_deref()
7808 != Some(head)
7809 {
7810 return Err("the branch moved while the summary was written".to_owned());
7811 }
7812 if git::merge_base(&winner.worktree, &tracking, head)
7813 .await
7814 .ok()
7815 .as_deref()
7816 != Some(base.as_str())
7817 || crate::pr_outcome::summary_evidence(
7818 &self.state,
7819 crate::pr_outcome::FinalHead::At(head),
7820 ) != evidence
7821 {
7822 return Err("the summary evidence changed while it was written".to_owned());
7823 }
7824 self.state.change_summary = Some(crate::run::ChangeSummary {
7825 head: head.to_owned(),
7826 base,
7827 evidence: fingerprint,
7828 title,
7829 body,
7830 });
7831 self.state.event(
7832 "pr-summary",
7833 format!("change summary written by {} for the final diff", spec.id),
7834 );
7835 if let Err(e) = self.state.save() {
7836 tracing::warn!("could not save the change summary: {e:#}");
7837 }
7838 return Ok(());
7839 }
7840 self.state
7841 .event("pr-summary", "no agent produced a change summary");
7842 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7843 }
7844
7845 async fn rewrite_pr_text(
7855 &mut self,
7856 winner: &Candidate,
7857 pr: &PrMessage,
7858 violations: &[crate::github_text::Violation],
7859 ) -> Option<String> {
7860 let prompt = format!(
7861 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7862 pr.title, pr.body
7863 );
7864 let sessions = self.state.config.graph.sessions;
7865 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7866 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7869 let key = format!("impl-{}", winner.label);
7870 if let Some(spec) = self
7871 .state
7872 .config
7873 .agents
7874 .iter()
7875 .find(|a| a.id == winner.agent)
7876 .cloned()
7877 && let Some(existing) = self.state.seats.get(&key)
7878 && existing.agent == spec.id
7879 && agent::has_session(spec.kind, existing, sessions)
7880 {
7881 attempts.push((spec, Some(existing.clone())));
7882 }
7883 match agent::pick_chain(
7884 &self.state.config.agents,
7885 self.state.config.roles.language_judge.as_ref(),
7886 &agent::installed,
7887 "language_judge",
7888 ) {
7889 Ok(chain) => {
7890 for spec in chain {
7891 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7892 continue;
7893 }
7894 attempts.push((spec, None));
7895 }
7896 }
7897 Err(e) if attempts.is_empty() => {
7898 tracing::warn!("no agent for the description rewrite: {e:#}");
7899 self.state
7900 .event("github-text", "no rewrite agent available");
7901 return None;
7902 }
7903 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7904 }
7905 let prompts = self.state.config.prompts.clone();
7906 let cache = self.state.config.cache_dir();
7907 let run = self.state.id.clone();
7908 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7909 let resumed = resumed_seat.is_some();
7910 let seat = resumed_seat.unwrap_or_else(|| {
7911 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7912 });
7913 let github_spec = match agent::for_github_text(
7914 &spec,
7915 &self.state.config.graph.github_language,
7916 &winner.worktree,
7917 ) {
7918 Ok(spec) => spec,
7919 Err(e) => {
7920 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7921 continue;
7922 }
7923 };
7924 let job = SeatJob {
7925 spec: github_spec,
7926 seat,
7927 cwd: winner.worktree.clone(),
7928 prompt: prompt.clone(),
7929 timeout: retry_budget(timeout_full, resumed),
7930 allow_write: false,
7931 sessions,
7932 artifacts: agent::artifacts_dir(&self.state.dir()),
7933 stem: if i == 0 {
7934 "github-text-rewrite".to_owned()
7935 } else {
7936 format!("github-text-rewrite-{}", spec.id)
7937 },
7938 handover: None,
7939 };
7940 let ctx = WaveCtx {
7941 run: &run,
7942 node: "github-text",
7943 prompts: &prompts,
7944 cache: cache.as_deref(),
7945 round: None,
7946 carry_seats: false,
7947 pauses: Some((&self.pause, &self.interrupt)),
7948 };
7949 let (seat, outcome) =
7950 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7951 self.state.seats.insert(seat.key.clone(), seat);
7952 match outcome {
7953 AgentOutcome::Ok(o)
7954 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7955 {
7956 return Some(o.text);
7957 }
7958 AgentOutcome::ResourceBlocked(_) => return None,
7959 _ => {}
7960 }
7961 }
7962 None
7963 }
7964
7965 fn retire_github_text_question(&mut self, why: &str) {
7967 let Some(id) = self
7968 .state
7969 .github_text
7970 .as_ref()
7971 .and_then(|g| g.question.clone())
7972 else {
7973 return;
7974 };
7975 let _ = ask::Questions::open().update(&id, |q| {
7976 q.abandon(why);
7977 Ok(())
7978 });
7979 }
7980
7981 async fn settle_withheld(
7985 &mut self,
7986 winner: &Candidate,
7987 pr: &PrMessage,
7988 mut g: crate::run::GithubTextAsk,
7989 ) -> Result<Option<PrMessage>> {
7990 use crate::github_text as gt;
7991 let store = ask::Questions::open();
7992 let timeout = self.state.config.graph.answer_timeout;
7993 if !g.resolved {
7994 let mut standing = g
7995 .question
7996 .as_ref()
7997 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7998 if let Some(q) = standing.as_ref()
7999 && q.status.open()
8000 && gt::expired(q, timeout)
8001 {
8002 let why = format!("no answer within {}s of asking", timeout.max(1));
8003 standing = store
8004 .update(&q.id, |q| {
8005 q.abandon(&why);
8006 Ok(())
8007 })
8008 .ok()
8009 .map(|(q, ())| q);
8010 }
8011 let mut ask_again: Option<Option<Vec<String>>> = None;
8012 match standing {
8013 None if g.asks <= 1 => {
8016 g.asks = 0;
8017 ask_again = Some(None);
8018 }
8019 None => {
8020 g.resolved = true;
8021 g.chosen_title = None;
8022 g.choice = Some(GithubTextChoice::Fallback);
8023 }
8024 Some(q) if q.status.open() => {
8025 self.park_on_github_text(&g)?;
8026 return Ok(None);
8027 }
8028 Some(q) => match gt::read_reply(&q) {
8029 gt::Reply::Fallback => {
8030 g.resolved = true;
8031 g.chosen_title = None;
8032 g.choice = Some(GithubTextChoice::Fallback);
8033 }
8034 gt::Reply::Original => {
8035 g.resolved = true;
8036 g.chosen_title = None;
8037 g.choice = Some(GithubTextChoice::Original);
8038 }
8039 gt::Reply::Rewrite => {
8040 g.resolved = true;
8041 g.chosen_title = None;
8042 g.choice = Some(if g.rewrite.is_some() {
8043 GithubTextChoice::Rewrite
8044 } else {
8045 GithubTextChoice::Fallback
8046 });
8047 }
8048 gt::Reply::Title(text) => {
8049 let first = text
8050 .lines()
8051 .map(str::trim)
8052 .find(|l| !l.is_empty())
8053 .unwrap_or("");
8054 let decision = if first.is_empty() {
8055 None
8056 } else {
8057 let candidate = PrMessage {
8058 title: first.to_owned(),
8059 body: String::new(),
8060 };
8061 self.judge_pr_language(&winner.worktree, &candidate).await
8062 };
8063 match gt::vet_title(
8064 &text,
8065 decision,
8066 self.state.config.graph.github_english(),
8067 ) {
8068 Ok(title) => {
8069 g.resolved = true;
8070 g.chosen_title = Some(title);
8071 g.choice = Some(GithubTextChoice::MyTitle);
8072 }
8073 Err(cats) if g.asks < 2 => {
8074 ask_again = Some(Some(
8075 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
8076 ))
8077 }
8078 Err(_) => {
8079 g.resolved = true;
8080 g.chosen_title = None;
8081 g.choice = Some(GithubTextChoice::Fallback);
8082 }
8083 }
8084 }
8085 },
8086 }
8087 if let Some(rejected) = ask_again {
8088 let w = gt::Withheld {
8091 title: g.title,
8092 body: g.body,
8093 categories: g.categories.clone(),
8094 };
8095 let artifact = match crate::run::write_artifact(
8098 &self.state,
8099 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
8100 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
8101 ) {
8102 Ok(p) => Some(p),
8103 Err(e) => {
8104 tracing::warn!("could not write the withheld-text artifact: {e:#}");
8105 None
8106 }
8107 };
8108 let (detail, comparison) = gt::question_content(
8109 &w,
8110 &pr.title,
8111 &pr.body,
8112 &self.state.id,
8113 rejected.as_deref(),
8114 artifact.as_deref(),
8115 g.rewrite.as_ref(),
8116 );
8117 let mut q = ask::Question::new(
8118 self.state.id.clone(),
8119 gt::ASK_NODE.to_owned(),
8120 gt::ASK_SEAT.to_owned(),
8121 gt::question_summary(&self.state.config.graph.language, &w),
8122 detail,
8123 gt::question_choices(&w, g.rewrite.as_ref()),
8124 );
8125 q.posting_comparison = Some(comparison);
8126 q.answer_timeout = timeout;
8127 g.question = Some(q.id.clone());
8130 g.asks += 1;
8131 self.state.github_text = Some(g.clone());
8132 self.state.event(
8133 "github-text",
8134 format!("asking the owner about the withheld text ({})", q.short()),
8135 );
8136 self.state.save()?;
8137 store
8138 .put_executable(&mut q)
8139 .context("file the posting-gate question")?;
8140 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8141 tracing::warn!(
8142 "could not notify about posting-gate question {}: {e:#}",
8143 q.short()
8144 );
8145 }
8146 self.park_on_github_text(&g)?;
8147 return Ok(None);
8148 }
8149 }
8150 let resolved = g.chosen_title.clone();
8151 let original = g.choice == Some(GithubTextChoice::Original);
8152 self.state.event(
8154 "github-text",
8155 if original {
8156 format!(
8157 "posting the original text for: {}",
8158 match (g.title, g.body) {
8159 (true, true) => "title, description",
8160 (true, false) => "title",
8161 _ => "description",
8162 }
8163 )
8164 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8165 "posting the saved rewrite title and description".to_owned()
8166 } else if resolved.is_some() {
8167 "posting the owner's replacement title".to_owned()
8168 } else {
8169 "posting the neutral text for the withheld fields".to_owned()
8170 },
8171 );
8172 self.state.github_text = Some(g.clone());
8173 self.state.save()?;
8174 if g.choice == Some(GithubTextChoice::Rewrite)
8175 && let Some(r) = g.rewrite.as_ref()
8176 {
8177 return Ok(Some(PrMessage {
8178 title: r.title.clone(),
8179 body: r.body.clone(),
8180 }));
8181 }
8182 let w = gt::Withheld {
8183 title: g.title,
8184 body: g.body,
8185 categories: g.categories.clone(),
8186 };
8187 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8188 if let Some(t) = resolved {
8189 title = t;
8190 }
8191 let english = gt::LanguageDecision {
8193 title_english: true,
8194 body_english: true,
8195 body_complete: true,
8196 };
8197 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8198 if original {
8199 if g.title {
8202 title = pr.title.clone();
8203 }
8204 if g.body {
8205 body = pr.body.clone();
8206 }
8207 }
8208 Ok(Some(PrMessage { title, body }))
8209 }
8210
8211 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8212 self.state.github_text = Some(g.clone());
8213 self.state.end_pass_parked(ParkCause::Question);
8214 self.state.parked = true;
8215 self.state.event(
8216 "github-text",
8217 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8218 );
8219 self.state.save()
8220 }
8221
8222 async fn judge_pr_language(
8225 &mut self,
8226 cwd: &Path,
8227 pr: &PrMessage,
8228 ) -> Option<crate::github_text::LanguageDecision> {
8229 self.state.config.roles.language_judge.as_ref()?;
8230 if !self.state.config.graph.language_gate() {
8231 return None;
8232 }
8233 let decision =
8234 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8235 self.state.event(
8236 "github-text",
8237 if decision.is_some() {
8238 "language decided by the language judge"
8239 } else {
8240 "language judge unavailable; using built-in heuristics"
8241 },
8242 );
8243 decision
8244 }
8245
8246 async fn merge(&mut self) -> Result<()> {
8247 if self
8262 .state
8263 .base_sync
8264 .as_ref()
8265 .is_some_and(|s| s.conflict.is_some())
8266 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8267 != Some(RunStatus::Gating)
8268 || !self.state.gate_allows_progress()
8276 {
8277 return Ok(());
8278 }
8279 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8288 return Ok(());
8289 }
8290 let Some(winner) = self.state.winner().cloned() else {
8291 return Ok(());
8292 };
8293 let repo = self.state.repo.clone();
8294 let base = self.state.base_branch.clone();
8295 let mode = self.state.config.merge.mode;
8296 let style = self.state.config.merge.style;
8297 let facts = if is_review_run(&self.state) {
8298 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8299 let start = review_base(
8300 &repo,
8301 &self.state.config.merge.remote,
8302 &base,
8303 &self.state.base_commit,
8304 &winner.branch,
8305 )
8306 .await;
8307 branch_facts(&repo, &start, &winner.branch).await
8308 } else {
8309 None
8310 };
8311 let leaked = if is_review_run(&self.state) {
8314 leaked_subjects(&self.state, &winner.branch).await
8315 } else {
8316 Some(Vec::new())
8317 };
8318 let Some(pr) = self
8322 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8323 .await?
8324 else {
8325 return Ok(());
8326 };
8327 let message = pr.commit_message();
8328
8329 let outcome = match mode {
8330 MergeMode::None => MergeOutcome {
8331 mode,
8332 ok: true,
8333 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8334 empty: false,
8335 },
8336 MergeMode::Pr | MergeMode::Local
8337 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8338 {
8339 MergeOutcome {
8340 mode,
8341 ok: false,
8342 detail: empty_candidate_detail(&self.state, &base),
8343 empty: true,
8344 }
8345 }
8346 MergeMode::Local => {
8347 let on = git::current_branch(&repo).await?;
8348 if on.as_deref() != Some(base.as_str()) {
8349 MergeOutcome {
8350 mode,
8351 ok: false,
8352 detail: format!(
8353 "{} has {} checked out, not the base branch {base}",
8354 repo.display(),
8355 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8356 ),
8357 empty: false,
8358 }
8359 } else if !git::is_clean(&repo).await? {
8360 MergeOutcome {
8361 mode,
8362 ok: false,
8363 detail: format!("{} is dirty; refusing to merge", repo.display()),
8364 empty: false,
8365 }
8366 } else {
8367 let out = match style {
8368 MergeStyle::Merge => {
8369 git::merge_no_ff(&repo, &winner.branch, &message).await?
8370 }
8371 MergeStyle::Squash => {
8372 git::merge_squash(&repo, &winner.branch, &message).await?
8373 }
8374 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8375 };
8376 MergeOutcome {
8377 mode,
8378 ok: out.ok(),
8379 detail: if out.ok() { out.stdout } else { out.stderr },
8380 empty: false,
8381 }
8382 }
8383 }
8384 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8385 self.state.pr_publication_pending = true;
8386 self.state.save()?;
8387 let out =
8388 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8389 self.state.pr_publication_pending = out.is_err();
8390 MergeOutcome {
8391 mode,
8392 ok: out.is_ok(),
8393 detail: match out {
8394 Ok(url) => url,
8395 Err(e) => format!("{e:#}"),
8396 },
8397 empty: false,
8398 }
8399 }
8400 MergeMode::Pr => {
8401 let remote = self.state.config.merge.remote.clone();
8402 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8403 if !pushed.ok() {
8404 MergeOutcome {
8405 mode,
8406 ok: false,
8407 detail: pushed.stderr,
8408 empty: false,
8409 }
8410 } else {
8411 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8417 let out = match pr_merge_plan(found) {
8418 PrPlan::Create => {
8419 gh_pr_create(
8420 &winner.worktree,
8421 &base,
8422 &winner.branch,
8423 &pr.title,
8424 &pr.body,
8425 )
8426 .await
8427 }
8428 PrPlan::Adopt { url, title } => {
8429 self.state
8430 .event("merge", format!("Pr: adopted open pull request {url}"));
8431 if self.state.github_text.as_ref().is_some_and(|g| {
8432 g.resolved
8433 && g.choice == Some(GithubTextChoice::Rewrite)
8434 && g.rewrite.is_some()
8435 }) {
8436 land::set_pr_text_verbatim(
8439 &winner.worktree,
8440 &url,
8441 &pr.title,
8442 &pr.body,
8443 )
8444 .await
8445 .map(|()| url)
8446 } else {
8447 if title != pr.title
8448 && (!is_review_run(&self.state)
8449 || leaked
8450 .as_deref()
8451 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8452 && let Err(e) =
8453 if self.state.github_text.as_ref().is_some_and(|g| {
8454 g.resolved
8455 && g.title
8456 && g.choice == Some(GithubTextChoice::Original)
8457 }) {
8458 land::set_pr_title_verbatim(
8460 &winner.worktree,
8461 &url,
8462 &pr.title,
8463 )
8464 .await
8465 } else {
8466 land::set_pr_title(
8467 &mut self.state,
8468 &winner.worktree,
8469 &url,
8470 &pr.title,
8471 )
8472 .await
8473 }
8474 {
8475 tracing::warn!("could not refresh title of {url}: {e:#}");
8476 self.state
8477 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8478 }
8479 Ok(url)
8480 }
8481 }
8482 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8483 };
8484 match out {
8485 Ok(url) => MergeOutcome {
8486 mode,
8487 ok: true,
8488 detail: url,
8489 empty: false,
8490 },
8491 Err(e) => MergeOutcome {
8492 mode,
8493 ok: false,
8494 detail: e.to_string(),
8495 empty: false,
8496 },
8497 }
8498 }
8499 }
8500 };
8501
8502 self.state.status = match (mode, outcome.ok) {
8503 (MergeMode::None, _) => RunStatus::Ready,
8504 (_, true) if outcome.empty => RunStatus::Ready,
8505 (MergeMode::Pr, true) => RunStatus::Ready,
8506 (_, true) => RunStatus::Merged,
8507 (_, false) => RunStatus::Blocked,
8508 };
8509 self.state.event(
8510 "merge",
8511 format!(
8512 "{:?}: {}",
8513 mode,
8514 outcome.detail.lines().next().unwrap_or("")
8515 ),
8516 );
8517 self.state.merge = Some(outcome);
8518 self.state.save()?;
8519
8520 if self.state.config.graph.land
8526 && mode == MergeMode::Pr
8527 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8528 {
8529 self.run_land().await?;
8530 }
8531 self.settle_questions();
8536 Ok(())
8537 }
8538
8539 async fn run_land(&mut self) -> Result<()> {
8550 crate::pr_head::ensure_owner(&self.state)?;
8551 let url = self
8552 .state
8553 .pr
8554 .as_ref()
8555 .map(|pr| pr.url.clone())
8556 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8557 .unwrap_or_default();
8558 let url = url.lines().next().unwrap_or("").trim().to_owned();
8559 if !url.starts_with("http") {
8560 return Ok(());
8561 }
8562 loop {
8565 match land::land(&mut self.state, &url).await {
8566 Ok(_)
8567 if self
8568 .state
8569 .merge_repairs
8570 .last()
8571 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8572 {
8573 if self.repair_merge_review().await? {
8574 continue;
8575 }
8576 }
8577 Ok(pr) if self.state.parked => {
8578 let _ = pr;
8582 }
8583 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8584 self.state.status = RunStatus::Superseded;
8585 self.state.save()?;
8586 }
8587 Ok(pr) => {
8588 self.state.status = match pr.state {
8589 land::PrLifecycle::Merged => RunStatus::Merged,
8590 _ => RunStatus::Blocked,
8591 };
8592 if bump::should_release_bump(self.state.status)
8599 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8600 {
8601 self.state
8607 .event("bump", format!("release bump skipped: {e:#}"));
8608 }
8609 if self.state.status == RunStatus::Merged {
8612 crate::followup::after_merge(&mut self.state, &pr.url).await;
8613 }
8614 self.state.save()?;
8615 }
8616 Err(e) => {
8617 if e.is::<crate::repo_lock::ResourceBlocked>() {
8618 return Err(e);
8619 }
8620 self.state.status = RunStatus::Blocked;
8621 self.state.event("land", format!("gave up: {e}"));
8622 self.state.save()?;
8623 }
8624 }
8625 break;
8626 }
8627 Ok(())
8628 }
8629
8630 async fn repair_merge_review(&mut self) -> Result<bool> {
8633 use crate::run::MergeRepairPhase as Phase;
8634 crate::pr_head::ensure_owner(&self.state)?;
8635 let i = self.state.merge_repairs.len() - 1;
8636 let request = self.state.merge_repairs[i].clone();
8637 if request.phase == Phase::Fixing {
8638 let restored = match self.state.winner().cloned() {
8639 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8640 None => Err(anyhow::anyhow!("repair has no winning branch")),
8641 };
8642 let why = match restored {
8643 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8644 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8645 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8646 };
8647 self.hold_merge_repair(i, why)?;
8648 return Ok(false);
8649 }
8650 let failure = if i >= self.state.config.graph.land_rounds
8651 || self.state.config.graph.review_rounds == 0
8652 {
8653 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8654 } else {
8655 None
8656 };
8657 if let Some(why) = failure {
8658 self.hold_merge_repair(i, why)?;
8659 return Ok(false);
8660 }
8661 if request.phase == Phase::Requested {
8662 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8663 Ok(pr) => pr,
8664 Err(e) => {
8665 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8666 return Ok(false);
8667 }
8668 };
8669 self.state.merge_repairs[i].phase = Phase::Fixing;
8670 self.state.land_approval = None;
8671 self.state.land_armed_head = None;
8672 self.state.save()?;
8673 let evidence = serde_json::to_string_pretty(&request)?;
8676 let reason = format!(
8677 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8678 );
8679 let budget = self.state.config.graph.land_rounds;
8680 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8681 match result {
8682 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8683 Ok(land::Fixed::Failed(why)) => {
8684 self.hold_merge_repair(
8685 i,
8686 format!(
8687 "repair failed: {why}; inspect the branch and run an explicit review"
8688 ),
8689 )?;
8690 return Ok(false);
8691 }
8692 Err(e) => {
8693 self.hold_merge_repair(
8694 i,
8695 format!(
8696 "repair failed: {e:#}; inspect the branch and run an explicit review"
8697 ),
8698 )?;
8699 return Ok(false);
8700 }
8701 }
8702 self.begin_merge_repair_review(i)?;
8706 }
8707 self.review_loop().await?;
8708 self.gate().await?;
8709 if self.resource_halt()? || self.state.parked {
8710 return Ok(false);
8711 }
8712 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8713 != Some(RunStatus::Gating)
8714 || !self.state.gate_allows_progress()
8715 {
8716 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8717 return Ok(false);
8718 }
8719 let winner = self
8722 .state
8723 .winner()
8724 .cloned()
8725 .context("repair has no winning branch")?;
8726 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8727 if self.state.gate_head.as_deref() != Some(after.as_str())
8728 || !git::is_clean(&winner.worktree).await?
8729 {
8730 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8731 return Ok(false);
8732 }
8733 let expected = self
8734 .state
8735 .pr_pushed_head
8736 .as_deref()
8737 .unwrap_or(&request.head)
8738 .to_owned();
8739 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8740 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8743 let refspec = format!("{after}:refs/heads/{public}");
8744 let push = git::git_raw(
8745 &winner.worktree,
8746 &["push", &lease, &self.state.config.merge.remote, &refspec],
8747 )
8748 .await;
8749 match push {
8750 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8751 Ok(out) => {
8752 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8753 return Ok(false);
8754 }
8755 Err(e) => {
8756 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8757 return Ok(false);
8758 }
8759 }
8760 self.state.merge_repairs[i].phase = Phase::Complete;
8761 self.state.status = RunStatus::Landing;
8762 self.state.event(
8763 "repair",
8764 "new review and gate completed; fresh owner merge decision required",
8765 );
8766 self.state.save()?;
8767 Ok(true)
8768 }
8769
8770 fn merge_repair_artifacts(&self) -> PathBuf {
8771 let artifacts = agent::artifacts_dir(&self.state.dir());
8772 match self
8773 .state
8774 .merge_repairs
8775 .last()
8776 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8777 {
8778 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8779 None => artifacts,
8780 }
8781 }
8782
8783 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8784 self.state.reviews.clear();
8785 self.state.gate.clear();
8786 self.state.gate_ran = false;
8787 self.state.gate_head = None;
8788 self.state.gate_fixes.clear();
8789 self.state.gate_repair_e2e = None;
8790 self.state.pre_gate.clear();
8791 self.state.pre_gate_commit = None;
8792 self.state.contested_handoff = None;
8793 self.state.land_reruns = None;
8794 self.state.land_approval = None;
8795 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8796 self.state.status = RunStatus::Reviewing;
8797 self.state.save()
8798 }
8799
8800 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8801 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8802 self.state.merge_repairs[index].failure = Some(why.clone());
8803 self.state.status = RunStatus::Blocked;
8804 self.state.event("repair", why);
8805 self.state.save()
8806 }
8807
8808 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8812 if let Some(existing) = self.state.seats.get(key)
8813 && existing.agent == agent
8814 {
8815 return existing.clone();
8816 }
8817 let fresh = if self.state.seats.contains_key(key) {
8820 handover_seat(key, agent, self.state.next_seat_seed())
8821 } else {
8822 SeatState::new(key, agent, self.state.seed)
8823 };
8824 self.state.seats.insert(key.to_owned(), fresh.clone());
8825 fresh
8826 }
8827
8828 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8834 match self.state.seats.get(key) {
8835 Some(s) if s.agent != spec.id => {
8836 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8837 }
8838 _ => spec,
8839 }
8840 }
8841
8842 fn view(&self, c: &Candidate) -> CandidateView {
8844 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8845 .unwrap_or_default();
8846 let (patch, _) = blind::sanitize_patch(
8847 &format!("candidate {} patch", c.label),
8848 &raw,
8849 &self.state.config.blind,
8850 );
8851 CandidateView {
8852 label: c.label,
8853 branch: c.branch.clone(),
8854 summary: c.summary.clone(),
8855 stat: c.stat.clone(),
8856 patch,
8857 }
8858 }
8859
8860 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8862 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8863 prompt::judge(
8864 "(see above)",
8865 &views,
8866 self.roles.judges.len(),
8867 base_short,
8868 "en",
8869 )
8870 }
8871
8872 fn vote_prompt_full(
8878 &self,
8879 j: usize,
8880 labels: &[char],
8881 language: &str,
8882 candidates: &[Candidate],
8883 base_short: &str,
8884 ) -> String {
8885 let mut text = format!(
8886 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8887 prompt::final_vote(labels, language),
8888 self.state.instruction,
8889 self.candidate_block(candidates, base_short)
8890 );
8891 if let Some(own) = self
8892 .state
8893 .judgements
8894 .get(j)
8895 .filter(|r| !r.ranking.is_empty())
8896 {
8897 let reasons = own
8898 .reasons
8899 .iter()
8900 .map(|(k, v)| format!("- {k}: {v}"))
8901 .collect::<Vec<_>>()
8902 .join("\n");
8903 text.push_str(&format!(
8904 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8905 own.ranking.iter().collect::<String>(),
8906 if reasons.is_empty() {
8907 ""
8908 } else {
8909 ", because:\n"
8910 }
8911 ));
8912 }
8913 if !self.state.deliberation.is_empty() {
8914 text.push_str("\n# What was argued before this vote\n");
8915 for t in self.transcript(&[], j) {
8916 text.push_str(&format!(
8917 "\n## {}{}\n\n{}\n",
8918 t.who,
8919 if t.is_self { " (you)" } else { "" },
8920 t.body.trim()
8921 ));
8922 }
8923 }
8924 text
8925 }
8926
8927 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8934 let mut turns = Vec::new();
8935 for j in &self.state.judgements {
8936 if j.ranking.is_empty() {
8937 continue;
8938 }
8939 let reasons = j
8940 .reasons
8941 .iter()
8942 .map(|(k, v)| format!("- {k}: {v}"))
8943 .collect::<Vec<_>>()
8944 .join("\n");
8945 turns.push(Turn {
8946 who: format!("Judge {} (opening ranking)", j.judge),
8947 is_self: j.judge == self_idx + 1,
8948 body: format!(
8949 "Ranked {}{}{reasons}",
8950 j.ranking.iter().collect::<String>(),
8951 if reasons.is_empty() {
8952 ""
8953 } else {
8954 ", because:\n"
8955 }
8956 ),
8957 });
8958 }
8959 for t in self
8960 .state
8961 .deliberation
8962 .iter()
8963 .flat_map(|r| r.turns.iter())
8964 .chain(current)
8965 {
8966 turns.push(Turn {
8967 who: format!("Judge {}", t.judge),
8968 is_self: t.judge == self_idx + 1,
8969 body: t.body.clone(),
8970 });
8971 }
8972 turns
8973 }
8974}
8975
8976fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8978 agent::has_session(spec.kind, seat, sessions)
8979}
8980
8981fn next_untried_in_roster<'a>(
9002 roster: &'a [AgentSpec],
9003 start: usize,
9004 tried: &BTreeSet<String>,
9005) -> Option<&'a AgentSpec> {
9006 roster
9007 .get(start + 1..)?
9008 .iter()
9009 .find(|s| !tried.contains(&s.id))
9010}
9011
9012fn pick_successor<'a>(
9033 roster: &'a [AgentSpec],
9034 start: usize,
9035 tried: &BTreeSet<String>,
9036 carried: Option<&BTreeSet<String>>,
9037 others: &BTreeSet<String>,
9038) -> Option<&'a AgentSpec> {
9039 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
9040 roster
9041 .get(start + 1..)
9042 .and_then(|tail| {
9043 tail.iter()
9044 .filter(free)
9045 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
9046 })
9047 .or_else(|| {
9048 carried
9049 .is_some()
9050 .then(|| roster.iter().find(free))
9051 .flatten()
9052 })
9053 .or_else(|| match carried {
9054 Some(c) => next_for_seat(roster, start, tried, c),
9055 None => next_untried_in_roster(roster, start, tried),
9056 })
9057}
9058
9059fn next_for_seat<'a>(
9069 roster: &'a [AgentSpec],
9070 start: usize,
9071 round_tried: &BTreeSet<String>,
9072 carried_failed: &BTreeSet<String>,
9073) -> Option<&'a AgentSpec> {
9074 roster
9075 .get(start + 1..)?
9076 .iter()
9077 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
9078 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
9079}
9080
9081fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
9087 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
9088 return spec;
9089 };
9090 let ok = |id: &str| !h.failed.contains(id);
9091 if let Some(last) = h.last_ok.as_deref()
9092 && ok(last)
9093 && let Some(s) = roster.iter().find(|s| s.id == last)
9094 {
9095 return s.clone();
9096 }
9097 if ok(&spec.id) {
9098 return spec;
9099 }
9100 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
9101 roster
9102 .get(from + 1..)
9103 .into_iter()
9104 .flatten()
9105 .chain(roster.iter())
9106 .find(|s| ok(&s.id))
9107 .cloned()
9108 .unwrap_or(spec)
9109}
9110
9111pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
9115 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
9116}
9117
9118const TIMED_OUT: &str = "timed out";
9121
9122impl FailClass {
9123 fn of(out: &AgentOutcome) -> Option<Self> {
9125 match out {
9126 AgentOutcome::Ok(_) => None,
9127 AgentOutcome::Quota(_) => Some(Self::Quota),
9128 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9129 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9130 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9131 AgentOutcome::ResourceBlocked(_) => None,
9134 }
9135 }
9136
9137 fn stem_word(&self) -> &'static str {
9139 match self {
9140 Self::Quota => "quota",
9141 _ => "handover",
9142 }
9143 }
9144}
9145
9146fn failure_signature(msg: &str) -> String {
9150 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9151 if let Some(code) = line
9154 .strip_prefix("process exited with no usable output (exit code ")
9155 .and_then(|s| s.strip_suffix(')'))
9156 .and_then(|s| s.parse::<i32>().ok())
9157 {
9158 line = format!("exited with some({code}) and no usable output");
9159 } else if matches!(
9160 line.as_str(),
9161 "process exited with no usable output (no exit code reported)"
9162 | "process exited with no usable output (exit code unavailable)"
9163 ) {
9164 line = "exited with none and no usable output".to_owned();
9165 }
9166 let mut out = Vec::new();
9167 for word in line.split_whitespace() {
9168 if word.contains('/') || word.contains('\\') {
9169 out.push("<path>".to_owned());
9170 continue;
9171 }
9172 let mut w = String::new();
9173 let mut in_digits = false;
9174 for c in word.chars() {
9175 if c.is_ascii_digit() {
9176 if !in_digits {
9177 w.push('#');
9178 }
9179 in_digits = true;
9180 } else {
9181 in_digits = false;
9182 w.push(c);
9183 }
9184 }
9185 out.push(w);
9186 }
9187 out.join(" ").chars().take(120).collect()
9188}
9189
9190fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9197 match cur {
9198 FailClass::Quota | FailClass::Timeout => true,
9199 FailClass::Other(_) => prev != Some(cur),
9200 }
9201}
9202
9203fn fail_reason(out: &AgentOutcome) -> String {
9205 match out {
9206 AgentOutcome::Ok(_) => String::new(),
9207 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9208 AgentOutcome::Dropped(o) => format!(
9209 "the CLI dropped the stream ({})",
9210 o.dropped
9211 .as_ref()
9212 .map(|d| d.why.as_str())
9213 .unwrap_or("it ended without delivering its answer")
9214 ),
9215 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9216 e.lines().next().unwrap_or("").chars().take(160).collect()
9217 }
9218 }
9219}
9220
9221pub(crate) fn record_handover(
9224 state: &mut RunState,
9225 node: &str,
9226 seat: &str,
9227 from: &str,
9228 to: &str,
9229 class: &FailClass,
9230 reason: &str,
9231) {
9232 let message = if *class == FailClass::Quota {
9233 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9234 } else {
9235 format!("{seat}: handed over {from} -> {to} ({reason})")
9236 };
9237 state.event(node, message);
9238 state.handovers.push(Handover {
9239 at: Timestamp::now(),
9240 node: node.to_owned(),
9241 seat: seat.to_owned(),
9242 from: from.to_owned(),
9243 to: to.to_owned(),
9244 reason: reason.to_owned(),
9245 });
9246}
9247
9248fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9262 commands.iter().any(|c| c.exit_code.is_none())
9263}
9264
9265fn verified_noop_claim(
9278 usable: bool,
9279 commands: &[agent::CommandEvidence],
9280 text: &str,
9281) -> Option<String> {
9282 (usable && !has_unconfirmed_command(commands))
9283 .then(|| verdict::verified_noop(text))
9284 .flatten()
9285}
9286
9287fn short(commit: &str) -> String {
9288 commit.chars().take(7).collect()
9289}
9290
9291fn make_executable(path: &Path) -> Result<()> {
9292 #[cfg(unix)]
9293 {
9294 use std::os::unix::fs::PermissionsExt as _;
9295 let mut perms = std::fs::metadata(path)?.permissions();
9296 perms.set_mode(0o755);
9297 std::fs::set_permissions(path, perms)?;
9298 }
9299 #[cfg(not(unix))]
9300 {
9301 let _ = path;
9302 }
9303 Ok(())
9304}
9305
9306struct WaveCtx<'a> {
9313 run: &'a str,
9316 node: &'a str,
9318 prompts: &'a Prompts,
9319 cache: Option<&'a Path>,
9321 round: Option<usize>,
9324 carry_seats: bool,
9328 pauses: Option<(&'a Pause, &'a Pause)>,
9331}
9332
9333fn repository_wait(
9335 state: &RunState,
9336 pauses: Option<(&Pause, &Pause)>,
9337) -> crate::repo_lock::WaitPolicy {
9338 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9339 let inherited = crate::repo_lock::WaitPolicy::current();
9340 crate::repo_lock::WaitPolicy::new(
9341 Duration::from_secs(state.config.graph.timeout_resource_wait),
9342 move || {
9343 inherited.cancelled()
9344 || pauses
9345 .as_ref()
9346 .is_some_and(|(a, b)| a.parked() || b.parked())
9347 },
9348 )
9349}
9350
9351fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9352 if error.is::<crate::repo_lock::ResourceBlocked>() {
9353 state.resource_block = Some(crate::run::ResourceBlock {
9354 cause: crate::run::ResourceBlockCause::Other,
9355 node: "repository".into(),
9356 kind: if error.to_string().contains("disk pressure") {
9357 "disk pressure"
9358 } else {
9359 "repository coordination"
9360 }
9361 .into(),
9362 detail: error.to_string(),
9363 at: Timestamp::now(),
9364 });
9365 state.event("resource", error.to_string());
9366 }
9367}
9368
9369async fn coordinate_operation(
9370 state: &mut RunState,
9371 cwd: &Path,
9372 wait: &crate::repo_lock::WaitPolicy,
9373) -> Result<Option<crate::repo_lock::SeatLease>> {
9374 let result =
9375 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9376 .await;
9377 if let Err(e) = &result {
9378 record_repository_block(state, e);
9379 }
9380 result
9381}
9382
9383async fn run_one(
9385 job: SeatJob,
9386 sem: Arc<Semaphore>,
9387 ctx: &WaveCtx<'_>,
9388 state: &mut RunState,
9389 attempt: usize,
9390) -> (SeatState, AgentOutcome) {
9391 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9392 .await
9393 .pop()
9394 .expect("one job in, one result out");
9395 (seat, out)
9396}
9397
9398async fn wave(
9404 jobs: Vec<SeatJob>,
9405 sem: Arc<Semaphore>,
9406 ctx: &WaveCtx<'_>,
9407 state: &mut RunState,
9408 attempt: usize,
9409) -> Vec<(usize, SeatState, AgentOutcome)> {
9410 let WaveCtx {
9411 run,
9412 node,
9413 prompts,
9414 cache,
9415 round,
9416 carry_seats: _,
9417 pauses,
9418 } = *ctx;
9419 for job in &jobs {
9420 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9421 }
9422 if let Err(e) = state.save() {
9423 tracing::warn!("could not persist in-progress seats: {e:#}");
9428 }
9429 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9445 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9449 let cache_guard = if let Some(cache_dir) = cache {
9450 if jobs_had_a_writer {
9451 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9452 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9453 match acquire_cache_lease(
9454 state,
9455 cache_dir,
9456 &owner,
9457 resource_limit,
9458 node,
9459 node,
9460 &cancel,
9461 )
9462 .await
9463 {
9464 Ok(Some(g)) => Some(g),
9465 Ok(None) => {
9468 if let Some(block) = &mut state.resource_block {
9469 block.cause = crate::run::ResourceBlockCause::Cancelled {
9470 park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
9471 };
9472 }
9473 return block_wave(
9474 jobs,
9475 state,
9476 node,
9477 attempt,
9478 "parked while waiting for the shared build cache",
9479 );
9480 }
9481 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9484 Err(e) => {
9485 let why = format!("{e:#}");
9489 state.event(
9490 "resource",
9491 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9492 );
9493 return block_wave(jobs, state, node, attempt, &why);
9494 }
9495 }
9496 } else {
9497 None
9498 }
9499 } else {
9500 None
9501 };
9502 state.seats_spawning(node, attempt);
9505 if let Err(e) = state.save() {
9506 tracing::warn!("could not persist the start of a wave: {e:#}");
9507 }
9508 let slot_limit = resource_limit
9512 + jobs
9513 .iter()
9514 .map(|j| j.timeout)
9515 .max()
9516 .unwrap_or_default()
9517 .saturating_mul(jobs.len() as u32);
9518 let mut set = tokio::task::JoinSet::new();
9519 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9522 let overlay = prompts.overlay(node);
9523 let repository_wait = repository_wait(state, pauses);
9524 for (i, mut job) in jobs.into_iter().enumerate() {
9525 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9526 if cache.is_some() {
9527 job.prompt.push('\n');
9528 job.prompt
9529 .push_str(&prompt::build_cache_note(node, job.allow_write));
9530 }
9531 let sem = Arc::clone(&sem);
9532 let run = run.to_owned();
9533 let node = node.to_owned();
9534 let attachments = if node == "implement" {
9537 state.attachments.clone()
9538 } else {
9539 Vec::new()
9540 };
9541 let cache = cache
9552 .filter(|_| job.allow_write && cache_guard.is_some())
9553 .map(Path::to_path_buf);
9554 let spawned_tx = spawned_tx.clone();
9555 let repository_wait = repository_wait.clone();
9556 let coordination = state.config.coordination.clone();
9557 let fetched_base = state.fetched_base.clone();
9558 set.spawn(async move {
9559 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9563 Ok(p) => p,
9564 Err(_) => {
9565 return (
9566 i,
9567 job.seat,
9568 AgentOutcome::ResourceBlocked(format!(
9569 "no parallel slot became free within {}s",
9570 slot_limit.as_secs()
9571 )),
9572 false,
9573 );
9574 }
9575 };
9576 spawned_tx.send(job.seat.key.clone()).ok();
9577 let mut seat = job.seat;
9578 let out = repository_wait
9579 .scope(agent::invoke(
9580 &job.spec,
9581 &mut seat,
9582 &Invocation {
9583 scope: crate::agent::InvocationScope::Repository,
9584 coordination: coordination.as_ref(),
9585 fetched_base: fetched_base.as_ref(),
9586 cwd: &job.cwd,
9587 prompt: &job.prompt,
9588 timeout: job.timeout,
9589 allow_write: job.allow_write,
9590 unsandboxed: false,
9591 sessions: job.sessions,
9592 artifacts: &job.artifacts,
9593 stem: &job.stem,
9594 run: &run,
9595 node: &node,
9596 cache_dir: cache.as_deref(),
9597 attachments: &attachments,
9598 writable: &[],
9599 },
9600 ))
9601 .await;
9602 let interrupted = out
9603 .as_ref()
9604 .err()
9605 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9606 let out = match out {
9607 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9608 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9609 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9617 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9618 Ok(o) => AgentOutcome::Failed(format!(
9619 "process exited with no usable output ({})",
9620 crate::notices::exit_status(o.exit_code)
9621 )),
9622 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9623 AgentOutcome::ResourceBlocked(e.to_string())
9624 }
9625 Err(e) => AgentOutcome::Failed(e.to_string()),
9626 };
9627 (i, seat, out, interrupted)
9628 });
9629 }
9630 drop(spawned_tx);
9631 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9632 loop {
9633 let joined = tokio::select! {
9634 Some(key) = spawned_rx.recv() => {
9635 state.seat_spawned(&key);
9637 if let Err(e) = state.save() {
9638 tracing::warn!("could not persist a seat's start: {e:#}");
9639 }
9640 continue;
9641 }
9642 joined = set.join_next() => joined,
9643 };
9644 let Some(joined) = joined else {
9645 break;
9646 };
9647 let (i, seat, out, interrupted) = match joined {
9648 Ok(v) => v,
9649 Err(e) => {
9653 tracing::error!("agent task panicked: {e}");
9654 continue;
9655 }
9656 };
9657 state.seat_finished(&seat.key);
9658 if interrupted
9659 && node == "implement"
9660 && let Some(label) = state
9661 .candidates
9662 .iter()
9663 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9664 .map(|candidate| candidate.label)
9665 {
9666 state.candidate_preservation.insert(
9667 label,
9668 "interrupted implementation awaiting persistence".into(),
9669 );
9670 state.interrupted_implementations.insert(label);
9671 }
9672 if let AgentOutcome::ResourceBlocked(why) = &out
9673 && state.resource_block.is_none()
9674 {
9675 state.resource_block = Some(crate::run::ResourceBlock {
9676 cause: crate::run::ResourceBlockCause::Other,
9677 node: node.to_owned(),
9678 kind: if why.contains("disk pressure") {
9679 "disk pressure"
9680 } else if why.contains("repository coordination") {
9681 "repository coordination"
9682 } else {
9683 "parallel slot"
9684 }
9685 .to_owned(),
9686 detail: why.clone(),
9687 at: Timestamp::now(),
9688 });
9689 let activity = if interrupted {
9690 "agent execution interrupted"
9691 } else {
9692 "no agent was started"
9693 };
9694 state.event("resource", format!("{node}: {activity} - {why}"));
9695 }
9696 record_jobs(state, node, round, &seat.key, &out);
9697 if let Err(e) = state.save() {
9698 tracing::warn!("could not persist a seat's completion: {e:#}");
9699 }
9700 if collected.len() <= i {
9701 collected.resize_with(i + 1, || None);
9702 }
9703 collected[i] = Some((i, seat, out));
9704 }
9705 if state
9711 .active
9712 .values()
9713 .any(|a| a.node == node && a.attempt == attempt)
9714 {
9715 state
9716 .active
9717 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9718 if let Err(e) = state.save() {
9719 tracing::warn!("could not persist the end of a wave: {e:#}");
9720 }
9721 }
9722 if let Some(cache_dir) = cache
9729 && jobs_had_a_writer
9730 {
9731 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9732 }
9733 if let Some(guard) = cache_guard {
9734 guard.release();
9735 }
9736 collected.into_iter().flatten().collect()
9737}
9738
9739fn record_jobs(
9750 state: &mut RunState,
9751 node: &str,
9752 round: Option<usize>,
9753 seat: &str,
9754 out: &AgentOutcome,
9755) {
9756 let commands: &[agent::CommandEvidence] = match out {
9757 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9758 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9759 };
9760 let checked_at = Timestamp::now();
9761 for c in commands {
9762 state.jobs.push(JobRecord {
9763 node: node.to_owned(),
9764 round,
9765 seat: seat.to_owned(),
9766 id: c.id.clone(),
9767 description: c.description.clone(),
9768 checked_at,
9769 status: match c.exit_code {
9770 Some(0) => JobStatus::Completed,
9771 Some(_) => JobStatus::Failed,
9772 None => JobStatus::Unknown,
9773 },
9774 exit_code: c.exit_code,
9775 result_summary: c.result_summary.clone(),
9776 source: c.source.clone(),
9777 });
9778 }
9779}
9780
9781fn round_is_clean(
9802 blocking: usize,
9803 e2e_ok: bool,
9804 answered: usize,
9805 expected: usize,
9806 quota_missing: usize,
9807 policy: IncompleteReviewPolicy,
9808) -> bool {
9809 if blocking != 0 || !e2e_ok {
9810 return false;
9811 }
9812 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9813 return true;
9814 }
9815 answered > 0 && expected - answered <= quota_missing
9816}
9817
9818pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9842 if max_rounds == 0
9843 || reviews.last().is_some_and(|r| {
9844 r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
9845 })
9846 {
9847 return Some(RunStatus::Gating);
9848 }
9849 let last = reviews.last()?;
9850 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9851 if reviews.len() < max_rounds && !stagnant {
9852 return None;
9853 }
9854 if last.incomplete() && last.blocking == 0 {
9855 return Some(RunStatus::Blocked);
9856 }
9857 if last.e2e_status() == E2eStatus::ResourceBlocked
9858 || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
9859 {
9860 return None;
9861 }
9862 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9863 RunStatus::Gating
9864 } else {
9865 RunStatus::Blocked
9866 })
9867}
9868
9869pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9884 if nudged {
9885 (full / 4).max(Duration::from_secs(120)).min(full)
9886 } else {
9887 full
9888 }
9889}
9890
9891#[allow(clippy::too_many_arguments)]
9920async fn ask_json_wave<T>(
9921 jobs: Vec<SeatJob>,
9922 sem: Arc<Semaphore>,
9923 retries: usize,
9924 roster: &[AgentSpec],
9925 ctx: &WaveCtx<'_>,
9926 losses: &mut Vec<QuotaLoss>,
9927 state: &mut RunState,
9928 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9929) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9930where
9931 T: serde::de::DeserializeOwned + Send + 'static,
9932{
9933 ask_wave_with(
9934 jobs,
9935 sem,
9936 retries,
9937 roster,
9938 ctx,
9939 losses,
9940 state,
9941 &|text: &str| {
9942 let v = verdict::extract_json::<T>(text)?;
9943 validate(&v)?;
9944 Ok(v)
9945 },
9946 )
9947 .await
9948}
9949
9950#[allow(clippy::too_many_arguments)]
9959async fn ask_wave_with<T>(
9960 jobs: Vec<SeatJob>,
9961 sem: Arc<Semaphore>,
9962 retries: usize,
9963 roster: &[AgentSpec],
9964 ctx: &WaveCtx<'_>,
9965 losses: &mut Vec<QuotaLoss>,
9966 state: &mut RunState,
9967 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9968) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9969where
9970 T: Send + 'static,
9971{
9972 let n = jobs.len();
9973 let originals: Vec<SeatJob> = jobs;
9974 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9975 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9976 let mut nudges: Vec<usize> = vec![0; n];
9983 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9987 let mut tried: Vec<BTreeSet<String>> = specs
9988 .iter()
9989 .map(|s| BTreeSet::from([s.id.clone()]))
9990 .collect();
9991 let starts: Vec<usize> = specs
9992 .iter()
9993 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9994 .collect();
9995 let carry = ctx.carry_seats && !roster.is_empty();
9996 let carried: Vec<BTreeSet<String>> = originals
9999 .iter()
10000 .map(|j| {
10001 state
10002 .seat_history
10003 .get(&j.seat.key)
10004 .filter(|_| carry)
10005 .map(|h| h.failed.clone())
10006 .unwrap_or_default()
10007 })
10008 .collect();
10009 let mut prev: Vec<Option<FailClass>> = originals
10010 .iter()
10011 .map(|j| {
10012 state
10013 .seat_history
10014 .get(&j.seat.key)
10015 .filter(|_| carry)
10016 .and_then(|h| h.last_fail.clone())
10017 })
10018 .collect();
10019 let next_agent =
10020 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
10021 let others: BTreeSet<String> = specs
10022 .iter()
10023 .enumerate()
10024 .filter(|(j, _)| *j != i)
10025 .map(|(_, s)| s.id.clone())
10026 .collect();
10027 pick_successor(
10028 roster,
10029 starts[i],
10030 tried,
10031 carry.then(|| &carried[i]),
10032 &others,
10033 )
10034 .cloned()
10035 };
10036 let mut fresh: Vec<Option<String>> = vec![None; n];
10037 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
10038 let mut pending: Vec<usize> = (0..n).collect();
10039
10040 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
10045 for round in 0..max_rounds {
10046 if pending.is_empty() {
10047 break;
10048 }
10049 let mut batch = Vec::with_capacity(pending.len());
10050 let mut renudged: Vec<&str> = Vec::new();
10051 for &i in &pending {
10052 let src = &originals[i];
10053 let full: &str = match &src.handover {
10058 Some(h) if specs[i].id != src.spec.id => h,
10059 _ => &src.prompt,
10060 };
10061 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
10064 (
10065 full.to_owned(),
10066 src.timeout,
10067 format!("{}-{word}-{}", src.stem, specs[i].id),
10068 )
10069 } else if nudges[i] == 0 {
10070 (full.to_owned(), src.timeout, src.stem.clone())
10071 } else {
10072 renudged.push(src.seat.key.as_str());
10073 let why = done[i]
10074 .as_ref()
10075 .and_then(|r| r.as_ref().err().map(ToString::to_string))
10076 .unwrap_or_else(|| "no parsable answer".to_owned());
10077 let nudge = prompt::nudge(&why);
10078 let nudged = has_context(&specs[i], &seats[i], src.sessions);
10079 let prompt = if nudged {
10080 nudge
10081 } else {
10082 format!("{full}\n\n---\n\n{nudge}")
10083 };
10084 (
10085 prompt,
10086 retry_budget(src.timeout, nudged),
10087 format!("{}-retry{}", src.stem, nudges[i]),
10088 )
10089 };
10090 batch.push(SeatJob {
10091 spec: specs[i].clone(),
10092 seat: seats[i].clone(),
10093 cwd: src.cwd.clone(),
10094 prompt,
10095 timeout,
10096 allow_write: src.allow_write,
10097 sessions: src.sessions,
10098 artifacts: src.artifacts.clone(),
10099 stem,
10100 handover: None,
10101 });
10102 }
10103
10104 if !renudged.is_empty() {
10105 state.event(
10106 ctx.node,
10107 format!("retry {round}: re-asking {}", renudged.join(", ")),
10108 );
10109 }
10110 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
10111 let mut still = Vec::new();
10112 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
10113 seats[i] = seat;
10114 if let AgentOutcome::ResourceBlocked(why) = &out {
10117 done[i] = Some(Err(anyhow::anyhow!("{why}")));
10118 continue;
10119 }
10120 let class = FailClass::of(&out);
10121 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
10124 && nudges[i] < retries
10125 && !roster.is_empty();
10126 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
10127 let next = should_hand_over(prev[i].as_ref(), &cur)
10128 .then(|| next_agent(i, &tried[i], &specs))
10129 .flatten();
10130 if carry {
10131 let h = state
10132 .seat_history
10133 .entry(originals[i].seat.key.clone())
10134 .or_default();
10135 h.failed.insert(specs[i].id.clone());
10136 h.last_fail = Some(cur.clone());
10137 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10138 h.last_ok = None;
10139 }
10140 if let Err(e) = state.save() {
10143 tracing::warn!("could not persist a seat's failure history: {e:#}");
10144 }
10145 }
10146 if let Some(next) = next {
10147 record_handover(
10148 state,
10149 ctx.node,
10150 &originals[i].seat.key,
10151 &specs[i].id,
10152 &next.id,
10153 &cur,
10154 &fail_reason(&out),
10155 );
10156 tried[i].insert(next.id.clone());
10157 prev[i] = Some(cur.clone());
10158 seats[i] =
10159 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10160 specs[i] = next;
10161 fresh[i] = Some(cur.stem_word().to_owned());
10162 nudges[i] = 0;
10163 done[i] = Some(Err(anyhow::anyhow!(
10164 "handed over after: {}",
10165 fail_reason(&out)
10166 )));
10167 still.push(i);
10168 continue;
10169 }
10170 }
10171 if carry
10172 && !nudge_first
10173 && let Some(cur) = class.clone()
10174 {
10175 let h = state
10177 .seat_history
10178 .entry(originals[i].seat.key.clone())
10179 .or_default();
10180 h.failed.insert(specs[i].id.clone());
10181 h.last_fail = Some(cur);
10182 }
10183 let parsed = match out {
10184 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10185 AgentOutcome::Quota(o) => {
10186 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10187 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10188 }
10189 AgentOutcome::Dropped(o) => {
10193 let why = o
10194 .dropped
10195 .as_ref()
10196 .map(|d| d.why.as_str())
10197 .unwrap_or("the CLI ended the stream without delivering its answer");
10198 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10199 }
10200 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10201 Err(anyhow::anyhow!(e))
10202 }
10203 };
10204 let quota = class == Some(FailClass::Quota);
10205 let failed = parsed.is_err();
10206 done[i] = Some(parsed);
10207 if carry && !failed {
10208 let h = state
10209 .seat_history
10210 .entry(originals[i].seat.key.clone())
10211 .or_default();
10212 h.failed.remove(&specs[i].id);
10213 h.last_ok = Some(specs[i].id.clone());
10214 h.last_fail = None;
10215 }
10216 let agent_failure = class.is_some()
10224 && !nudge_first
10225 && !roster.is_empty()
10226 && next_agent(i, &tried[i], &specs).is_some();
10227 if failed && !quota && !agent_failure && nudges[i] < retries {
10228 nudges[i] += 1;
10229 still.push(i);
10230 }
10231 }
10232 pending = still;
10233 }
10234
10235 for (i, q) in last_quota.into_iter().enumerate() {
10238 if let Some(reset) = q {
10239 losses.push(QuotaLoss {
10240 seat: originals[i].seat.key.clone(),
10241 node: ctx.node.to_owned(),
10242 at: Timestamp::now(),
10243 reset,
10244 });
10245 }
10246 }
10247
10248 seats
10249 .into_iter()
10250 .zip(done)
10251 .zip(nudges)
10252 .map(|((seat, res), attempts)| {
10253 (
10254 seat,
10255 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10256 attempts,
10257 )
10258 })
10259 .collect()
10260}
10261
10262fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10264 results
10265 .iter()
10266 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10267}
10268
10269fn block_wave(
10273 jobs: Vec<SeatJob>,
10274 state: &mut RunState,
10275 node: &str,
10276 attempt: usize,
10277 why: &str,
10278) -> Vec<(usize, SeatState, AgentOutcome)> {
10279 state.resource_wait = None;
10280 state
10281 .active
10282 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10283 if let Err(e) = state.save() {
10284 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10285 }
10286 jobs.into_iter()
10287 .enumerate()
10288 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10289 .collect()
10290}
10291
10292async fn acquire_cache_lease(
10305 state: &mut RunState,
10306 cache_dir: &Path,
10307 owner: &crate::cache::Owner,
10308 budget: Duration,
10309 node: &str,
10310 context: &str,
10311 cancel: &(dyn Fn() -> bool + Sync),
10312) -> Result<Option<crate::cache::Guard>> {
10313 let home = crate::run::home();
10314 let started = Instant::now();
10315 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10316 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10317 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10318 Err(e) => {
10319 state.event(
10320 "verify",
10321 format!("{context}: could not check the shared build cache: {e:#}"),
10322 );
10323 if let Err(e2) = state.save() {
10324 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10325 }
10326 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10327 }
10328 };
10329 state.event(
10330 "verify",
10331 format!(
10332 "{context}: waiting for the shared build cache at {} ({})",
10333 cache_dir.display(),
10334 busy.describe()
10335 ),
10336 );
10337 state.resource_wait = Some(crate::run::ResourceWait {
10340 node: context.to_owned(),
10341 kind: "shared build cache".to_owned(),
10342 holder: busy.describe(),
10343 since: Timestamp::now(),
10344 limit_secs: budget.as_secs(),
10345 });
10346 if let Err(e) = state.save() {
10347 tracing::warn!("could not persist a cache wait: {e:#}");
10348 }
10349 let remaining = budget.saturating_sub(started.elapsed());
10350 let waited = crate::cache::wait_for_unless(
10351 &home,
10352 cache_dir,
10353 owner,
10354 remaining,
10355 Duration::from_secs(5),
10356 cancel,
10357 )
10358 .await;
10359 state.resource_wait = None;
10360 let cause = match &waited {
10361 Ok(Some(_)) => None,
10362 Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
10363 Err(error) if error.is::<crate::cache::WaitTimeout>() => {
10364 Some(crate::run::ResourceBlockCause::Timeout {
10365 waited_secs: started.elapsed().as_secs(),
10366 limit_secs: budget.as_secs(),
10367 })
10368 }
10369 Err(_) => Some(crate::run::ResourceBlockCause::Other),
10370 };
10371 if let Some(cause) = cause {
10372 let detail = match &waited {
10373 Ok(_) => "the wait for the build cache was cancelled".to_owned(),
10374 Err(error) => format!("{error:#}"),
10375 };
10376 state.resource_block = Some(crate::run::ResourceBlock {
10377 cause,
10378 node: node.to_owned(),
10379 kind: "shared build cache".into(),
10380 detail,
10381 at: Timestamp::now(),
10382 });
10383 state.event(
10384 "resource",
10385 state.resource_block.as_ref().unwrap().description(),
10386 );
10387 }
10388 if let Err(error) = state.save() {
10389 tracing::warn!("could not persist cache acquisition outcome: {error:#}");
10390 }
10391 waited
10392}
10393
10394#[derive(Debug)]
10399struct CacheCheckError(String);
10400
10401impl std::fmt::Display for CacheCheckError {
10402 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10403 f.write_str(&self.0)
10404 }
10405}
10406
10407impl std::error::Error for CacheCheckError {}
10408
10409fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
10411 state
10412 .resource_block
10413 .get_or_insert_with(|| crate::run::ResourceBlock {
10414 cause: crate::run::ResourceBlockCause::Other,
10415 node: node.to_owned(),
10416 kind: "shared build cache".into(),
10417 detail: format!("{error:#}"),
10418 at: Timestamp::now(),
10419 });
10420 if let Err(error) = state.save() {
10421 tracing::warn!("could not persist cache resource block: {error:#}");
10422 }
10423}
10424
10425#[allow(clippy::too_many_arguments)]
10446async fn with_cache_lease<'s, F, Fut>(
10447 state: &'s mut RunState,
10448 cache_dir: Option<&Path>,
10449 node: &str,
10450 seat: &str,
10451 worktree: &Path,
10452 head: &str,
10453 budget: Duration,
10454 context: &str,
10455 wait: &crate::repo_lock::WaitPolicy,
10456 body: F,
10457) -> Result<(Vec<CommandOutcome>, bool)>
10458where
10459 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10460 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10461{
10462 let started = Instant::now();
10465 let home = crate::run::home();
10466 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10467 let guard = if let Some(cache_dir) = cache_dir {
10468 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10469 match acquire_cache_lease(
10470 state,
10471 cache_dir,
10472 &owner,
10473 resource_limit,
10474 node,
10475 context,
10476 &|| wait.cancelled(),
10477 )
10478 .await
10479 .and_then(|g| {
10480 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10481 }) {
10482 Ok(g) => Some(g),
10483 Err(e) => {
10484 record_cache_error(state, node, &e);
10485 return Ok((
10486 vec![CommandOutcome {
10487 command: "(waiting for the shared build cache)".into(),
10488 code: None,
10489 output_tail: e.to_string(),
10490 duration_ms: started.elapsed().as_millis() as u64,
10491 resource_blocked: true,
10492 }],
10493 false,
10494 ));
10495 }
10496 }
10497 } else {
10498 None
10499 };
10500 let _coordination = coordinate_operation(state, worktree, wait).await?;
10501 if let Some(cache_dir) = cache_dir {
10502 let identity = crate::cache::Identity::new(worktree, head);
10503 if let Err(e) = crate::cache::ensure_fresh(
10504 &home,
10505 cache_dir,
10506 &identity,
10507 &state.config.verify.cargo_manifests(),
10508 ) {
10509 record_cache_error(state, node, &e);
10510 state.event(
10511 "verify",
10512 format!(
10513 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10514 worktree.display(),
10515 short(head)
10516 ),
10517 );
10518 if let Some(guard) = guard {
10519 guard.release();
10520 }
10521 return Ok((
10522 vec![CommandOutcome {
10523 command: "(confirming the shared build cache is fresh)".into(),
10524 code: None,
10525 output_tail: e.to_string(),
10526 duration_ms: started.elapsed().as_millis() as u64,
10527 resource_blocked: true,
10528 }],
10529 false,
10530 ));
10531 }
10532 }
10533 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10534 if !timed_out_pids.is_empty() {
10535 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10536 }
10537 drop(_coordination);
10540 if let Some(guard) = guard {
10541 guard.release();
10542 }
10543 Ok((outcomes, retried))
10544}
10545
10546async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10558 wait_for_pids_with(
10559 pids,
10560 crate::proc::pid_alive,
10561 LEASE_RELEASE_POLL,
10562 LEASE_RELEASE_MAX_WAIT,
10563 )
10564 .await;
10565}
10566
10567async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10573 pids: &[u32],
10574 alive: F,
10575 poll: Duration,
10576 max_wait: Duration,
10577) {
10578 let deadline = Instant::now() + max_wait;
10579 loop {
10580 if pids.iter().all(|&pid| !alive(pid)) {
10581 return;
10582 }
10583 if Instant::now() >= deadline {
10584 return;
10585 }
10586 tokio::time::sleep(poll).await;
10587 }
10588}
10589
10590fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10597 outcomes.iter().any(|o| o.resource_blocked)
10598}
10599
10600pub(crate) enum GateFix {
10602 Retry,
10604 Stop,
10607 Defer,
10610}
10611
10612fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10620 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10621 red.peek().is_some()
10622 && red.all(|o| {
10623 !o.resource_blocked
10624 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10625 && !o.output_tail.trim().is_empty()
10626 })
10627}
10628
10629fn e2e_outcome_label(o: &CommandOutcome) -> String {
10633 if o.ok() {
10634 return "pass".to_owned();
10635 }
10636 let reason = if o.build_failed() {
10637 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10638 } else {
10639 format!("FAIL ({:?})", o.code)
10640 };
10641 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10642}
10643
10644async fn run_e2e_with_retry(
10652 state: &mut RunState,
10653 shell: &[String],
10654 commands: &[String],
10655 worktree: &Path,
10656 timeout: Duration,
10657 context: &str,
10658) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10659 let (mut e2e, mut timed_out_pids) = run_commands(
10660 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10661 )
10662 .await;
10663 for o in &e2e {
10664 state.event(
10665 "verify",
10666 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10667 );
10668 }
10669 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10673 if verify_retried {
10674 state.event(
10675 "verify",
10676 format!(
10677 "{context}: verify could not build/link, not a test result — retrying once \
10678 before concluding"
10679 ),
10680 );
10681 let retried = run_commands(
10682 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10683 )
10684 .await;
10685 e2e = retried.0;
10686 timed_out_pids.extend(retried.1);
10689 for o in &e2e {
10690 state.event(
10691 "verify",
10692 format!(
10693 "{context}: retry `{}` -> {}",
10694 o.command,
10695 e2e_outcome_label(o)
10696 ),
10697 );
10698 }
10699 }
10700 (e2e, verify_retried, timed_out_pids)
10701}
10702
10703#[allow(clippy::too_many_arguments)]
10719async fn run_commands(
10720 state: &mut RunState,
10721 node: &str,
10722 task: &str,
10723 attempt: usize,
10724 shell: &[String],
10725 commands: &[String],
10726 cwd: &Path,
10727 timeout: Duration,
10728) -> (Vec<CommandOutcome>, Vec<u32>) {
10729 if commands.is_empty() {
10730 return (Vec::new(), Vec::new());
10735 }
10736 let mut out = Vec::new();
10737 let mut timed_out_pids = Vec::new();
10738 let total = commands.len();
10739 for (idx, command) in commands.iter().enumerate() {
10740 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10741 if let Err(e) = state.save() {
10742 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10743 }
10744 let started = Instant::now();
10745 let mut cmd = tokio::process::Command::new(&shell[0]);
10746 cmd.quiet();
10747 for key in crate::worker::INVOCATION_ENV {
10751 cmd.env_remove(key);
10752 }
10753 cmd.args(&shell[1..])
10754 .arg(command)
10755 .current_dir(cwd)
10756 .stdin(std::process::Stdio::null())
10757 .stdout(std::process::Stdio::piped())
10758 .stderr(std::process::Stdio::piped())
10759 .kill_on_drop(true);
10760 let mut outputs = Vec::new();
10761 let protected = (|| -> Result<()> {
10762 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10763 outputs.push(guard);
10764 }
10765 if let Some(cache) = state.config.cache_dir() {
10766 outputs.push(crate::retention::OutputLease::cache(
10767 &crate::run::home(),
10768 &cache,
10769 false,
10770 )?);
10771 }
10772 Ok(())
10773 })();
10774 if let Err(e) = protected {
10775 out.push(CommandOutcome {
10776 command: command.clone(),
10777 code: None,
10778 output_tail: format!("output ownership unavailable: {e:#}"),
10779 duration_ms: 0,
10780 resource_blocked: true,
10781 });
10782 continue;
10783 }
10784 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10785 let (code, body) = match spawned {
10786 Ok(mut running) => {
10787 let pid = running.child().id();
10788 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10789 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10790 let mut paths = vec![cwd.to_owned()];
10791 if let Some(cache) = state.config.cache_dir() {
10792 paths.push(cache);
10793 }
10794 let mut pressure = None;
10795 let status = tokio::select! {
10796 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10797 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10798 pressure = Some(reason); None
10799 }
10800 };
10801 let (code, failure) = match status {
10802 Some(Ok(Ok(status))) => (
10803 status.code(),
10804 status
10805 .code()
10806 .is_none()
10807 .then(|| format!("command terminated: {status}")),
10808 ),
10809 Some(Ok(Err(error))) => {
10810 (None, Some(format!("failed waiting for command: {error}")))
10811 }
10812 Some(Err(_)) => {
10813 if !running.coordinated()
10817 && let Some(pid) = pid
10818 {
10819 timed_out_pids.push(pid);
10820 }
10821 (
10822 None,
10823 Some(format!("timed out after {}s", timeout.as_secs())),
10824 )
10825 }
10826 None => (None, None),
10827 };
10828 running.finish().await;
10829 if let Some(reason) = pressure {
10830 state.resource_block = Some(crate::run::ResourceBlock {
10831 cause: crate::run::ResourceBlockCause::Other,
10832 node: node.to_owned(),
10833 kind: "disk pressure".to_owned(),
10834 detail: reason.clone(),
10835 at: Timestamp::now(),
10836 });
10837 out.push(CommandOutcome {
10838 command: command.clone(),
10839 code: None,
10840 output_tail: reason,
10841 duration_ms: started.elapsed().as_millis() as u64,
10842 resource_blocked: true,
10843 });
10844 break;
10845 }
10846 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10847 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10848 if let Some(failure) = failure {
10849 body.push('\n');
10850 body.push_str(&failure);
10851 }
10852 (code, body)
10853 }
10854 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10855 };
10856 out.push(CommandOutcome {
10857 command: command.clone(),
10858 code,
10859 output_tail: tail(&body, OUTPUT_TAIL),
10860 duration_ms: started.elapsed().as_millis() as u64,
10861 resource_blocked: false,
10862 });
10863 }
10864 state.task_finished(task);
10865 if let Err(e) = state.save() {
10866 tracing::warn!("could not persist the end of {task}: {e:#}");
10867 }
10868 (out, timed_out_pids)
10869}
10870
10871fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10883 let repo = repo.display();
10884 match style {
10885 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10886 MergeStyle::Squash => {
10887 let subject = message
10890 .lines()
10891 .next()
10892 .unwrap_or(branch)
10893 .replace(['\\', '"', '$', '`'], "");
10894 format!(
10895 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10896 )
10897 }
10898 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10899 }
10900}
10901
10902const PR_TITLE_MAX: usize = 240;
10913
10914fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10925 let line = queue::first_line(text)?;
10926 let chars: Vec<char> = line.chars().collect();
10927 let (kept, mark) = if chars.len() <= max {
10928 (line.to_owned(), "")
10929 } else if let Some(end) = sentence_end(&chars, max) {
10930 (chars[..end].iter().collect::<String>(), "")
10931 } else {
10932 let room = max.saturating_sub(3);
10933 let cut = if chars[room].is_whitespace() {
10936 room
10937 } else {
10938 chars[..room]
10939 .iter()
10940 .rposition(|c| c.is_whitespace())
10941 .unwrap_or(room)
10942 };
10943 let head: String = chars[..cut].iter().collect();
10944 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10945 (head.to_owned(), "...")
10946 };
10947 if kept.is_empty() {
10948 return None;
10949 }
10950 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10951 return None;
10952 }
10953 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10954 return None;
10955 }
10956 Some(format!("{kept}{mark}"))
10957}
10958
10959fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10963 const MIN: usize = 20;
10964 for i in MIN..max.min(chars.len()) {
10965 if !matches!(chars[i], '.' | '!' | '?') {
10966 continue;
10967 }
10968 let Some(&next) = chars.get(i + 1) else {
10969 continue;
10970 };
10971 if !next.is_whitespace() {
10972 continue;
10973 }
10974 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10975 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10976 continue;
10977 }
10978 let word: String = chars[..i]
10979 .iter()
10980 .rev()
10981 .take_while(|c| !c.is_whitespace())
10982 .collect::<Vec<_>>()
10983 .into_iter()
10984 .rev()
10985 .collect();
10986 let word = word.to_ascii_lowercase();
10987 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10988 continue;
10989 }
10990 let end = chars[..i]
10991 .iter()
10992 .rposition(|c| !c.is_whitespace())
10993 .map_or(i, |p| p + 1);
10994 return Some(end);
10995 }
10996 None
10997}
10998
10999struct PrMessage {
11004 title: String,
11005 body: String,
11006}
11007
11008impl PrMessage {
11009 fn commit_message(&self) -> String {
11013 format!("{}\n\n{}", self.title, self.body)
11014 }
11015}
11016
11017fn title_marker(line: &str) -> Option<&str> {
11019 let line = line.trim();
11020 let head = line.get(..6)?;
11021 head.eq_ignore_ascii_case("title:")
11022 .then(|| line[6..].trim())
11023}
11024
11025fn summary_title(summary: &str) -> Option<String> {
11030 let first = summary.lines().find(|l| !l.trim().is_empty())?;
11031 let raw = title_marker(first)?;
11032 if raw.is_empty() {
11033 return None;
11034 }
11035 let title = queue::title_from(raw, PR_TITLE_MAX);
11036 let lower = title.to_ascii_lowercase();
11037 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11038 return None;
11039 }
11040 Some(title)
11041}
11042
11043const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
11045
11046pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
11048
11049pub(crate) fn is_review_run(state: &RunState) -> bool {
11053 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
11054}
11055
11056fn review_title(state: &RunState) -> Option<String> {
11061 english_subject(
11062 state.reviewed_commits.as_ref()?.first()?,
11063 state.config.graph.github_english(),
11064 )
11065}
11066
11067fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
11070 let raw = raw.trim();
11071 if raw.is_empty() {
11072 return None;
11073 }
11074 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
11075 return None;
11076 }
11077 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
11078 return None;
11079 }
11080 let title = queue::title_from(raw, PR_TITLE_MAX);
11081 let lower = title.to_ascii_lowercase();
11082 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
11083 return None;
11084 }
11085 Some(title)
11086}
11087
11088#[derive(Debug, Clone, PartialEq, Eq)]
11091struct BranchFacts {
11092 commits: Vec<(String, String)>,
11094 stat: String,
11096}
11097
11098const STAT_FILE_LINES: usize = 25;
11100const COMMITS_MAX_CHARS: usize = 20_000;
11102
11103async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
11106 let commits = git::commit_log(repo, base, branch).await.ok()?;
11107 if commits.is_empty() {
11108 return None;
11109 }
11110 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
11111 let lines: Vec<&str> = stat.lines().collect();
11112 let stat = if lines.len() > STAT_FILE_LINES + 1 {
11113 let omitted = lines.len() - 1 - STAT_FILE_LINES;
11114 let more = format!(" ... {omitted} more file(s)");
11115 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
11116 kept.push(&more);
11117 kept.push(lines[lines.len() - 1]);
11118 kept.join("\n")
11119 } else {
11120 lines.join("\n")
11121 };
11122 Some(BranchFacts { commits, stat })
11123}
11124
11125fn markdown_safe(text: &str) -> String {
11128 text.replace("</details>", "</details>")
11129 .replace("\x60\x60\x60", "~~~")
11130}
11131
11132fn neutral_title(state: &RunState, winner: char) -> String {
11133 format!(
11134 "chore: land candidate {} of run {}",
11135 winner.to_ascii_uppercase(),
11136 state.id
11137 )
11138}
11139
11140pub(crate) async fn repair_landing_gate(
11143 state: &mut RunState,
11144 winner: &Candidate,
11145 outcomes: &[CommandOutcome],
11146) -> Result<GateFix> {
11147 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
11148 return Ok(GateFix::Stop);
11149 }
11150 let mut runner = Runner {
11151 roles: state.config.resolve_roles()?,
11152 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
11153 state: state.clone(),
11154 pause: Pause::new(),
11155 interrupt: Pause::new(),
11156 };
11157 let result = runner.gate_fix_round(winner, outcomes).await;
11158 *state = runner.state;
11159 result
11160}
11161
11162pub(crate) async fn check_gate_repair_e2e(
11165 state: &mut RunState,
11166 winner: &Candidate,
11167 wait: &crate::repo_lock::WaitPolicy,
11168) -> Result<GateFix> {
11169 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
11170 return Ok(GateFix::Retry);
11171 }
11172 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11173 if state
11174 .gate_repair_e2e
11175 .as_ref()
11176 .is_some_and(|r| r.head == head && r.passed)
11177 {
11178 return Ok(GateFix::Retry);
11179 }
11180 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11181 head: head.clone(),
11182 passed: false,
11183 outcomes: Vec::new(),
11184 });
11185 state.save()?;
11186 let commands = state.config.verify.e2e.clone();
11187 let outcomes = if commands.is_empty() {
11188 Vec::new()
11189 } else {
11190 let shell = state.config.shell();
11191 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11192 let cache_dir = state.config.cache_dir();
11193 let (outcomes, _) = with_cache_lease(
11194 state,
11195 cache_dir.as_deref(),
11196 "e2e",
11197 "e2e",
11198 &winner.worktree,
11199 &head,
11200 timeout,
11201 "gate repair e2e",
11202 wait,
11203 |state, budget| async move {
11204 run_e2e_with_retry(
11205 state,
11206 &shell,
11207 &commands,
11208 &winner.worktree,
11209 budget,
11210 "gate repair e2e",
11211 )
11212 .await
11213 },
11214 )
11215 .await?;
11216 outcomes
11217 };
11218 let deferred = verify_inconclusive(&outcomes);
11219 let passed = outcomes.iter().all(CommandOutcome::ok)
11220 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
11221 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
11222 head,
11223 passed,
11224 outcomes,
11225 });
11226 state.save()?;
11227 if deferred {
11228 Ok(GateFix::Defer)
11229 } else if !passed {
11230 state.event(
11231 "gate",
11232 "verify.e2e failed after gate repair; nothing published",
11233 );
11234 Ok(GateFix::Stop)
11235 } else {
11236 Ok(GateFix::Retry)
11237 }
11238}
11239
11240pub(crate) async fn run_landing_gate(
11242 state: &mut RunState,
11243 winner: &Candidate,
11244 wait: &crate::repo_lock::WaitPolicy,
11245) -> Result<Vec<CommandOutcome>> {
11246 let shell = state.config.shell();
11247 let gate_commands = state.config.verify.gate.clone();
11248 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11249 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11250 Vec::new()
11251 } else {
11252 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11253 let cache_dir = state.config.cache_dir();
11254 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11255 let (outcomes, _) = with_cache_lease(
11256 state,
11257 cache_dir.as_deref(),
11258 "gate",
11259 "gate",
11260 &winner.worktree,
11261 &head,
11262 timeout,
11263 "final gate",
11264 wait,
11265 |state, budget| {
11266 let shell = shell.clone();
11267 let gate_commands = gate_commands.clone();
11268 let winner = winner.clone();
11269 async move {
11270 Runner::run_pre_gate(state, &winner).await;
11271 let (outcomes, timed_out_pids) = run_commands(
11272 state,
11273 "gate",
11274 "gate",
11275 0,
11276 &shell,
11277 &gate_commands,
11278 &winner.worktree,
11279 budget,
11280 )
11281 .await;
11282 (outcomes, false, timed_out_pids)
11283 }
11284 },
11285 )
11286 .await?;
11287 outcomes
11288 };
11289 if outcomes.is_empty() {
11290 state.event("gate", "no gate commands configured; not tested");
11295 }
11296 for o in &outcomes {
11297 state.event(
11298 "gate",
11299 format!(
11300 "`{}` -> {}",
11301 o.command,
11302 if o.ok() {
11303 "pass".to_owned()
11304 } else {
11305 format!(
11306 "FAIL ({:?})\n{}",
11307 o.code,
11308 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11309 )
11310 }
11311 ),
11312 );
11313 }
11314 Ok(outcomes)
11315}
11316
11317pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11323 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11324 ""
11325 } else {
11326 pr_title
11327 }
11328}
11329
11330pub fn landing_subject_source(state: &RunState) -> String {
11334 if is_review_run(state) {
11335 let winner = state.candidates.first().map_or('A', |c| c.label);
11336 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11337 }
11338 state.instruction.clone()
11339}
11340
11341fn summary_without_title(summary: &str) -> String {
11344 let mut lines = summary.trim().lines().peekable();
11345 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11346 lines.next();
11347 }
11348 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11349}
11350
11351#[cfg(test)]
11363fn pr_message(state: &RunState, winner: char) -> PrMessage {
11364 pr_message_with(state, winner, None)
11365}
11366
11367#[cfg(test)]
11370fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11371 let raw = pr_message_raw(
11372 state,
11373 winner,
11374 facts,
11375 crate::pr_outcome::FinalHead::Unreported,
11376 ChangeNote::NotAsked,
11377 );
11378 let id = crate::scrub::Identity::current();
11379 PrMessage {
11380 title: crate::scrub::scrub(&raw.title, &id),
11381 body: crate::scrub::scrub(&raw.body, &id),
11382 }
11383}
11384
11385#[derive(Debug, Clone, Copy)]
11387enum ChangeNote<'a> {
11388 NotAsked,
11390 Summary(&'a crate::run::ChangeSummary),
11392 Unavailable(&'a str),
11394}
11395
11396fn pr_message_raw(
11397 state: &RunState,
11398 winner: char,
11399 facts: Option<&BranchFacts>,
11400 head: crate::pr_outcome::FinalHead<'_>,
11401 change: ChangeNote<'_>,
11402) -> PrMessage {
11403 if !matches!(change, ChangeNote::NotAsked) {
11404 let (title, mut body) = match change {
11405 ChangeNote::Summary(c) => (
11406 c.title.clone(),
11407 format!("## Summary\n\n{}\n\n", c.body.trim()),
11408 ),
11409 ChangeNote::Unavailable(why) => (
11410 neutral_title(state, winner),
11411 format!(
11412 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11413 ),
11414 ),
11415 ChangeNote::NotAsked => unreachable!(),
11416 };
11417 body.push_str(&public_pr_evidence(state, head));
11418 body.push_str(&format!(
11419 "\n---\nmagi:run/{} magi:candidate-{}\n",
11420 state.id,
11421 winner.to_ascii_lowercase()
11422 ));
11423 return PrMessage { title, body };
11424 }
11425 pr_preview(state, winner, facts, head)
11426}
11427
11428fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11431 let mut body = crate::pr_outcome::review_rounds(state);
11432 body.push_str(&crate::pr_outcome::final_verification(state, head));
11433 let open = state.last_round_findings();
11434 if !open.is_empty() {
11435 body.push_str("## Open review findings\n\n");
11436 for f in open {
11437 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11438 }
11439 body.push('\n');
11440 }
11441 if let Some(c) = state.change_summary.as_ref()
11442 && head == crate::pr_outcome::FinalHead::At(&c.head)
11443 {
11444 body.push_str(&format!(
11445 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11446 c.base, c.head
11447 ));
11448 }
11449 body
11450}
11451
11452fn pr_preview(
11454 state: &RunState,
11455 winner: char,
11456 facts: Option<&BranchFacts>,
11457 head: crate::pr_outcome::FinalHead<'_>,
11458) -> PrMessage {
11459 let summary = state
11460 .candidates
11461 .iter()
11462 .find(|c| c.label == winner)
11463 .map(|c| c.summary.as_str())
11464 .unwrap_or_default();
11465 let review = is_review_run(state);
11468 let title = if review {
11469 facts
11470 .and_then(|f| {
11471 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11472 })
11473 .or_else(|| review_title(state))
11474 .or_else(|| {
11475 state
11476 .candidates
11477 .iter()
11478 .find(|c| c.label == winner)
11479 .filter(|c| !c.branch.starts_with("magi/"))
11480 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11481 })
11482 .unwrap_or_else(|| neutral_title(state, winner))
11483 } else {
11484 summary_title(summary).unwrap_or_else(|| {
11485 english_title(
11486 &state.instruction,
11487 PR_TITLE_MAX,
11488 state.config.graph.github_english(),
11489 )
11490 .unwrap_or_else(|| neutral_title(state, winner))
11491 })
11492 };
11493
11494 let mut body = String::new();
11495 let what = summary_without_title(summary);
11496 if !what.is_empty() {
11497 body.push_str("## Summary\n\n");
11498 body.push_str(&what);
11499 body.push_str("\n\n");
11500 }
11501
11502 body.push_str(&crate::pr_outcome::review_rounds(state));
11504 body.push_str(&crate::pr_outcome::final_verification(state, head));
11505 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11506
11507 let open = state.open_findings();
11508 if !open.is_empty() {
11509 body.push_str("## Open review findings\n\n");
11510 for f in &open {
11511 body.push_str(&format!(
11512 "- `{}` [{:?}] {}\n",
11513 f.id, f.severity, "Context retained in local review records."
11514 ));
11515 }
11516 body.push('\n');
11517 }
11518
11519 if let Some(fix) = fix
11520 && !fix.rejected.is_empty()
11521 {
11522 body.push_str("## Declined by the fixer\n\n");
11523 for r in &fix.rejected {
11524 body.push_str(&format!(
11525 "- `{}`: {}\n",
11526 r.id, "Reason retained in local review records."
11527 ));
11528 }
11529 body.push('\n');
11530 }
11531
11532 if review {
11533 body.push_str("## Commits under review\n\n");
11535 if let Some(facts) = facts {
11536 let mut left = COMMITS_MAX_CHARS;
11537 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11538 let mut entry = format!("- {}\n", markdown_safe(subject));
11539 for l in markdown_safe(text).lines() {
11540 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11541 }
11542 if left == 0 {
11543 body.push_str(&format!(
11544 "- ... {} more commit(s)\n",
11545 facts.commits.len() - i
11546 ));
11547 break;
11548 }
11549 if entry.len() > left {
11550 let mut end = left;
11553 while !entry.is_char_boundary(end) {
11554 end -= 1;
11555 }
11556 entry.truncate(end);
11557 entry.push_str("\n ... (truncated)\n");
11558 left = 0;
11559 } else {
11560 left -= entry.len();
11561 }
11562 body.push_str(&entry);
11563 }
11564 if !facts.stat.trim().is_empty() {
11565 body.push_str(&format!(
11566 "\n## Diff stat\n\n```\n{}\n```\n",
11567 markdown_safe(facts.stat.trim())
11568 ));
11569 }
11570 } else {
11571 match &state.reviewed_commits {
11572 Some(subjects) => {
11573 for s in subjects {
11574 body.push_str(&format!("- {}\n", s.trim()));
11575 }
11576 }
11577 None => {
11578 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11581 body.push_str(rest.trim());
11582 body.push('\n');
11583 }
11584 }
11585 }
11586 } else {
11587 let task = state.instruction.trim();
11588 let task = if task.is_empty() {
11589 "(empty task)"
11590 } else {
11591 task
11592 };
11593 body.push_str(&format!(
11594 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11595 task.replace("</details>", "</details>")
11596 ));
11597 }
11598
11599 body.push_str(&format!(
11600 "\n---\nmagi:run/{} magi:candidate-{}\n",
11601 state.id,
11602 winner.to_ascii_lowercase()
11603 ));
11604
11605 PrMessage { title, body }
11606}
11607
11608fn seeded_instruction(state: &RunState) -> String {
11611 let mut inputs = Vec::new();
11612 if let Some(start) = &state.sources.start_from {
11613 inputs.push(format!(
11614 "Start from `{}` at `{}` (including ancestry).",
11615 start.token, start.sha
11616 ));
11617 }
11618 for pick in &state.sources.pick_commits {
11619 inputs.push(format!(
11620 "Pick `{}` at `{}` (single commit).",
11621 pick.token, pick.sha
11622 ));
11623 }
11624 if inputs.is_empty() {
11625 state.instruction.clone()
11626 } else {
11627 format!(
11628 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11629 state.instruction,
11630 inputs.join("\n")
11631 )
11632 }
11633}
11634
11635async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11639 let base = &state.base_branch;
11640 let mut against = base.clone();
11641 if mode == MergeMode::Pr {
11642 let remote = &state.config.merge.remote;
11646 let tracking = format!("{remote}/{base}");
11647 let fetched = git::fetch(repo, remote, base).await;
11648 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11649 against = tracking;
11650 } else {
11651 return false;
11652 }
11653 }
11654 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11655}
11656
11657fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11660 let mut detail = format!(
11661 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11662 nothing to open a pull request for"
11663 );
11664 detail.push_str(
11665 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11666 );
11667 detail
11668}
11669
11670#[derive(Debug, PartialEq, Eq)]
11673enum PrPlan {
11674 Create,
11675 Adopt { url: String, title: String },
11676 Stop(String),
11677}
11678
11679fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11682 match found {
11683 Ok(land::OpenPr::None) => PrPlan::Create,
11684 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11685 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11686 "several open pull requests exist for this branch, not picking one: {}",
11687 urls.join(" ")
11688 )),
11689 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11690 }
11691}
11692
11693pub(crate) async fn gh_pr_create(
11695 cwd: &Path,
11696 base: &str,
11697 head: &str,
11698 title: &str,
11699 body: &str,
11700) -> Result<String> {
11701 let out = tokio::process::Command::new("gh")
11702 .args([
11703 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11704 ])
11705 .current_dir(cwd)
11706 .quiet()
11707 .stdin(std::process::Stdio::null())
11708 .output()
11709 .await
11710 .context("spawn gh")?;
11711 if out.status.success() {
11712 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11713 } else {
11714 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11715 }
11716}
11717
11718async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11720 match std::fs::symlink_metadata(path) {
11721 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11722 git::worktree_remove_clean(repo, path).await?;
11725 return Ok(());
11726 }
11727 Err(e) => return Err(e.into()),
11728 Ok(_) => {}
11729 }
11730 if !git::is_clean(path).await? {
11731 bail!(
11732 "worktree has uncommitted work; refusing removal: {}",
11733 path.display()
11734 );
11735 }
11736 if !git::worktree_remove_clean(repo, path).await? {
11737 bail!("worktree removal refused: {}", path.display());
11738 }
11739 Ok(())
11740}
11741
11742pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11751 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11752 bail!("refusing fold: {reason}");
11753 }
11754 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11755 bail!("candidate preservation incomplete; refusing cleanup");
11756 }
11757 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11761 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11762 let _driver = crate::driver::maintenance(home, &state.id)?;
11763 let mut output_guards = Vec::new();
11764 for entry in std::fs::read_dir(state.worktree_root())
11765 .into_iter()
11766 .flatten()
11767 .flatten()
11768 {
11769 if entry.path().join(".git").is_file() {
11770 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11771 }
11772 }
11773 let repo = state.repo.clone();
11774 let wait = repository_wait(state, None);
11777 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11778 Ok(lease) => lease,
11779 Err(e) => {
11780 state.save_under(home)?;
11781 return Err(e);
11782 }
11783 };
11784 let _source_outputs = if state.status == RunStatus::Merged
11787 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11788 {
11789 verify_merged_fold_sources(state, drop_winner, home).await?
11790 } else {
11791 Vec::new()
11792 };
11793 let root = state.worktree_root();
11794 let winner = state.tally.as_ref().map(|t| t.winner);
11795 let mut removed = Vec::new();
11796
11797 for i in 0..state.candidates.len() {
11798 let c = state.candidates[i].clone();
11799 let is_winner = Some(c.label) == winner;
11800 if is_winner && !drop_winner {
11801 continue;
11802 }
11803 let existed = c.worktree.exists();
11804 remove_preserved_worktree(&repo, &c.worktree).await?;
11807 if existed {
11808 removed.push(c.worktree.to_string_lossy().into_owned());
11809 }
11810 let handed_over = state.released_branches.contains(&c.branch);
11813 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11814 if !git::branch_delete(&repo, &c.branch).await? {
11815 bail!("candidate branch removal refused");
11816 }
11817 removed.push(c.branch.clone());
11818 }
11819 state.candidates[i].folded = true;
11820 }
11821
11822 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11823 let path = name.path();
11824 let keep = !drop_winner
11825 && winner.is_some_and(|w| {
11826 path.file_name()
11827 .is_some_and(|n| n == format!("cand-{w}").as_str())
11828 });
11829 if keep {
11830 continue;
11831 }
11832 remove_preserved_worktree(&repo, &path).await?;
11833 removed.push(path.to_string_lossy().into_owned());
11834 }
11835
11836 remove_if_empty(&root);
11845
11846 if state.enabled_worktree_config && drop_winner {
11847 git::release_worktree_config(&repo).await.ok();
11851 state.enabled_worktree_config = false;
11852 }
11853 state.save_under(home)?;
11854 Ok(removed)
11855}
11856
11857async fn verify_merged_fold_sources(
11858 state: &RunState,
11859 drop_winner: bool,
11860 home: &Path,
11861) -> Result<Vec<std::fs::File>> {
11862 let winner = state.tally.as_ref().map(|t| t.winner);
11863 let mut heads = BTreeSet::new();
11864 for candidate in &state.candidates {
11865 if Some(candidate.label) == winner && !drop_winner {
11866 continue;
11867 }
11868 anyhow::ensure!(
11869 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11870 "candidate recovery source is outside the inventoried run root: {}",
11871 candidate.worktree.display()
11872 );
11873 match std::fs::symlink_metadata(&candidate.worktree) {
11874 Ok(_) => {}
11875 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11876 Err(error) => return Err(error.into()),
11877 }
11878 if !state.released_branches.contains(&candidate.branch)
11879 && git::branch_exists(&state.repo, &candidate.branch).await?
11880 {
11881 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11882 }
11883 }
11884 let mut cargo_guards = Vec::new();
11885 let entries = match std::fs::read_dir(state.worktree_root()) {
11886 Ok(entries) => Some(entries),
11887 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11888 Err(error) => return Err(error.into()),
11889 };
11890 for entry in entries.into_iter().flatten() {
11891 let path = entry?.path();
11892 let metadata = std::fs::symlink_metadata(&path)?;
11893 anyhow::ensure!(
11894 metadata.is_dir() && !metadata.file_type().is_symlink(),
11895 "non-directory or symlink recovery source retained: {}",
11896 path.display()
11897 );
11898 if !drop_winner
11899 && winner.is_some_and(|w| {
11900 path.file_name()
11901 .is_some_and(|name| name == format!("cand-{w}").as_str())
11902 })
11903 {
11904 continue;
11905 }
11906 anyhow::ensure!(
11907 path.join(".git").is_file(),
11908 "unregistered recovery source retained: {}",
11909 path.display()
11910 );
11911 anyhow::ensure!(
11912 git::git(
11913 &path,
11914 &[
11915 "status",
11916 "--porcelain",
11917 "--untracked-files=all",
11918 "--ignored",
11919 "--",
11920 ".",
11921 ":(exclude)target"
11922 ]
11923 )
11924 .await?
11925 .is_empty(),
11926 "source changes or ignored recovery files retained: {}",
11927 path.display()
11928 );
11929 match std::fs::symlink_metadata(path.join("target")) {
11930 Ok(_) => {
11931 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11932 }
11933 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11934 Err(error) => return Err(error.into()),
11935 }
11936 for checkpoint in [
11937 "rebase-merge",
11938 "rebase-apply",
11939 "sequencer",
11940 "CHERRY_PICK_HEAD",
11941 "MERGE_HEAD",
11942 "REBASE_HEAD",
11943 "REVERT_HEAD",
11944 "BISECT_LOG",
11945 "AUTO_MERGE",
11946 "MERGE_MSG",
11947 ] {
11948 let location = git::git(
11949 &path,
11950 &[
11951 "rev-parse",
11952 "--path-format=absolute",
11953 "--git-path",
11954 checkpoint,
11955 ],
11956 )
11957 .await?;
11958 anyhow::ensure!(
11959 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11960 if e.kind() == std::io::ErrorKind::NotFound),
11961 "Git recovery checkpoint retained: {checkpoint}"
11962 );
11963 }
11964 heads.insert(git::rev_parse(&path, "HEAD").await?);
11965 }
11966 crate::fold_evidence::preserve(state, home, &heads).await?;
11967
11968 Ok(cargo_guards)
11969}
11970
11971fn remove_if_empty(dir: &Path) {
11982 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11983 std::fs::remove_dir(dir).ok();
11984 }
11985}
11986
11987pub fn worst_open(state: &RunState) -> Option<Severity> {
11989 state
11990 .reviews
11991 .last()?
11992 .reviews
11993 .iter()
11994 .flat_map(|r| r.findings.iter())
11995 .map(|f| f.severity)
11996 .max()
11997}
11998
11999async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
12003 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
12004 git::acquire_worktree_config(repo).await?;
12005 state.enabled_worktree_config = true;
12006 }
12007 Ok(())
12008}
12009
12010#[cfg(test)]
12011mod tests {
12012 #[tokio::test]
12013 async fn launcher_base_fetch_requires_remote_success() {
12014 let tmp = tempfile::tempdir().unwrap();
12015 let repo = tmp.path().join("repo");
12016 let remote = tmp.path().join("remote");
12017 std::fs::create_dir(&repo).unwrap();
12018 std::fs::create_dir(&remote).unwrap();
12019 git::git(&remote, &["init", "--bare", "-b", "main"])
12020 .await
12021 .unwrap();
12022 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
12023 git::git(
12024 &repo,
12025 &[
12026 "-c",
12027 "user.name=Fixture",
12028 "-c",
12029 "user.email=fixture@example.invalid",
12030 "commit",
12031 "--allow-empty",
12032 "-m",
12033 "fixture",
12034 ],
12035 )
12036 .await
12037 .unwrap();
12038 git::git(
12039 &repo,
12040 &["remote", "add", "origin", remote.to_str().unwrap()],
12041 )
12042 .await
12043 .unwrap();
12044 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
12045 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12046 assert_eq!(
12047 resolve_base(&repo, "main", "origin", None)
12048 .await
12049 .unwrap()
12050 .commit,
12051 head
12052 );
12053 #[cfg(unix)]
12054 {
12055 let policy: crate::config::Coordination = toml::from_str(
12056 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
12057 ).unwrap();
12058 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
12059 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
12060 let held = crate::repo_lock::SeatLease::operation_with(
12061 &repo,
12062 Some(&policy),
12063 &crate::repo_lock::WaitPolicy::default(),
12064 )
12065 .await
12066 .unwrap();
12067 assert!(held.is_some());
12068 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
12069 tokio::pin!(fetch);
12070 assert!(
12071 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
12072 .await
12073 .is_err()
12074 );
12075 assert!(
12076 !repo.join(".git/FETCH_HEAD").exists(),
12077 "no fetch before admission"
12078 );
12079 drop(held);
12080 let fetched = fetch.await.unwrap();
12081 assert_eq!(fetched.commit, head);
12082 assert_eq!(fetched.remote, "origin");
12083 assert_eq!(fetched.branch, "main");
12084 }
12085 git::git(
12087 &repo,
12088 &[
12089 "remote",
12090 "set-url",
12091 "origin",
12092 tmp.path().join("missing").to_str().unwrap(),
12093 ],
12094 )
12095 .await
12096 .unwrap();
12097 let error = resolve_base(&repo, "main", "origin", None)
12098 .await
12099 .unwrap_err()
12100 .to_string();
12101 assert!(error.contains("cannot read origin/main"));
12102 assert!(error.contains("refusing to branch off"));
12103 }
12104
12105 #[test]
12106 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
12107 let leaked = vec!["chore(deps): update a crate".to_owned()];
12108 let own = "fix(daemon): apply a chosen action";
12109 assert!(should_retitle("", own, &leaked));
12110 assert!(should_retitle(
12111 &format!("{REVIEW_PROMPT_OPENING} `x`"),
12112 own,
12113 &leaked
12114 ));
12115 assert!(should_retitle(
12116 "chore: land candidate A of run 1",
12117 own,
12118 &leaked
12119 ));
12120 assert!(should_retitle(
12121 "magi: candidate A (uncommitted work)",
12122 own,
12123 &leaked
12124 ));
12125 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
12126 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
12127 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
12128 }
12129
12130 #[test]
12131 fn pr_merge_plan_creates_adopts_or_stops() {
12132 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
12133 assert_eq!(
12134 pr_merge_plan(Ok(land::OpenPr::One {
12135 url: "u".into(),
12136 title: "t".into()
12137 })),
12138 PrPlan::Adopt {
12139 url: "u".into(),
12140 title: "t".into()
12141 }
12142 );
12143 let PrPlan::Stop(many) =
12144 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
12145 else {
12146 panic!("many must stop");
12147 };
12148 assert!(many.contains('a') && many.contains('b'));
12149 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
12150 panic!("a failed lookup must stop");
12151 };
12152 assert!(err.contains("bad token"));
12153 }
12154
12155 use super::*;
12156 use crate::run::GateStatus;
12157 use std::collections::BTreeMap;
12158 use std::time::Duration;
12159
12160 fn conductor() -> AgentSpec {
12161 AgentSpec {
12162 id: "conductor".to_owned(),
12163 kind: crate::config::AgentKind::Command,
12164 model: None,
12165 command: vec!["true".to_owned()],
12166 extra_args: Vec::new(),
12167 env: BTreeMap::new(),
12168 prompt_delivery: None,
12169 max_turn_secs: None,
12170 }
12171 }
12172
12173 fn spec(id: &str) -> AgentSpec {
12174 AgentSpec {
12175 id: id.to_owned(),
12176 kind: crate::config::AgentKind::Command,
12177 model: None,
12178 command: vec!["true".to_owned()],
12179 extra_args: Vec::new(),
12180 env: BTreeMap::new(),
12181 prompt_delivery: None,
12182 max_turn_secs: None,
12183 }
12184 }
12185
12186 fn ids(xs: &[&str]) -> BTreeSet<String> {
12187 xs.iter().map(|s| (*s).to_owned()).collect()
12188 }
12189
12190 #[test]
12191 fn pick_successor_skips_an_agent_another_seat_holds() {
12192 let roster = [spec("a"), spec("b"), spec("c")];
12194 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12195 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12196 }
12197
12198 #[test]
12199 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12200 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12202 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12203 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12204 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12205 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12206 }
12207
12208 #[test]
12209 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12210 let roster = [spec("a"), spec("b")];
12211 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12212 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12213 let carried = ids(&["b"]);
12214 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12215 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12216 }
12217
12218 #[test]
12219 fn pick_successor_returns_none_without_any_untried_successor() {
12220 let roster = [spec("a"), spec("b")];
12221 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12222 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12223 let carried = ids(&["a"]);
12224 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12225 }
12226
12227 #[test]
12228 fn pick_successor_rescue_avoids_another_seats_occupant() {
12229 let roster = [spec("a"), spec("b"), spec("c")];
12231 let carried = ids(&["b", "c"]);
12232 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12233 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12234 }
12235
12236 #[test]
12237 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12238 let roster = [spec("a"), spec("b"), spec("c")];
12239 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12240 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12241 }
12242
12243 #[test]
12244 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12245 let roster = [spec("a"), spec("b"), spec("c")];
12246 let failed = ids(&["a", "b", "c"]);
12247 let mut tried = ids(&["b"]);
12249 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12250 assert_eq!(first.id, "a");
12251 tried.insert(first.id.clone());
12252 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12253 assert_eq!(second.id, "c");
12254 tried.insert(second.id.clone());
12255 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12256 }
12257
12258 #[test]
12259 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12260 let roster = [spec("a"), spec("b")];
12261 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12262 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12263 }
12264
12265 #[test]
12266 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12267 let roster = [spec("a"), spec("b"), spec("c")];
12268 let h = SeatHistory {
12269 failed: ids(&["a"]),
12270 last_ok: Some("b".to_owned()),
12271 last_fail: None,
12272 };
12273 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12274 let gone = SeatHistory {
12276 last_ok: Some("zzz".to_owned()),
12277 ..h.clone()
12278 };
12279 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12280 let failed = SeatHistory {
12281 failed: ids(&["a", "b"]),
12282 last_ok: Some("b".to_owned()),
12283 last_fail: None,
12284 };
12285 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12286 }
12287
12288 #[test]
12289 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12290 let first = SeatState::new("review-1", "alpha", 7);
12291 let next = handover_seat("review-1", "gamma", 7);
12292 assert_ne!(first.claude_session, next.claude_session);
12293 }
12294
12295 #[test]
12296 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12297 let mut state = state_with_summary("x", "y");
12298 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12299 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12300 assert_ne!(a.claude_session, b.claude_session);
12301 }
12302
12303 #[test]
12304 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12305 let roster = [spec("a"), spec("b")];
12306 let h = SeatHistory {
12307 failed: ids(&["a", "b"]),
12308 ..SeatHistory::default()
12309 };
12310 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12311 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12312 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12313 }
12314
12315 #[test]
12322 fn failure_signature_ignores_numbers_and_paths() {
12323 assert_eq!(
12324 failure_signature("exited with Some(2) and no usable output"),
12325 failure_signature("exited with Some(137) and no usable output")
12326 );
12327 assert_eq!(
12328 failure_signature("process exited with no usable output (exit code 2)"),
12329 failure_signature("process exited with no usable output (exit code 137)")
12330 );
12331 for (old, new) in [
12332 (
12333 "exited with Some(2) and no usable output",
12334 "process exited with no usable output (exit code 2)",
12335 ),
12336 (
12337 "exited with None and no usable output",
12338 "process exited with no usable output (exit code unavailable)",
12339 ),
12340 ] {
12341 let previous = FailClass::Other(failure_signature(old));
12342 let current = FailClass::Other(failure_signature(new));
12343 assert_eq!(previous, current, "resumption retains the failure class");
12344 assert!(!should_hand_over(Some(&previous), ¤t));
12345 }
12346 assert_eq!(
12347 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12348 failure_signature("cannot open /var/x.txt: denied")
12349 );
12350 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12351 }
12352
12353 #[test]
12354 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12355 let other = FailClass::Other("x".into());
12356 assert!(should_hand_over(None, &FailClass::Quota));
12357 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12358 assert!(should_hand_over(
12359 Some(&FailClass::Timeout),
12360 &FailClass::Timeout
12361 ));
12362 assert!(should_hand_over(None, &other));
12363 assert!(!should_hand_over(Some(&other), &other));
12364 assert!(should_hand_over(
12365 Some(&other),
12366 &FailClass::Other("y".into())
12367 ));
12368 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12370 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12371 }
12372
12373 #[test]
12374 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12375 let a = SeatState::new("judge-1", "alpha", 7);
12376 let b = handover_seat("judge-1", "beta", 7);
12377 assert_ne!(a.claude_session, b.claude_session);
12378 assert_eq!(b.turns, 0);
12379 }
12380
12381 #[test]
12382 fn a_timeout_is_classified_apart_from_other_failures() {
12383 assert_eq!(
12384 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12385 Some(FailClass::Timeout)
12386 );
12387 assert!(matches!(
12388 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12389 Some(FailClass::Other(_))
12390 ));
12391 }
12392
12393 #[test]
12394 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12395 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12396 let tried = BTreeSet::from(["beta".to_owned()]);
12397 let next = next_untried_in_roster(&roster, 1, &tried);
12400 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12401 }
12402
12403 #[test]
12404 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12405 let roster = vec![spec("alpha"), spec("beta")];
12406 let tried = BTreeSet::from(["beta".to_owned()]);
12407 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12411 }
12412
12413 #[test]
12414 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12415 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12416 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12417 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12421 }
12422
12423 #[test]
12424 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12425 let roster = vec![spec("a"), spec("a"), spec("b")];
12426 let tried = BTreeSet::from(["a".to_owned()]);
12427 let next = next_untried_in_roster(&roster, 0, &tried);
12428 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12429 }
12430
12431 #[test]
12432 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12433 let roster = vec![spec("a"), spec("b")];
12434 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12435 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12436 }
12437
12438 #[test]
12439 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12440 let dir = tempfile::tempdir().unwrap();
12441 let bay = dir.path().join("ffff");
12442
12443 remove_if_empty(&bay);
12445 assert!(!bay.exists());
12446
12447 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12450 remove_if_empty(&bay);
12451 assert!(bay.exists(), "non-empty directory must survive");
12452
12453 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12455 remove_if_empty(&bay);
12456 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12457 }
12458
12459 #[test]
12468 fn a_full_panel_that_found_nothing_is_clean() {
12469 assert!(round_is_clean(
12470 0,
12471 true,
12472 2,
12473 2,
12474 0,
12475 IncompleteReviewPolicy::Block
12476 ));
12477 }
12478
12479 #[test]
12480 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12481 assert!(!round_is_clean(
12482 0,
12483 true,
12484 1,
12485 2,
12486 0,
12487 IncompleteReviewPolicy::Block
12488 ));
12489 }
12490
12491 #[test]
12492 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12493 assert!(!round_is_clean(
12494 1,
12495 true,
12496 1,
12497 2,
12498 0,
12499 IncompleteReviewPolicy::Warn
12500 ));
12501 }
12502
12503 #[test]
12504 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12505 assert!(round_is_clean(
12506 0,
12507 true,
12508 1,
12509 2,
12510 0,
12511 IncompleteReviewPolicy::Warn
12512 ));
12513 }
12514
12515 #[test]
12516 fn a_full_panel_with_an_open_finding_is_not_clean() {
12517 assert!(!round_is_clean(
12518 1,
12519 true,
12520 2,
12521 2,
12522 0,
12523 IncompleteReviewPolicy::Block
12524 ));
12525 }
12526
12527 #[test]
12528 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12529 assert!(!round_is_clean(
12530 0,
12531 false,
12532 2,
12533 2,
12534 0,
12535 IncompleteReviewPolicy::Block
12536 ));
12537 }
12538
12539 #[test]
12546 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12547 assert!(round_is_clean(
12550 0,
12551 true,
12552 1,
12553 2,
12554 1,
12555 IncompleteReviewPolicy::Block
12556 ));
12557 }
12558
12559 #[test]
12560 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12561 assert!(!round_is_clean(
12564 0,
12565 true,
12566 1,
12567 2,
12568 0,
12569 IncompleteReviewPolicy::Block
12570 ));
12571 }
12572
12573 #[test]
12574 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12575 assert!(!round_is_clean(
12576 1,
12577 true,
12578 1,
12579 2,
12580 1,
12581 IncompleteReviewPolicy::Block
12582 ));
12583 assert!(!round_is_clean(
12584 0,
12585 false,
12586 1,
12587 2,
12588 1,
12589 IncompleteReviewPolicy::Block
12590 ));
12591 }
12592
12593 #[test]
12594 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12595 assert!(!round_is_clean(
12599 0,
12600 true,
12601 0,
12602 2,
12603 2,
12604 IncompleteReviewPolicy::Block
12605 ));
12606 }
12607
12608 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12609 CommandOutcome {
12610 command: "test".to_owned(),
12611 code,
12612 output_tail: String::new(),
12613 duration_ms: 0,
12614 resource_blocked,
12615 }
12616 }
12617
12618 #[test]
12619 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12620 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12621 assert!(
12622 !verify_inconclusive(&[outcome(Some(1), false)]),
12623 "an ordinary failure is still evidence about the patch"
12624 );
12625 assert!(verify_inconclusive(&[outcome(None, true)]));
12626 assert!(
12627 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12628 "one inconclusive outcome taints the whole batch"
12629 );
12630 assert!(!verify_inconclusive(&[]));
12631 }
12632
12633 #[tokio::test]
12634 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12635 let calls = std::sync::atomic::AtomicUsize::new(0);
12639 let started = Instant::now();
12640 wait_for_pids_with(
12641 &[123],
12642 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12643 Duration::from_millis(5),
12644 Duration::from_secs(5),
12645 )
12646 .await;
12647 assert!(
12648 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12649 "must keep checking rather than deciding on the first answer"
12650 );
12651 assert!(
12652 started.elapsed() < Duration::from_secs(1),
12653 "must return the moment it is confirmed dead, not wait out the ceiling"
12654 );
12655 }
12656
12657 #[tokio::test]
12658 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12659 let started = Instant::now();
12660 wait_for_pids_with(
12661 &[123],
12662 |_| true, Duration::from_millis(5),
12664 Duration::from_millis(30),
12665 )
12666 .await;
12667 let elapsed = started.elapsed();
12668 assert!(
12669 elapsed >= Duration::from_millis(30),
12670 "must not give up before its own ceiling: {elapsed:?}"
12671 );
12672 assert!(
12673 elapsed < Duration::from_secs(1),
12674 "must not wait past its own ceiling either: {elapsed:?}"
12675 );
12676 }
12677
12678 #[tokio::test]
12679 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12680 let started = Instant::now();
12681 wait_for_pids_with(
12682 &[],
12683 |_| true,
12684 Duration::from_secs(5),
12685 Duration::from_secs(5),
12686 )
12687 .await;
12688 assert!(
12689 started.elapsed() < Duration::from_millis(200),
12690 "an empty pid list has nothing to confirm"
12691 );
12692 }
12693
12694 fn review_round(
12700 clean: bool,
12701 blocking: usize,
12702 answered: usize,
12703 expected: usize,
12704 progressed: bool,
12705 e2e_ok: bool,
12706 ) -> ReviewRound {
12707 ReviewRound {
12708 round: 1,
12709 head: "h".to_owned(),
12710 verified_head: None,
12711 verified_at: None,
12712 reviews: Vec::new(),
12713 e2e: vec![CommandOutcome {
12714 command: "test".to_owned(),
12715 code: Some(if e2e_ok { 0 } else { 1 }),
12716 output_tail: String::new(),
12717 duration_ms: 0,
12718 resource_blocked: false,
12719 }],
12720 verify_retried: false,
12721 e2e_deferred: false,
12722 e2e_defer_reason: None,
12723 fix: None,
12724 blocking,
12725 answered,
12726 expected,
12727 clean,
12728 progressed,
12729 vote_split: false,
12730 reconsideration: Vec::new(),
12731 verdict: None,
12732 }
12733 }
12734
12735 #[test]
12736 fn review_conclusion_is_none_when_nothing_has_run() {
12737 assert_eq!(review_conclusion(&[], 3), None);
12738 }
12739
12740 #[test]
12741 fn review_conclusion_is_none_while_rounds_remain() {
12742 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12743 assert_eq!(review_conclusion(&rounds, 3), None);
12744 }
12745
12746 #[test]
12747 fn review_conclusion_is_gating_once_a_round_is_clean() {
12748 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12749 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12750 }
12751
12752 #[test]
12753 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12754 let rounds = vec![
12755 review_round(false, 1, 2, 2, true, true),
12756 review_round(false, 1, 2, 2, true, true),
12757 ];
12758 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12759 }
12760
12761 #[test]
12762 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12763 let rounds = vec![
12764 review_round(false, 1, 2, 2, true, true),
12765 review_round(false, 1, 2, 2, true, false),
12766 ];
12767 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12768 }
12769
12770 #[test]
12771 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12772 let mut blocked = review_round(false, 1, 2, 2, true, false);
12779 blocked.e2e[0].resource_blocked = true;
12780 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12781 assert_eq!(review_conclusion(&rounds, 2), None);
12782 }
12783
12784 #[test]
12785 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12786 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12788 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12789 }
12790
12791 #[test]
12792 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12793 let rounds = vec![
12794 review_round(false, 1, 2, 2, false, true),
12795 review_round(false, 1, 2, 2, false, true),
12796 ];
12797 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12798 }
12799
12800 fn secs(n: u64) -> Duration {
12801 Duration::from_secs(n)
12802 }
12803
12804 #[tokio::test]
12807 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12808 ask_test_home();
12809 let tmp = tempfile::tempdir().unwrap();
12810 let repo = tmp.path().join("repo");
12811 std::fs::create_dir_all(&repo).unwrap();
12812 init_repo(&repo);
12813 let g = |args: &[&str]| {
12814 let out = std::process::Command::new("git")
12815 .args(args)
12816 .current_dir(&repo)
12817 .quiet()
12818 .output()
12819 .unwrap();
12820 assert!(
12821 out.status.success(),
12822 "{}",
12823 String::from_utf8_lossy(&out.stderr)
12824 );
12825 };
12826 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12827 g(&["checkout", "-b", "historical"]);
12828 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12829 g(&["add", "."]);
12830 g(&["commit", "-m", "old alternative"]);
12831 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12832 g(&["checkout", "-b", "magi/task-selected", &base]);
12833 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12834 g(&["add", "."]);
12835 g(&["commit", "-m", "measurement infrastructure"]);
12836 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12837 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12838 g(&["add", "."]);
12839 g(&["commit", "-m", "dependent implementation"]);
12840 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12841 g(&["checkout", "main"]);
12842 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12843 g(&["add", "."]);
12844 g(&["commit", "-m", "unrelated base change"]);
12845 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12846 let origin = tmp.path().join("origin.git");
12847 g(&[
12848 "clone",
12849 "--bare",
12850 repo.to_str().unwrap(),
12851 origin.to_str().unwrap(),
12852 ]);
12853 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12854 g(&["fetch", "origin"]);
12855 let selection = crate::pr_recovery::Selection {
12856 url: "https://example.test/project/pull/1".into(),
12857 branch: "magi/task-selected".into(),
12858 head: head.clone(),
12859 source_head: None,
12860 source_run: None,
12861 source_proof: None,
12862 disposition: Some(crate::pr_recovery::Disposition::Review),
12863 instruction: "preserve measured behavior".into(),
12864 approved_at: jiff::Timestamp::now(),
12865 };
12866 let mut template = runner_at(RunStatus::Reviewing);
12867 template.state.repo = repo.clone();
12868 template.state.base_commit = current.clone();
12869 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12870 template.state.origin = Some(crate::run::Origin::queue("selected"));
12871 template.state.pr_head = Some(selection.branch.clone());
12872 template.state.pr_recovery = Some(selection.clone());
12873 let mut runner =
12874 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12875 .await
12876 .unwrap();
12877 let wt = runner.state.winner().unwrap().worktree.clone();
12878 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12879 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12880 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12881 assert!(runner.state.seeds.is_empty());
12882 assert_eq!(runner.state.instruction, "preserve measured behavior");
12883 runner.sync_to_base().await.unwrap();
12884 assert_eq!(
12885 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12886 "prerequisite"
12887 );
12888 assert_eq!(
12889 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12890 "measured implementation"
12891 );
12892 assert_eq!(
12893 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12894 "current main change"
12895 );
12896 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12897 assert_eq!(
12898 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12899 head
12900 );
12901
12902 let mut partial = runner_at(RunStatus::Reviewing);
12904 partial.state.repo = repo.clone();
12905 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12906 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12907 run: Some("exit 1".into()),
12908 ..Default::default()
12909 }];
12910 let selection = runner.state.pr_recovery.clone().unwrap();
12911 partial.state.pr_recovery = Some(selection.clone());
12912 partial.state.pr_head = Some(selection.branch.clone());
12913 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12914 let id = partial.state.id.clone();
12915 let error = match Runner::open_selected(
12916 &repo,
12917 partial.state,
12918 partial.roles,
12919 1,
12920 runner.state.base_commit.clone(),
12921 selection,
12922 )
12923 .await
12924 {
12925 Ok(_) => panic!("setup must fail"),
12926 Err(e) => e,
12927 };
12928 assert_eq!(
12929 error
12930 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12931 .unwrap()
12932 .run,
12933 id
12934 );
12935 let failed = RunState::load(&id).unwrap();
12936 assert_eq!(failed.status, RunStatus::Blocked);
12937 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12938 assert!(Runner::resume(&id).is_err());
12939 runner.state.status = RunStatus::Ready;
12940 runner.state.driver_exited = true;
12941 runner.state.save().unwrap();
12942 let mut retry = runner_at(RunStatus::Reviewing);
12943 retry.state.repo = repo.clone();
12944 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12945 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12946 reselected.approved_at = jiff::Timestamp::now();
12947 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12948 retry.state.pr_head = Some(reselected.branch.clone());
12949 retry.state.pr_recovery = Some(reselected.clone());
12950 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12951 .await
12952 .unwrap();
12953 let restarted = Runner::open_selected(
12954 &repo,
12955 retry.state,
12956 retry.roles,
12957 1,
12958 runner.state.base_commit.clone(),
12959 reselected,
12960 )
12961 .await
12962 .unwrap();
12963 assert_eq!(
12964 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12965 .await
12966 .unwrap(),
12967 head
12968 );
12969 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12970 assert_eq!(
12971 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12972 head
12973 );
12974 }
12975
12976 #[tokio::test]
12977 #[cfg(unix)]
12978 async fn fallback_and_interrupted_restoration_require_coordination() {
12979 use std::os::fd::AsRawFd as _;
12980 ask_test_home();
12981 let tmp = tempfile::tempdir().unwrap();
12982 let mut runner = registered_runner(tmp.path()).await;
12983 runner.state.config.graph.timeout_resource_wait = 1;
12984 let winner = runner.state.candidates[0].clone();
12985 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12986 let leftover = winner.worktree.join("leftover");
12987 std::fs::write(&leftover, "untrusted edits").unwrap();
12988 let next = tmp.path().join("next-fixer");
12989 let mut failed = spec("first");
12990 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12991 let mut successor = spec("second");
12992 successor.command = vec![
12993 "sh".into(),
12994 "-c".into(),
12995 format!("touch '{}'; echo done", next.display()),
12996 ];
12997 let attempts = vec![
12998 (failed.clone(), "first".into()),
12999 (successor.clone(), "second".into()),
13000 ];
13001 let lock_path = runner.state.repo.join(".git/updater.lock");
13002 let external = std::fs::File::create(&lock_path).unwrap();
13003 assert_eq!(
13004 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13005 0
13006 );
13007 let artifacts = tmp.path().join("artifacts");
13008 let build = |spec, seat| SeatJob {
13009 spec,
13010 seat,
13011 cwd: winner.worktree.clone(),
13012 prompt: "fixture".into(),
13013 timeout: Duration::from_secs(10),
13014 allow_write: true,
13015 sessions: false,
13016 artifacts: artifacts.clone(),
13017 stem: "fix".into(),
13018 handover: None,
13019 };
13020 let (_, _, out) = runner
13021 .ask_fixer_among(
13022 attempts.clone(),
13023 "graft",
13024 None,
13025 Some((&winner.worktree, &head)),
13026 build,
13027 )
13028 .await;
13029 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
13030 assert!(leftover.exists(), "contention must prevent clean");
13031 assert!(!next.exists());
13032 assert!(runner.state.handovers.is_empty());
13033 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13035 from_run: "earlier".into(),
13036 generation: 1,
13037 mode: crate::recovery::Mode::VerifyRepair,
13038 reasons: Vec::new(),
13039 reviewed_head: None,
13040 rounds: Vec::new(),
13041 open_findings: Vec::new(),
13042 evidence: None,
13043 criteria_inherited: true,
13044 repair: crate::recovery::Repair::Started {
13045 before: head.clone(),
13046 },
13047 });
13048 assert!(
13049 runner
13050 .recovery_repair(&winner)
13051 .await
13052 .unwrap_err()
13053 .is::<crate::repo_lock::ResourceBlocked>()
13054 );
13055 assert!(matches!(
13056 runner.state.recovery.as_ref().unwrap().repair,
13057 crate::recovery::Repair::Started { .. }
13058 ));
13059 assert!(leftover.exists());
13060 drop(external);
13061 runner.recovery_repair(&winner).await.unwrap();
13062 assert!(!leftover.exists());
13063 assert!(matches!(
13064 runner.state.recovery.as_ref().unwrap().repair,
13065 crate::recovery::Repair::Done { .. }
13066 ));
13067 std::fs::remove_file(&lock_path).unwrap();
13069 std::fs::create_dir(&lock_path).unwrap();
13070 std::fs::write(&leftover, "untrusted edits").unwrap();
13071 let (_, _, out) = runner
13072 .ask_fixer_among(
13073 attempts,
13074 "graft",
13075 None,
13076 Some((&winner.worktree, &head)),
13077 build,
13078 )
13079 .await;
13080 assert!(matches!(out, AgentOutcome::Failed(_)));
13081 assert!(leftover.exists());
13082 assert!(!next.exists());
13083 assert_eq!(
13084 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13085 head
13086 );
13087 }
13088
13089 #[cfg(unix)]
13090 async fn registered_runner(root: &Path) -> Runner {
13091 ask_test_home();
13092 let repo = root.join("repo");
13093 std::fs::create_dir_all(&repo).unwrap();
13094 init_repo(&repo);
13095 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13096 let mut runner = runner_at(RunStatus::Prep);
13097 runner.state.repo = repo.clone();
13098 runner.state.config.coordination = crate::config::test_coordination();
13099 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13100 runner.state.config.graph.worktree_root = Some(root.join("wt"));
13101 runner.state.config.graph.advise = false;
13102 runner.roles.implementers = vec![spec("implement")];
13103 runner.roles.judges = vec![spec("judge")];
13104 runner.prep().await.unwrap();
13105 runner
13106 }
13107
13108 #[tokio::test]
13109 #[cfg(unix)]
13110 async fn parked_implementer_persists_completed_work() {
13111 for registered in [false, true] {
13112 let tmp = tempfile::tempdir().unwrap();
13113 let mut runner = registered_runner(tmp.path()).await;
13114 if !registered {
13115 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
13116 }
13117 let marker = tmp.path().join("started");
13118 let release = tmp.path().join("release");
13119 let mut implementer = spec("implement");
13120 implementer.command = vec![
13121 "sh".into(),
13122 "-c".into(),
13123 format!(
13124 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
13125 marker.display(),
13126 release.display()
13127 ),
13128 ];
13129 runner.roles.implementers = vec![implementer.clone()];
13130 runner.state.config.agents = vec![implementer];
13131 let pause = runner.pause.clone();
13132 let (result, ()) = tokio::join!(runner.implement(), async {
13133 tokio::time::timeout(Duration::from_secs(10), async {
13134 while !marker.exists() {
13135 tokio::time::sleep(Duration::from_millis(10)).await;
13136 }
13137 })
13138 .await
13139 .unwrap();
13140 pause.park();
13141 std::fs::write(&release, "go").unwrap();
13142 });
13143 result.unwrap();
13144 let candidate = &runner.state.candidates[0];
13145 assert!(candidate.commits > 0, "{candidate:?}");
13146 assert!(candidate.summary.contains("saved-result"));
13147 assert_eq!(candidate.files, 1);
13148 let before = candidate.commits;
13149 runner.implement().await.unwrap();
13150 assert_eq!(runner.state.candidates[0].commits, before);
13151 }
13152 }
13153
13154 #[tokio::test]
13155 #[cfg(unix)]
13156 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
13157 let tmp = tempfile::tempdir().unwrap();
13158 let mut runner = registered_runner(tmp.path()).await;
13159 let winner = runner.state.candidates[0].clone();
13160 let cache = tmp.path().join("cache");
13161 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
13162 runner.state.config.graph.timeout_resource_wait = 0;
13163 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
13164 let crate::cache::AcquireOutcome::Acquired(held) =
13165 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
13166 else {
13167 panic!("cache must be free")
13168 };
13169 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13170 from_run: "earlier".into(),
13171 generation: 1,
13172 mode: crate::recovery::Mode::VerifyRepair,
13173 reasons: Vec::new(),
13174 reviewed_head: None,
13175 rounds: Vec::new(),
13176 open_findings: Vec::new(),
13177 evidence: None,
13178 criteria_inherited: true,
13179 repair: crate::recovery::Repair::NotStarted,
13180 });
13181 runner.recovery_repair(&winner).await.unwrap();
13182 assert!(matches!(
13183 runner.state.recovery.as_ref().unwrap().repair,
13184 crate::recovery::Repair::WaitingForRestore { .. }
13185 ));
13186 drop(held);
13187 runner.state.resource_block = None;
13188 runner.state.config.graph.timeout_resource_wait = 1;
13189 runner.recovery_repair(&winner).await.unwrap();
13190 assert!(matches!(
13191 runner.state.recovery.as_ref().unwrap().repair,
13192 crate::recovery::Repair::Done { .. }
13193 ));
13194 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13195 }
13196
13197 #[tokio::test]
13198 #[cfg(unix)]
13199 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13200 use std::os::fd::AsRawFd as _;
13201 for graft in [false, true] {
13202 let tmp = tempfile::tempdir().unwrap();
13203 let mut runner = registered_runner(tmp.path()).await;
13204 runner.state.config.graph.timeout_resource_wait = 1;
13205 let winner = runner.state.candidates[0].clone();
13206 let marker = tmp.path().join("fixer-started");
13207 let mut fixer = spec("fixer");
13208 fixer.command = vec![
13209 "sh".into(),
13210 "-c".into(),
13211 format!("echo started >> '{}'; echo done", marker.display()),
13212 ];
13213 runner.roles.fixer = Some(vec![fixer]);
13214 if graft {
13215 let mut loser = winner.clone();
13216 loser.label = 'B';
13217 loser.index = 1;
13218 runner.state.candidates.push(loser);
13219 runner.state.config.graph.graft_runner_up = true;
13220 runner.state.tally = Some(Tally {
13221 first_choice: BTreeMap::new(),
13222 borda: BTreeMap::new(),
13223 winner: winner.label,
13224 rankings: 1,
13225 unanimous_initial: true,
13226 deliberated: false,
13227 changed_votes: 0,
13228 unanimous_final: true,
13229 tie_break: None,
13230 judges: 1,
13231 present: 1,
13232 quorum: 1,
13233 met_quorum: true,
13234 uncontested: None,
13235 });
13236 runner.state.judgements.push(crate::run::Judgement {
13237 judge: 1,
13238 seat: "judge-1".into(),
13239 agent: "judge".into(),
13240 ranking: vec![winner.label, 'B'],
13241 reasons: BTreeMap::new(),
13242 confidence: None,
13243 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13244 order: vec![0, 1],
13245 failed: None,
13246 duration_ms: 0,
13247 });
13248 } else {
13249 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13250 from_run: "earlier".into(),
13251 generation: 1,
13252 mode: crate::recovery::Mode::VerifyRepair,
13253 reasons: Vec::new(),
13254 reviewed_head: None,
13255 rounds: Vec::new(),
13256 open_findings: Vec::new(),
13257 evidence: None,
13258 criteria_inherited: true,
13259 repair: crate::recovery::Repair::NotStarted,
13260 });
13261 }
13262 let reader = crate::repo_lock::SeatLease::for_checkout(
13263 &winner.worktree,
13264 false,
13265 crate::config::test_coordination().as_ref(),
13266 )
13267 .await
13268 .unwrap();
13269 let pause = runner.pause.clone();
13270 let (result, ()) = tokio::join!(
13271 async {
13272 if graft {
13273 runner.graft().await
13274 } else {
13275 runner.recovery_repair(&winner).await
13276 }
13277 },
13278 async {
13279 tokio::time::sleep(Duration::from_millis(100)).await;
13280 pause.park();
13281 }
13282 );
13283 result.unwrap();
13284 assert!(!marker.exists(), "a blocked seat must not spawn");
13285 runner.state = RunState::load(&runner.state.id).unwrap();
13287 if graft {
13288 assert_eq!(
13289 runner.state.graft.as_ref().unwrap().outcome,
13290 GraftOutcome::WaitingForRestore
13291 );
13292 } else {
13293 assert!(matches!(
13294 runner.state.recovery.as_ref().unwrap().repair,
13295 crate::recovery::Repair::WaitingForRestore { .. }
13296 ));
13297 }
13298 drop(reader);
13299 runner.pause = Pause::new();
13300 runner.state.resource_block = None;
13301 let external =
13303 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13304 assert_eq!(
13305 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13306 0
13307 );
13308 let leftover = winner.worktree.join("leftover");
13309 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13310 let result = if graft {
13311 runner.graft().await
13312 } else {
13313 runner.recovery_repair(&winner).await
13314 };
13315 assert!(
13316 result
13317 .unwrap_err()
13318 .is::<crate::repo_lock::ResourceBlocked>()
13319 );
13320 assert!(leftover.exists());
13321 assert!(!marker.exists());
13322 drop(external);
13323 runner.state.resource_block = None;
13324 let result = if graft {
13325 runner.graft().await
13326 } else {
13327 runner.recovery_repair(&winner).await
13328 };
13329 result.unwrap();
13330 assert!(!leftover.exists(), "restore must precede retry");
13331 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13332 if graft {
13333 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13334 } else {
13335 assert!(matches!(
13336 runner.state.recovery.as_ref().unwrap().repair,
13337 crate::recovery::Repair::Done { .. }
13338 ));
13339 }
13340 }
13341 }
13342
13343 #[tokio::test]
13344 #[cfg(unix)]
13345 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13346 use std::os::fd::AsRawFd as _;
13347 let tmp = tempfile::tempdir().unwrap();
13348 let mut runner = registered_runner(tmp.path()).await;
13349 runner.state.config.graph.timeout_resource_wait = 1;
13350 let winner = runner.state.candidates[0].clone();
13351 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13352 runner.state.tally = Some(Tally {
13353 first_choice: BTreeMap::new(),
13354 borda: BTreeMap::new(),
13355 winner: winner.label,
13356 rankings: 0,
13357 unanimous_initial: false,
13358 deliberated: false,
13359 changed_votes: 0,
13360 unanimous_final: false,
13361 tie_break: None,
13362 judges: 0,
13363 present: 0,
13364 quorum: 0,
13365 met_quorum: true,
13366 uncontested: Some("fixture".into()),
13367 });
13368 runner.state.graft = Some(GraftRecord {
13369 pre: pre.clone(),
13370 post: None,
13371 lines: None,
13372 limit: None,
13373 agent: None,
13374 outcome: GraftOutcome::Running,
13375 requested: Vec::new(),
13376 adoption: None,
13377 adoption_error: None,
13378 });
13379 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13380 git::commit_all(&winner.worktree, "unfinished graft")
13381 .await
13382 .unwrap();
13383 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13384 let leftover = winner.worktree.join("leftover");
13385 std::fs::write(&leftover, "untrusted edits").unwrap();
13386 let path = runner.state.repo.join(".git/updater.lock");
13387 let external = std::fs::File::create(&path).unwrap();
13388 assert_eq!(
13389 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13390 0
13391 );
13392 assert!(
13393 runner
13394 .graft()
13395 .await
13396 .unwrap_err()
13397 .is::<crate::repo_lock::ResourceBlocked>()
13398 );
13399 assert!(matches!(
13400 runner.state.graft.as_ref().unwrap().outcome,
13401 GraftOutcome::Running
13402 ));
13403 assert_eq!(
13404 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13405 unfinished
13406 );
13407 assert!(leftover.exists());
13408 drop(external);
13409 std::fs::remove_file(&path).unwrap();
13410 std::fs::create_dir(&path).unwrap();
13411 assert!(runner.graft().await.is_err());
13412 assert!(matches!(
13413 runner.state.graft.as_ref().unwrap().outcome,
13414 GraftOutcome::Running
13415 ));
13416 assert_eq!(
13417 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13418 unfinished
13419 );
13420 assert!(leftover.exists());
13421 std::fs::remove_dir(&path).unwrap();
13422 runner.graft().await.unwrap();
13423 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13424 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13425 assert!(!leftover.exists());
13426 }
13427
13428 #[tokio::test]
13429 #[cfg(unix)]
13430 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13431 use std::os::fd::AsRawFd as _;
13432 let tmp = tempfile::tempdir().unwrap();
13433 let mut runner = registered_runner(tmp.path()).await;
13434 runner.state.prep_complete = Some(false);
13435 runner.state.config.graph.timeout_resource_wait = 1;
13436 runner.state.save().unwrap();
13437 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13438 assert_eq!(
13439 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13440 0
13441 );
13442 runner.execute().await.unwrap();
13443 let disk = RunState::load(&runner.state.id).unwrap();
13444 assert!(disk.driver_exited);
13445 assert_eq!(disk.driver_pid, None);
13446 assert_eq!(runner.state.status, RunStatus::Blocked);
13447 assert_eq!(
13448 runner.state.resource_block.as_ref().unwrap().kind,
13449 "repository coordination"
13450 );
13451 assert!(runner.state.prep_failure.is_none());
13452 assert!(runner.state.seats.is_empty());
13453 runner.pause.park();
13454 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13457 .await
13458 .unwrap()
13459 .unwrap();
13460 assert!(runner.state.parked);
13461 assert!(runner.state.prep_failure.is_none());
13462 let disk = RunState::load(&runner.state.id).unwrap();
13463 assert!(disk.driver_exited);
13464 assert_eq!(disk.driver_pid, None);
13465 }
13466
13467 #[tokio::test]
13468 #[cfg(unix)]
13469 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13470 use std::os::fd::AsRawFd as _;
13471 let tmp = tempfile::tempdir().unwrap();
13472 let mut runner = registered_runner(tmp.path()).await;
13473 runner.state.prep_complete = Some(false);
13474 runner.state.config.graph.timeout_resource_wait = 30;
13475 runner.state.save().unwrap();
13476 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13477 assert_eq!(
13478 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13479 0
13480 );
13481 assert!(
13482 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13483 .await
13484 .is_err()
13485 );
13486 let disk = RunState::load(&runner.state.id).unwrap();
13487 assert!(disk.driver_exited);
13488 assert_eq!(disk.driver_pid, None);
13489 assert!(disk.status.resumable());
13490 assert!(runner.state.driver_exited);
13491 assert_eq!(runner.state.driver_pid, None);
13492 runner.state.save().unwrap();
13493 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13494 }
13495
13496 #[tokio::test]
13497 #[cfg(unix)]
13498 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13499 let tmp = tempfile::tempdir().unwrap();
13500 let mut runner = registered_runner(tmp.path()).await;
13501 let winner = runner.state.candidates[0].clone();
13502 let cache = tmp.path().join("cache");
13503 let started = tmp.path().join("pre-gate-started");
13504 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13505 runner.state.config.verify.gate =
13506 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13507 let home = crate::run::home();
13508 let owner =
13509 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13510 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13511 crate::cache::AcquireOutcome::Acquired(g) => g,
13512 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13513 };
13514 let mut gate = Box::pin(runner.run_gate(&winner));
13515 assert!(
13516 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13517 .await
13518 .is_err()
13519 );
13520 assert!(!started.exists(), "pre-gate must wait for the cache too");
13521 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13525 let editor = crate::repo_lock::SeatLease::checkout_with(
13526 &winner.worktree,
13527 true,
13528 crate::config::test_coordination().as_ref(),
13529 &wait,
13530 )
13531 .await
13532 .unwrap();
13533 drop(editor);
13534 guard.release();
13535 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13536 .await
13537 .unwrap()
13538 .unwrap();
13539 drop(gate);
13540 assert!(outcomes.iter().all(CommandOutcome::ok));
13541 assert!(started.exists());
13542 }
13543
13544 #[tokio::test]
13545 #[cfg(unix)]
13546 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13547 use std::os::fd::AsRawFd as _;
13548
13549 ask_test_home();
13550 let tmp = tempfile::tempdir().unwrap();
13551 let repo = tmp.path().join("repo");
13552 std::fs::create_dir_all(&repo).unwrap();
13553 init_repo(&repo);
13554 let common = repo.join(".git");
13555 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13556 let mut runner = runner_at(RunStatus::Prep);
13557 runner.state.repo = repo.clone();
13558 runner.state.config.coordination = crate::config::test_coordination();
13559 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13560 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13561 runner.state.config.graph.advise = false;
13562 runner.roles.implementers = vec![spec("implement")];
13563 runner.roles.judges = vec![spec("judge")];
13564 runner.prep().await.unwrap();
13565 let winner = runner.state.candidates[0].clone();
13566 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13569 let completed = tmp.path().join("seat-completed");
13570 runner.roles.implementers[0].command = vec![
13571 "sh".into(),
13572 "-c".into(),
13573 format!("touch '{}'; echo completed", completed.display()),
13574 ];
13575 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13576 assert_eq!(
13577 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13578 0
13579 );
13580 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13581 let mut implement = Box::pin(runner.implement());
13582 tokio::select! {
13583 result = &mut implement => panic!("rescue must wait: {result:?}"),
13584 result = tokio::time::timeout(Duration::from_secs(10), async {
13585 while !completed.exists() {
13586 tokio::time::sleep(Duration::from_millis(20)).await;
13587 }
13588 }) => result.unwrap(),
13589 }
13590 assert!(
13591 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13592 .await
13593 .is_err()
13594 );
13595 assert_eq!(
13596 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13597 before
13598 );
13599 assert_eq!(
13600 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13601 0
13602 );
13603 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13604 .await
13605 .unwrap()
13606 .unwrap();
13607 drop(implement);
13608 assert_ne!(
13609 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13610 before
13611 );
13612 assert!(git::is_clean(&winner.worktree).await.unwrap());
13613
13614 let started = tmp.path().join("pre-gate-started");
13615 let proceed = tmp.path().join("pre-gate-proceed");
13616 let checked = tmp.path().join("gate-ran");
13617 runner.state.config.verify.pre_gate = vec![format!(
13618 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13619 started.display(),
13620 proceed.display()
13621 )];
13622 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13623 assert_eq!(
13624 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13625 0
13626 );
13627 let mut gate = Box::pin(runner.run_gate(&winner));
13628 assert!(
13629 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13630 .await
13631 .is_err()
13632 );
13633 assert!(!started.exists());
13634 assert!(!checked.exists());
13635 assert!(git::is_clean(&winner.worktree).await.unwrap());
13636 assert_eq!(
13637 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13638 0
13639 );
13640 tokio::select! {
13641 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13642 result = tokio::time::timeout(Duration::from_secs(10), async {
13643 while !started.exists() {
13644 tokio::time::sleep(Duration::from_millis(20)).await;
13645 }
13646 }) => result.unwrap(),
13647 }
13648 assert_eq!(
13649 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13650 -1,
13651 "mechanical edits must hold the real lock"
13652 );
13653 std::fs::write(proceed, "ready").unwrap();
13654 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13655 .await
13656 .unwrap()
13657 .unwrap();
13658 drop(gate);
13659 assert!(outcomes.iter().all(CommandOutcome::ok));
13660 assert!(checked.exists());
13661 assert!(runner.state.pre_gate_commit.is_some());
13662 assert!(git::is_clean(&winner.worktree).await.unwrap());
13663 assert_eq!(
13664 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13665 0,
13666 "gate completion must release coordination"
13667 );
13668 drop(outsider);
13669 std::fs::remove_file(common.join("updater.lock")).unwrap();
13670 std::fs::create_dir(common.join("updater.lock")).unwrap();
13671 std::fs::remove_file(&started).unwrap();
13672 std::fs::remove_file(&checked).unwrap();
13673 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13674 let error = runner.run_gate(&winner).await.unwrap_err();
13675 assert!(
13676 error
13677 .to_string()
13678 .contains("acquire mandatory repository lock")
13679 );
13680 assert!(
13681 !started.exists(),
13682 "failed acquisition must prevent pre-gate"
13683 );
13684 assert!(!checked.exists(), "failed acquisition must prevent gate");
13685 assert!(git::is_clean(&winner.worktree).await.unwrap());
13686 assert_eq!(
13687 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13688 head
13689 );
13690 }
13691
13692 #[tokio::test]
13693 #[cfg(unix)]
13694 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13695 use std::os::fd::AsRawFd as _;
13696 use std::time::Duration;
13697
13698 ask_test_home();
13699 let tmp = tempfile::tempdir().unwrap();
13700 let repo = tmp.path().join("repo");
13701 std::fs::create_dir_all(&repo).unwrap();
13702 init_repo(&repo);
13703 let common = repo.join(".git");
13704 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13705 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13706 assert_eq!(
13707 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13708 0
13709 );
13710 let mut runner = runner_at(RunStatus::Prep);
13711 runner.state.repo = repo.clone();
13712 runner.state.config.coordination = crate::config::test_coordination();
13713 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13714 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13715 runner.state.config.graph.advise = true;
13716 runner.state.config.graph.advisors = 1;
13717 runner.roles.implementers = vec![spec("implement")];
13718 runner.roles.judges = vec![spec("judge")];
13719 let root = runner.state.worktree_root();
13720 let started = tmp.path().join("setup-started");
13721 let proceed = tmp.path().join("setup-proceed");
13722 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13723 run: Some(format!(
13724 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13725 started.display(),
13726 proceed.display()
13727 )),
13728 ..Default::default()
13729 }];
13730 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13731 let config = git::git(&repo, &["config", "--local", "--list"])
13732 .await
13733 .unwrap();
13734 let mut prep = Box::pin(runner.prep());
13735 assert!(
13736 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13737 .await
13738 .is_err()
13739 );
13740 assert!(
13741 !root.exists(),
13742 "contention must prevent even checkout creation"
13743 );
13744 assert!(!started.exists(), "contention must prevent setup");
13745 assert_eq!(
13746 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13747 branches
13748 );
13749 assert_eq!(
13750 git::git(&repo, &["config", "--local", "--list"])
13751 .await
13752 .unwrap(),
13753 config
13754 );
13755 drop(lock);
13756 tokio::select! {
13757 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13758 result = tokio::time::timeout(Duration::from_secs(10), async {
13759 while !started.exists() {
13760 tokio::time::sleep(Duration::from_millis(20)).await;
13761 }
13762 }) => result.unwrap(),
13763 }
13764 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13765 assert_eq!(
13766 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13767 -1
13768 );
13769 assert!(
13770 tokio::time::timeout(
13771 Duration::from_millis(100),
13772 crate::repo_lock::SeatLease::for_checkout(
13773 &repo,
13774 true,
13775 crate::config::test_coordination().as_ref()
13776 )
13777 )
13778 .await
13779 .is_err(),
13780 "preparation must also exclude local editors"
13781 );
13782 std::fs::write(proceed, "ready").unwrap();
13783 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13784 .await
13785 .unwrap()
13786 .unwrap();
13787 drop(prep);
13788 assert_eq!(runner.state.prep_complete, Some(true));
13789 assert!(root.join("judge-1").exists());
13790 assert!(root.join("advisor-1").exists());
13791 assert_eq!(
13792 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13793 0
13794 );
13795 }
13796
13797 #[tokio::test]
13798 #[cfg(unix)]
13799 async fn preparation_lock_failure_prevents_repository_changes() {
13800 ask_test_home();
13801 let tmp = tempfile::tempdir().unwrap();
13802 let repo = tmp.path().join("repo");
13803 std::fs::create_dir_all(&repo).unwrap();
13804 init_repo(&repo);
13805 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13806 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13809 let mut runner = runner_at(RunStatus::Prep);
13810 runner.state.repo = repo.clone();
13811 runner.state.config.coordination = crate::config::test_coordination();
13812 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13813 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13814 runner.roles.implementers = vec![spec("implement")];
13815 let error = runner.prep().await.unwrap_err();
13816 assert!(
13817 error
13818 .to_string()
13819 .contains("acquire mandatory repository lock")
13820 );
13821 assert!(!runner.state.worktree_root().exists());
13822 assert!(runner.state.candidates.is_empty());
13823 assert!(!repo.join(".git/config.worktree").exists());
13824 assert_eq!(
13825 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13826 "* main"
13827 );
13828 }
13829
13830 #[tokio::test]
13831 async fn preparation_records_creation_intent_before_git_can_fail() {
13832 ask_test_home();
13833 let tmp = tempfile::tempdir().unwrap();
13834 let repo = tmp.path().join("repo");
13835 std::fs::create_dir_all(&repo).unwrap();
13836 init_repo(&repo);
13837 let mut runner = runner_at(RunStatus::Prep);
13838 runner.state.repo = repo.clone();
13839 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13840 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13841 runner.state.config.blind.commit_msg_hook = false;
13842 runner.roles.implementers = vec![spec("one")];
13843 let label = blind::assign_labels(1, runner.state.seed)[0];
13844 let branch = runner.state.branch_for(label);
13845 git::git(&repo, &["branch", &branch]).await.unwrap();
13848 assert!(runner.prep().await.is_err());
13849 let saved = RunState::load(&runner.state.id).unwrap();
13850 assert!(saved.candidates.is_empty());
13851 let event = saved
13852 .events
13853 .iter()
13854 .find(|e| e.node == "prep-intent")
13855 .unwrap();
13856 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13857 assert_eq!(intent["branch"], branch);
13858 assert_eq!(intent["start"], runner.state.base_commit);
13859 assert_eq!(
13860 intent["worktree"],
13861 runner
13862 .state
13863 .worktree_root()
13864 .join(format!("cand-{label}"))
13865 .to_string_lossy()
13866 .as_ref()
13867 );
13868 assert_eq!(intent["picks"], serde_json::json!([]));
13869 }
13870
13871 #[tokio::test]
13872 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13873 ask_test_home();
13874 let tmp = tempfile::tempdir().unwrap();
13875 let repo = tmp.path().join("repo");
13876 std::fs::create_dir_all(&repo).unwrap();
13877 init_repo(&repo);
13878 let mut runner = runner_at(RunStatus::Prep);
13879 runner.state.repo = repo.clone();
13880 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13881 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13882 runner.state.sources.pick_commits.push(refs::Input {
13883 token: "missing".into(),
13884 sha: "0".repeat(40),
13885 provenance: refs::Provenance::ExplicitCli,
13886 });
13887 assert!(runner.prep().await.is_err());
13888 assert!(runner.state.candidates.is_empty());
13889 assert!(!runner.state.worktree_root().exists());
13890 }
13891
13892 #[tokio::test]
13893 async fn prose_and_legacy_inference_never_import_into_candidates() {
13894 ask_test_home();
13895 let tmp = tempfile::tempdir().unwrap();
13896 let repo = tmp.path().join("repo");
13897 std::fs::create_dir_all(&repo).unwrap();
13898 init_repo(&repo);
13899 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13900 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13901 .await
13902 .unwrap();
13903 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13904 git::git(&repo, &["add", "."]).await.unwrap();
13905 git::git(&repo, &["commit", "-m", "historical implementation"])
13906 .await
13907 .unwrap();
13908 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13909 git::git(&repo, &["checkout", "main"]).await.unwrap();
13910 for instruction in [
13911 format!("Example: magi/history/A at {source}"),
13912 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13913 format!("Never import magi/history/A or {source}"),
13914 format!("Conversation answer: I mentioned {source} only as background"),
13915 ] {
13916 let mut runner = runner_at(RunStatus::Prep);
13917 runner.state.repo = repo.clone();
13918 runner.state.base_commit = base.clone();
13919 runner.state.instruction = instruction.clone();
13920 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13921 runner.state.config.blind.commit_msg_hook = false;
13922 runner.state.seeds = vec![refs::Seed {
13923 token: "magi/history/A".into(),
13924 sha: source.clone(),
13925 kind: refs::SeedKind::Unmerged,
13926 branch: true,
13927 detail: String::new(),
13928 }];
13929 runner.roles.implementers = vec![spec("one")];
13930 assert_eq!(seeded_instruction(&runner.state), instruction);
13931 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13933 legacy.as_object_mut().unwrap().remove("sources");
13934 runner.state = serde_json::from_value(legacy).unwrap();
13935 assert_eq!(runner.state.sources, refs::Sources::default());
13936 runner.state.save().unwrap();
13937 runner.state = RunState::load(&runner.state.id).unwrap();
13938 runner.prep().await.unwrap();
13939 for candidate in &runner.state.candidates {
13940 assert_eq!(
13941 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13942 base
13943 );
13944 assert!(!candidate.worktree.join("unwanted").exists());
13945 }
13946 }
13947 }
13948
13949 #[tokio::test]
13950 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13951 ask_test_home();
13952 let tmp = tempfile::tempdir().unwrap();
13953 let repo = tmp.path().join("repo");
13954 std::fs::create_dir_all(&repo).unwrap();
13955 init_repo(&repo);
13956 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13957 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13958 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13959 git::git(&repo, &["add", "."]).await.unwrap();
13960 git::git(&repo, &["commit", "-m", "required seed"])
13961 .await
13962 .unwrap();
13963 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13964 git::git(&repo, &["checkout", "main"]).await.unwrap();
13965 let mut runner = runner_at(RunStatus::Prep);
13966 runner.state.repo = repo.clone();
13967 runner.state.base_commit = base.clone();
13968 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13969 runner.state.config.graph.advise = false;
13970 runner.state.config.blind.commit_msg_hook = false;
13971 runner.state.sources =
13972 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13973 .await
13974 .unwrap();
13975 runner.roles.implementers = vec![spec("one"), spec("two")];
13976 runner.roles.judges = vec![spec("judge")];
13977 let label = blind::assign_labels(2, runner.state.seed)[0];
13978 let branch = runner.state.branch_for(label);
13979 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13980 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13981 .await
13982 .unwrap();
13983 runner.state.candidates.push(Candidate {
13984 index: 0,
13985 label,
13986 agent: "one".into(),
13987 branch,
13988 worktree,
13989 summary: String::new(),
13990 stat: String::new(),
13991 files: 0,
13992 commits: 0,
13993 empty: false,
13994 failed: None,
13995 verified_noop: None,
13996 duration_ms: 0,
13997 folded: false,
13998 });
13999 runner.state.save().unwrap();
14000 runner.state = RunState::load(&runner.state.id).unwrap();
14002 assert_eq!(runner.state.prep_complete, Some(false));
14003 runner.prep().await.unwrap();
14004 assert_eq!(runner.state.prep_complete, Some(true));
14005 assert_eq!(runner.state.candidates.len(), 2);
14006 for candidate in &runner.state.candidates {
14007 assert_eq!(
14008 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14009 git::tree_of(&repo, &seed).await.unwrap()
14010 );
14011 assert_eq!(
14012 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
14013 "required commit"
14014 );
14015 }
14016 assert!(runner.state.worktree_root().join("judge-1").exists());
14017 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
14018 runner.prep().await.unwrap();
14019 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
14020
14021 runner.state.prep_complete = Some(false);
14022 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
14023 run: Some(format!(
14024 "test -f '{}'",
14025 tmp.path().join("dependency-ready").display()
14026 )),
14027 ..Default::default()
14028 }];
14029 assert!(runner.prep().await.is_err());
14030 assert!(runner.state.prep_failure.is_some());
14031 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
14032 runner.state = RunState::load(&runner.state.id).unwrap();
14033 assert!(
14034 runner.prep().await.is_err(),
14035 "correcting the cause alone must not auto-retry"
14036 );
14037 let failed_at = runner.state.events.last().unwrap().at;
14038 assert!(
14039 runner.retry_preparation(failed_at).is_err(),
14040 "old approvals cannot unlock the hold"
14041 );
14042 let mut task = crate::queue::Task::new(
14043 "retry".into(),
14044 "retry setup".into(),
14045 repo.clone(),
14046 crate::queue::Source::Human,
14047 );
14048 task.start(runner.state.id.clone());
14049 task.hold_machine(Some("setup failed".into()));
14050 task.release_operator("cli", None);
14051 runner
14052 .retry_preparation(task.resumption.as_ref().unwrap().at)
14053 .unwrap();
14054 runner.prep().await.unwrap();
14055 assert!(runner.state.prep_failure.is_none());
14056 assert_eq!(runner.state.prep_complete, Some(true));
14057 assert_eq!(runner.state.candidates.len(), 2);
14058 assert!(runner.state.worktree_root().join("judge-1").exists());
14059 for candidate in &runner.state.candidates {
14060 assert_eq!(
14061 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
14062 git::tree_of(&repo, &seed).await.unwrap()
14063 );
14064 }
14065 }
14066
14067 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
14068 let mut heads = Vec::new();
14069 for candidate in candidates {
14070 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
14071 }
14072 heads
14073 }
14074
14075 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
14076 ask_test_home();
14077 let tmp = tempfile::tempdir().unwrap();
14078 let repo = tmp.path().join("repo");
14079 std::fs::create_dir(&repo).unwrap();
14080 init_repo(&repo);
14081 let mut runner = runner_at(RunStatus::Reviewing);
14082 runner.state.repo = repo.clone();
14083 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14084 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14085 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14086 let branch = format!("candidate-{label}");
14087 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
14088 git::worktree_add_branch(&repo, &worktree, &branch, "main")
14089 .await
14090 .unwrap();
14091 if label == 'B' {
14092 std::fs::write(worktree.join("change"), "winner").unwrap();
14093 git::rescue_commit(&worktree, "candidate change")
14094 .await
14095 .unwrap();
14096 }
14097 runner.state.candidates.push(Candidate {
14098 index,
14099 label,
14100 agent: "alpha".into(),
14101 branch,
14102 worktree,
14103 summary: "completed reply".into(),
14104 stat: String::new(),
14105 files: usize::from(label == 'B'),
14106 commits: usize::from(label == 'B'),
14107 empty: label == 'A',
14108 failed: None,
14109 verified_noop: None,
14110 duration_ms: 1,
14111 folded: false,
14112 });
14113 }
14114 runner.state.judge_skipped = true;
14115 runner.state.tally = Some(Tally {
14116 first_choice: BTreeMap::new(),
14117 borda: BTreeMap::new(),
14118 winner: 'B',
14119 rankings: 0,
14120 unanimous_initial: false,
14121 deliberated: false,
14122 changed_votes: 0,
14123 unanimous_final: false,
14124 tie_break: None,
14125 judges: 0,
14126 present: 0,
14127 quorum: 0,
14128 met_quorum: true,
14129 uncontested: Some("only B produced a change".into()),
14130 });
14131 runner.state.save().unwrap();
14132 (tmp, runner)
14133 }
14134
14135 #[tokio::test]
14136 #[cfg(unix)]
14137 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
14138 for (solo, failed_checkpoint) in
14139 [(true, false), (false, false), (true, true), (false, true)]
14140 {
14141 let (_tmp, mut runner) = preservation_resume_fixture().await;
14142 if solo {
14143 runner.state.candidates.retain(|c| c.label == 'A');
14144 }
14145 runner.state.candidates[0].empty = false;
14146 runner.state.candidates[0].summary.clear();
14147 runner.state.tally = None;
14148 runner.state.judge_skipped = false;
14149 runner.state.status = RunStatus::Implementing;
14150 runner.state.config.disk.min_free_bytes = u64::MAX;
14151 let mut implementer = spec("alpha");
14152 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14153 runner.roles.implementers = vec![implementer];
14154 let patch = runner.state.dir().join("artifacts/cand-A.patch");
14155 if failed_checkpoint {
14156 std::fs::create_dir_all(&patch).unwrap();
14157 }
14158 runner.state.save().unwrap();
14159 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
14160 runner.state = RunState::load(&runner.state.id).unwrap();
14161 assert_eq!(runner.state.status, RunStatus::Blocked);
14162 assert!(runner.state.interrupted_implementations.contains(&'A'));
14163 assert_eq!(runner.state.candidates[0].commits, 0);
14164 assert!(!runner.state.candidates[0].empty);
14165 assert!(runner.state.candidates[0].failed.is_none());
14166 if failed_checkpoint {
14167 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14168 std::fs::remove_dir(patch).unwrap();
14169 }
14170 runner.state.resource_block = None;
14171 runner.state.config.disk.min_free_bytes = 0;
14172 runner.roles.implementers[0].command = vec![
14173 "sh".into(),
14174 "-c".into(),
14175 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
14176 ];
14177 runner.implement().await.unwrap();
14178 assert!(runner.state.interrupted_implementations.is_empty());
14179 assert!(runner.state.candidate_preservation.is_empty());
14180 assert_eq!(runner.state.candidates[0].commits, 1);
14181 assert!(!runner.state.candidates[0].empty);
14182 assert!(
14183 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
14184 .unwrap()
14185 .contains("retried empty interrupted seat")
14186 );
14187 if !solo {
14188 assert_eq!(runner.state.candidates[1].commits, 1);
14189 }
14190 }
14191 }
14192
14193 #[tokio::test]
14194 #[cfg(unix)]
14195 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14196 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14197 let (_tmp, mut runner) = preservation_resume_fixture().await;
14198 runner.state.candidates.retain(|c| c.label == 'A');
14199 runner.state.candidates[0].empty = false;
14200 runner.state.candidates[0].summary.clear();
14201 runner.state.tally = None;
14202 runner.state.judge_skipped = false;
14203 runner.state.status = RunStatus::Implementing;
14204 runner.state.config.disk.min_free_bytes = u64::MAX;
14207 let wt = runner.state.candidates[0].worktree.clone();
14208 std::fs::write(
14209 wt.join("interrupted-source"),
14210 "retain implementation work\n",
14211 )
14212 .unwrap();
14213 if committed {
14214 git::rescue_commit(&wt, "implementation checkpoint")
14215 .await
14216 .unwrap();
14217 }
14218 let lock = git::git(
14219 &wt,
14220 &[
14221 "rev-parse",
14222 "--path-format=absolute",
14223 "--git-path",
14224 "index.lock",
14225 ],
14226 )
14227 .await
14228 .unwrap();
14229 if block_rescue {
14230 std::fs::write(&lock, "fixture-owned lock").unwrap();
14231 }
14232 let mut implementer = spec("alpha");
14233 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14234 runner.roles.implementers = vec![implementer];
14235 runner.state.save().unwrap();
14236 let result = runner.implement().await;
14237 assert_eq!(result.is_err(), block_rescue);
14238 assert_eq!(
14239 runner.state.resource_block.as_ref().unwrap().kind,
14240 "disk pressure"
14241 );
14242 assert!(
14243 runner
14244 .state
14245 .events
14246 .iter()
14247 .any(|e| e.message.contains("agent execution interrupted"))
14248 );
14249 assert!(
14250 runner
14251 .state
14252 .dir()
14253 .join("artifacts/candidate-A-recovery/staged.patch")
14254 .exists()
14255 );
14256 if block_rescue {
14257 let saved = RunState::load(&runner.state.id).unwrap();
14258 assert!(saved.candidate_preservation.contains_key(&'A'));
14259 assert!(!saved.candidates[0].empty);
14260 assert_eq!(
14261 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14262 "retain implementation work\n"
14263 );
14264 std::fs::remove_file(lock).unwrap();
14265 runner.state = saved;
14266 } else {
14267 assert_eq!(runner.state.status, RunStatus::Blocked);
14268 assert_eq!(runner.state.candidates[0].commits, 1);
14269 }
14270 runner.roles.implementers.clear();
14273 runner.state.config.disk.min_free_bytes = 0;
14274 runner.state.resource_block = None;
14275 runner.implement().await.unwrap();
14276 assert!(runner.state.candidate_preservation.is_empty());
14277 assert_eq!(runner.state.candidates[0].commits, 1);
14278 assert!(!runner.state.candidates[0].empty);
14279 assert!(git::is_clean(&wt).await.unwrap());
14280 assert!(
14281 crate::run::read_artifact(&runner.state, "cand-A.patch")
14282 .unwrap()
14283 .contains("retain implementation work")
14284 );
14285 }
14286 }
14287
14288 #[tokio::test]
14289 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14290 let (_tmp, mut runner) = preservation_resume_fixture().await;
14291 runner.implement().await.unwrap();
14293 assert!(runner.state.candidate_preservation.is_empty());
14294 assert!(
14295 !runner
14296 .state
14297 .dir()
14298 .join("artifacts/candidate-A-recovery")
14299 .exists()
14300 );
14301 runner.fold_losers().await.unwrap();
14302 assert!(!runner.state.candidates[0].worktree.exists());
14303 for schema in [14, crate::run::SCHEMA] {
14304 runner.state.schema = schema;
14305 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14306 runner.state.save().unwrap();
14307 runner.state = RunState::load(&runner.state.id).unwrap();
14308 runner.implement().await.unwrap();
14309 assert!(runner.state.candidate_preservation.is_empty());
14310 assert!(runner.state.candidates[0].folded);
14311 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14312 }
14313 }
14314
14315 #[tokio::test]
14316 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14317 let (_tmp, mut runner) = preservation_resume_fixture().await;
14318 let wt = runner.state.candidates[0].worktree.clone();
14319 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14320 runner.state.schema = 14;
14321 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14322 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14323 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14324 for _ in 0..2 {
14325 assert!(runner.implement().await.is_err());
14326 assert_eq!(runner.state.status, RunStatus::Blocked);
14327 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14328 assert!(
14329 runner.state.candidates[0].empty,
14330 "old verdict must remain unusable"
14331 );
14332 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14333 assert!(runner.state.judge_skipped);
14334 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14335 assert!(
14336 crate::run::read_artifact(&runner.state, "cand-A.patch")
14337 .unwrap()
14338 .contains("owned edits")
14339 );
14340 assert!(wt.exists());
14341 assert_eq!(
14342 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14343 .await
14344 .unwrap(),
14345 1
14346 );
14347 runner.state = RunState::load(&runner.state.id).unwrap();
14348 }
14349 }
14350
14351 #[tokio::test]
14352 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14353 let (_tmp, mut runner) = preservation_resume_fixture().await;
14354 let wt = runner.state.candidates[0].worktree.clone();
14355 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14356 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14357 let index_lock = PathBuf::from(
14358 git::git(
14359 &wt,
14360 &[
14361 "rev-parse",
14362 "--path-format=absolute",
14363 "--git-path",
14364 "index.lock",
14365 ],
14366 )
14367 .await
14368 .unwrap(),
14369 );
14370 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14371 runner.state.schema = 14;
14372 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14373 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14374 assert!(runner.implement().await.is_err());
14375 assert!(runner.after_implement().is_err());
14376 assert!(runner.fold_losers().await.is_err());
14377 assert_eq!(
14378 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14379 "failed seat edits"
14380 );
14381 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14382 runner.state = RunState::load(&runner.state.id).unwrap();
14383 std::fs::remove_file(index_lock).unwrap();
14384 assert!(runner.implement().await.is_err());
14386 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14387 assert!(runner.state.candidates[0].empty);
14388 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14389 assert!(
14390 crate::run::read_artifact(&runner.state, "cand-A.patch")
14391 .unwrap()
14392 .contains("failed seat edits")
14393 );
14394 assert!(wt.exists());
14395 }
14396
14397 #[tokio::test]
14398 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14399 let (_tmp, mut runner) = preservation_resume_fixture().await;
14400 let repo = runner.state.repo.clone();
14401 let common = git::git(
14402 &repo,
14403 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14404 )
14405 .await
14406 .unwrap();
14407 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14408 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14409 assert!(runner.fold_losers().await.is_err());
14410 assert!(!runner.state.candidates[0].worktree.exists());
14411 assert!(!runner.state.candidates[0].folded);
14412 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14413 runner.state = RunState::load(&runner.state.id).unwrap();
14414 std::fs::remove_file(branch_lock).unwrap();
14415 let judge = runner.state.worktree_root().join("judge-1");
14416 git::worktree_add_detached(&repo, &judge, "main")
14417 .await
14418 .unwrap();
14419 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14420 runner.roles.judges = vec![spec("judge")];
14421 assert!(runner.fold_losers().await.is_err());
14422 runner.state = RunState::load(&runner.state.id).unwrap();
14423 assert!(
14424 runner.state.candidates[0].folded,
14425 "candidate progress saved before judge failure"
14426 );
14427 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14428 assert!(judge.join("uncommitted").exists());
14429 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14430 runner.fold_losers().await.unwrap();
14431 assert!(!judge.exists());
14432 runner.fold_losers().await.unwrap();
14433 }
14434
14435 #[tokio::test]
14436 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14437 let (_tmp, mut runner) = preservation_resume_fixture().await;
14438 runner.state.candidates.retain(|c| c.label == 'B');
14439 runner.state.tally = None;
14440 runner.state.judge_skipped = false;
14441 runner.state.prep_complete = Some(true);
14442 runner.state.config.graph.advise = false;
14443 runner.state.config.graph.selected_implementer = Some("alpha".into());
14444 let wt = runner.state.candidates[0].worktree.clone();
14445 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14446 runner
14447 .state
14448 .candidate_preservation
14449 .insert('B', "awaiting persistence".into());
14450 let index_lock = git::git(
14451 &wt,
14452 &[
14453 "rev-parse",
14454 "--path-format=absolute",
14455 "--git-path",
14456 "index.lock",
14457 ],
14458 )
14459 .await
14460 .unwrap();
14461 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14462 runner.state.save().unwrap();
14463 assert!(runner.execute_graph().await.is_err());
14464 assert_eq!(runner.state.status, RunStatus::Blocked);
14465 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14466 std::fs::remove_file(index_lock).unwrap();
14467 for _ in 0..2 {
14470 runner.state = RunState::load(&runner.state.id).unwrap();
14471 runner.pause.park();
14472 runner.execute_graph().await.unwrap();
14473 assert_eq!(runner.state.status, RunStatus::Implementing);
14474 assert!(runner.state.candidate_preservation.is_empty());
14475 assert_eq!(runner.state.candidates[0].commits, 2);
14476 assert!(runner.state.tally.is_none());
14477 }
14478 runner.implement().await.unwrap();
14479 assert_eq!(runner.state.status, RunStatus::Judging);
14480 runner.state.status = RunStatus::Blocked;
14482 runner.state.candidates[0].failed = Some("implementation failed".into());
14483 runner.state.save().unwrap();
14484 let error = runner.execute_graph().await.unwrap_err();
14485 assert!(
14486 error
14487 .to_string()
14488 .contains("selected implementation is held")
14489 );
14490 }
14491
14492 #[tokio::test]
14493 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14494 let (_tmp, mut runner) = preservation_resume_fixture().await;
14495 let repo = runner.state.repo.clone();
14496 let candidate = runner.state.candidates[0].clone();
14497 let registered = git::worktree_holding(&repo, &candidate.branch)
14500 .await
14501 .unwrap()
14502 .unwrap();
14503 assert_eq!(
14504 registered.canonicalize().unwrap(),
14505 candidate.worktree.canonicalize().unwrap()
14506 );
14507 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14508 assert_eq!(
14509 git::worktree_holding(&repo, &candidate.branch)
14510 .await
14511 .unwrap(),
14512 Some(registered)
14513 );
14514 assert!(
14515 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14516 "the stale registration initially keeps the branch checked out"
14517 );
14518 runner.fold_losers().await.unwrap();
14519 assert!(runner.state.candidates[0].folded);
14520 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14521 assert_eq!(
14522 git::worktree_holding(&repo, &candidate.branch)
14523 .await
14524 .unwrap(),
14525 None
14526 );
14527 runner.state = RunState::load(&runner.state.id).unwrap();
14528 runner.fold_losers().await.unwrap();
14529 }
14530
14531 #[tokio::test]
14532 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14533 let (_tmp, mut runner) = preservation_resume_fixture().await;
14534 let repo = runner.state.repo.clone();
14535 let candidate = runner.state.candidates[0].clone();
14536 let winner = runner.state.candidates[1].clone();
14537 let registered = git::worktree_holding(&repo, &candidate.branch)
14538 .await
14539 .unwrap()
14540 .unwrap();
14541 assert_eq!(
14542 registered.canonicalize().unwrap(),
14543 candidate.worktree.canonicalize().unwrap()
14544 );
14545 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14546 assert_eq!(
14547 git::worktree_holding(&repo, &candidate.branch)
14548 .await
14549 .unwrap(),
14550 Some(registered)
14551 );
14552 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14553
14554 for _ in 0..2 {
14555 fold_run(&mut runner.state, false, &crate::run::home())
14556 .await
14557 .unwrap();
14558 runner.state = RunState::load(&runner.state.id).unwrap();
14559 assert!(runner.state.candidates[0].folded);
14560 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14561 assert_eq!(
14562 git::worktree_holding(&repo, &candidate.branch)
14563 .await
14564 .unwrap(),
14565 None
14566 );
14567 assert!(!runner.state.candidates[1].folded);
14568 assert!(winner.worktree.exists());
14569 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14570 }
14571 }
14572
14573 #[tokio::test]
14574 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14575 ask_test_home();
14576 let tmp = tempfile::tempdir().unwrap();
14577 let repo = tmp.path().join("repo");
14578 std::fs::create_dir(&repo).unwrap();
14579 init_repo(&repo);
14580 let mut runner = runner_at(RunStatus::Implementing);
14581 runner.state.repo = repo.clone();
14582 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14583 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14584 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14585 let branch = format!("candidate-{label}");
14586 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14587 git::worktree_add_branch(&repo, &wt, &branch, "main")
14588 .await
14589 .unwrap();
14590 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14591 git::git(&wt, &["add", "README.md"]).await.unwrap();
14592 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14593 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14594 runner.state.candidates.push(Candidate {
14595 index,
14596 label,
14597 agent: "alpha".to_owned(),
14598 branch,
14599 worktree: wt,
14600 summary: "completed implementation".to_owned(),
14601 stat: String::new(),
14602 files: 0,
14603 commits: 0,
14604 empty: false,
14605 failed: None,
14606 verified_noop: None,
14607 duration_ms: 42,
14608 folded: false,
14609 });
14610 runner
14611 .state
14612 .candidate_preservation
14613 .insert(label, "awaiting persistence".to_owned());
14614 }
14615 let wt = runner.state.candidates[0].worktree.clone();
14616 let index_lock = git::git(
14617 &wt,
14618 &[
14619 "rev-parse",
14620 "--path-format=absolute",
14621 "--git-path",
14622 "index.lock",
14623 ],
14624 )
14625 .await
14626 .unwrap();
14627 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14628 runner.state.save().unwrap();
14629 assert!(runner.persist_candidates().await.is_err());
14630 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14631 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14632 assert_eq!(
14633 runner.state.candidates[1].commits, 1,
14634 "other results are still collected"
14635 );
14636 assert!(
14637 !runner.state.candidates[0].empty,
14638 "failed rescue is not no change"
14639 );
14640 assert_eq!(
14641 runner.state.candidates[0].summary,
14642 "completed implementation"
14643 );
14644 assert!(runner.after_implement().is_err());
14645 let home = tmp.path().join("fold-home");
14646 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14647 assert!(wt.join("new-file").exists());
14648 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14649 assert!(recovery.join("staged.patch").exists());
14650 assert!(recovery.join("unstaged.patch").exists());
14651 assert_eq!(
14652 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14653 "untracked edits\n"
14654 );
14655 runner.state.begin_pass();
14656 runner.state.end_pass_parked(ParkCause::Upgrade {
14657 from: "1.0".into(),
14658 to: Some("1.1".into()),
14659 });
14660 runner.state.save().unwrap();
14661 runner.state = RunState::load(&runner.state.id).unwrap();
14662 assert!(matches!(
14663 runner.state.passes.last().unwrap().end,
14664 Some(crate::run::PassEnd::Parked {
14665 cause: ParkCause::Upgrade { .. }
14666 })
14667 ));
14668 std::fs::remove_file(index_lock).unwrap();
14669 runner.implement().await.unwrap();
14671 assert!(runner.state.candidate_preservation.is_empty());
14672 assert!(
14673 runner
14674 .state
14675 .candidates
14676 .iter()
14677 .all(|c| c.commits == 1 && !c.empty)
14678 );
14679 assert!(git::is_clean(&wt).await.unwrap());
14680 assert!(
14681 crate::run::read_artifact(&runner.state, "cand-A.patch")
14682 .unwrap()
14683 .contains("later tracked edits")
14684 );
14685 }
14686
14687 #[tokio::test]
14688 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14689 ask_test_home();
14690 let tmp = tempfile::tempdir().unwrap();
14691 let repo = tmp.path().join("repo");
14692 std::fs::create_dir(&repo).unwrap();
14693 init_repo(&repo);
14694 let wt = tmp.path().join("candidate");
14695 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14696 .await
14697 .unwrap();
14698 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14699 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14700 assert!(wt.join("uncommitted").exists());
14701 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14702 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14703 let mut runner = runner_at(RunStatus::Implementing);
14704 runner.state.repo = repo.clone();
14705 runner.state.base_commit = "missing-base".to_owned();
14706 runner.state.candidates.push(Candidate {
14707 index: 0,
14708 label: 'A',
14709 agent: "alpha".into(),
14710 branch: "candidate".into(),
14711 worktree: wt.clone(),
14712 summary: "completed".into(),
14713 stat: String::new(),
14714 files: 0,
14715 commits: 0,
14716 empty: false,
14717 failed: None,
14718 verified_noop: None,
14719 duration_ms: 1,
14720 folded: false,
14721 });
14722 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14723 assert!(
14724 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14725 .await
14726 .is_err()
14727 );
14728 assert!(!runner.state.candidates[0].folded);
14729 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14730 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14731 runner
14732 .state
14733 .candidate_preservation
14734 .insert('A', "pending".into());
14735 assert!(runner.persist_candidates().await.is_err());
14736 assert!(!runner.state.candidates[0].empty);
14737 assert!(!runner.state.candidates[0].folded);
14738 assert!(wt.exists());
14739 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14740 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14741 std::fs::create_dir_all(&patch_path).unwrap();
14742 assert!(
14743 runner.persist_candidates().await.is_err(),
14744 "artifact failure stays pending"
14745 );
14746 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14747 assert!(!runner.state.candidates[0].empty);
14748 std::fs::remove_dir(patch_path).unwrap();
14749 runner.persist_candidates().await.unwrap();
14750 assert!(
14751 runner.state.candidates[0].empty,
14752 "only successful inspection confirms no change"
14753 );
14754 }
14755
14756 fn init_repo(dir: &Path) {
14757 let run = |args: &[&str]| {
14758 let out = std::process::Command::new("git")
14759 .args(args)
14760 .current_dir(dir)
14761 .quiet()
14762 .output()
14763 .expect("spawn git");
14764 assert!(
14765 out.status.success(),
14766 "git {args:?} failed: {}",
14767 String::from_utf8_lossy(&out.stderr)
14768 );
14769 };
14770 run(&["init", "-b", "main"]);
14771 run(&["config", "user.name", "magi test"]);
14772 run(&["config", "user.email", "magi@example.com"]);
14773 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14774 run(&["add", "-A"]);
14775 run(&["commit", "-m", "init"]);
14776 }
14777
14778 fn ask_test_home() {
14786 crate::run::pin_test_home();
14787 }
14788
14789 fn runner_at(status: RunStatus) -> Runner {
14792 let mut state = RunState::new(
14793 PathBuf::from("/nonexistent/repo"),
14794 "main".to_owned(),
14795 "deadbeef".to_owned(),
14796 "task".to_owned(),
14797 Config::default(),
14798 );
14799 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14802 state.id = format!(
14803 "20000101-000000-{:04x}",
14804 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14805 );
14806 state.status = status;
14807 Runner {
14808 state,
14809 roles: ResolvedRoles {
14810 implementers: Vec::new(),
14811 judges: Vec::new(),
14812 reviewers: Vec::new(),
14813 fixer: None,
14814 conductor: conductor(),
14815 implementer_roster: Vec::new(),
14816 judge_roster: Vec::new(),
14817 reviewer_roster: Vec::new(),
14818 },
14819 sem: Arc::new(Semaphore::new(1)),
14820 pause: Pause::new(),
14821 interrupt: Pause::new(),
14822 }
14823 }
14824
14825 #[tokio::test]
14826 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14827 crate::run::pin_test_home();
14828 let mut runner = runner_at(RunStatus::Gating);
14829 runner.state.save().unwrap();
14830 let id = runner.state.id.clone();
14831 let passes = runner.state.passes.len();
14832 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14833 let mut claiming = Box::pin(runner.claim_driver());
14834 std::future::poll_fn(|cx| {
14835 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14836 std::task::Poll::Ready(())
14837 })
14838 .await;
14839 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14840 drop(owner);
14841 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14842 .await
14843 .unwrap()
14844 .unwrap();
14845 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14846 assert!(execution.runner.state.resource_block.is_none());
14847 assert_eq!(
14848 execution.runner.state.liveness(false),
14849 crate::run::Liveness::Live
14850 );
14851 drop(execution);
14852 assert!(RunState::load(&id).unwrap().driver_exited);
14853 }
14854
14855 #[tokio::test]
14856 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14857 crate::run::pin_test_home();
14858 let mut runner = runner_at(RunStatus::Gating);
14859 runner.state.save().unwrap();
14860 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14863 assert!(runner.claim_driver().await.is_err());
14864 assert!(runner.state.driver_exited);
14865 assert_eq!(runner.state.driver_pid, None);
14866 let disk = RunState::load(&runner.state.id).unwrap();
14867 assert!(disk.driver_exited);
14868 assert_eq!(disk.driver_pid, None);
14869 assert_eq!(disk.status, RunStatus::Gating);
14870 }
14871
14872 #[tokio::test]
14873 async fn an_early_graph_error_releases_its_driver() {
14874 crate::run::pin_test_home();
14875 let mut runner = runner_at(RunStatus::Blocked);
14876 runner.state.config.graph.selected_implementer = Some("selected".into());
14877 runner.state.save().unwrap();
14878 assert!(runner.execute().await.is_err());
14879 let disk = RunState::load(&runner.state.id).unwrap();
14880 assert!(disk.driver_exited);
14881 assert_eq!(disk.driver_pid, None);
14882 assert_eq!(disk.status, RunStatus::Blocked);
14883 }
14884
14885 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14886 crate::run::MergeRepair {
14887 question: "repair-question".into(),
14888 head: "old-head".into(),
14889 pr: "https://github.com/o/r/pull/42".into(),
14890 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14891 gate: vec![outcome(Some(0), false)],
14892 gate_head: Some("old-head".into()),
14893 gate_fixes: Vec::new(),
14894 pre_gate: Vec::new(),
14895 phase,
14896 failure: None,
14897 }
14898 }
14899
14900 #[tokio::test]
14901 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14902 crate::run::pin_test_home();
14903 let mut runner = runner_at(RunStatus::Landing);
14904 runner.state.merge_repairs.push(merge_repair_request(
14905 crate::run::MergeRepairPhase::Requested,
14906 ));
14907 runner.state.save().unwrap();
14908 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14909 assert!(
14910 runner
14911 .execute()
14912 .await
14913 .unwrap_err()
14914 .to_string()
14915 .contains("already running")
14916 );
14917 assert_eq!(
14918 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14919 crate::run::MergeRepairPhase::Requested
14920 );
14921 drop(claim);
14922 let mut disk = runner.state.clone();
14923 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14924 disk.save().unwrap();
14925 assert!(
14926 runner
14927 .execute()
14928 .await
14929 .unwrap_err()
14930 .to_string()
14931 .contains("decision changed")
14932 );
14933 assert_eq!(
14934 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14935 crate::run::MergeRepairPhase::Complete
14936 );
14937 }
14938
14939 #[test]
14940 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14941 crate::run::pin_test_home();
14942 let mut runner = runner_at(RunStatus::Landing);
14943 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14944 runner.state.reviews = request.reviews.clone();
14945 runner.state.gate = request.gate.clone();
14946 runner.state.gate_ran = true;
14947 runner.state.land_approval = Some(crate::run::LandApproval {
14948 question: "old".into(),
14949 head: "old-head".into(),
14950 });
14951 runner.state.merge_repairs.push(request);
14952 runner.begin_merge_repair_review(0).unwrap();
14953 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14954 assert!(!runner.state.gate_status().ok());
14955 assert!(runner.state.land_approval.is_none());
14956 let disk = RunState::load(&runner.state.id).unwrap();
14957 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14958 assert!(disk.merge_repairs[0].gate[0].ok());
14959 assert_eq!(
14960 disk.merge_repairs[0].phase,
14961 crate::run::MergeRepairPhase::Reviewing
14962 );
14963 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14964 assert!(disk.reviews.is_empty());
14965 }
14966
14967 #[tokio::test]
14968 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14969 crate::run::pin_test_home();
14970 for phase in [
14971 crate::run::MergeRepairPhase::Fixing,
14972 crate::run::MergeRepairPhase::Requested,
14973 ] {
14974 let mut runner = runner_at(RunStatus::Landing);
14975 runner.state.config.graph.land_rounds = 0;
14976 runner.state.merge_repairs.push(merge_repair_request(phase));
14977 runner.state.save().unwrap();
14978 assert!(!runner.repair_merge_review().await.unwrap());
14979 assert_eq!(runner.state.status, RunStatus::Blocked);
14980 assert_eq!(
14981 runner.state.merge_repairs[0].phase,
14982 crate::run::MergeRepairPhase::Failed
14983 );
14984 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14985 assert!(
14986 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14987 "interrupted"
14988 } else {
14989 "budget exhausted"
14990 })
14991 );
14992 assert!(runner.state.merge.is_none());
14993 }
14994 }
14995
14996 #[test]
15000 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
15001 crate::run::pin_test_home();
15002 let mut runner = runner_at(RunStatus::Implementing);
15003 let interrupt = Pause::new();
15004 runner.watch_interrupt(interrupt.clone());
15005
15006 interrupt.park_because("task a1b2 asked to run first");
15007
15008 assert!(runner.park_here().expect("park_here"));
15009 assert!(runner.state.parked);
15010 let last = runner.state.events.last().expect("a park event");
15011 assert_eq!(last.node, "park");
15012 assert!(
15013 last.message.contains("task a1b2 asked to run first"),
15014 "expected the interrupt reason in {:?}",
15015 last.message
15016 );
15017 }
15018
15019 #[test]
15022 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
15023 crate::run::pin_test_home();
15024 let mut runner = runner_at(RunStatus::Implementing);
15025 runner.state.begin_pass();
15026 let shutdown = Pause::new();
15027 runner.on_pause(shutdown.clone());
15028 let upgrade = ParkCause::Upgrade {
15029 from: "0.121.0".to_owned(),
15030 to: Some("0.121.1".to_owned()),
15031 };
15032 shutdown.park_for(upgrade.clone());
15033 shutdown.park_for(ParkCause::Operator);
15034
15035 assert!(runner.park_here().expect("park_here"));
15036 assert_eq!(
15037 runner.state.passes[0].end,
15038 Some(crate::run::PassEnd::Parked { cause: upgrade })
15039 );
15040 let last = runner.state.events.last().expect("a park event");
15041 assert!(last.message.contains("0.121.0"), "{}", last.message);
15042 }
15043
15044 #[tokio::test]
15045 async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
15046 crate::run::pin_test_home();
15047 let mut runner = runner_at(RunStatus::Reviewing);
15048 runner.state.save().unwrap();
15049 let mut execution = runner.claim_driver().await.expect("first execution");
15050 assert_eq!(execution.runner.state.passes.len(), 1);
15051 execution
15052 .driver
15053 .finish(&mut execution.runner.state)
15054 .await
15055 .unwrap();
15056 }
15057
15058 #[tokio::test]
15059 async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
15060 crate::run::pin_test_home();
15061 let mut runner = runner_at(RunStatus::Reviewing);
15062 runner.state.config.agents = vec![conductor()];
15063 runner.state.config.graph.timeout_resource_wait = 0;
15064 runner.state.save().unwrap();
15065 let id = runner.state.id.clone();
15066 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
15067 assert!(runner.claim_driver().await.is_err());
15068 drop(owner);
15069 drop(runner);
15070 let mut resumed = Runner::resume(&id).unwrap();
15071 assert_eq!(resumed.state.driver_claimed, Some(false));
15072 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
15073 assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
15074 let mut execution = resumed
15075 .claim_driver()
15076 .await
15077 .expect("retry initial claim after reload");
15078 assert_eq!(execution.runner.state.driver_claimed, Some(true));
15079 assert_eq!(execution.runner.state.passes.len(), 1);
15080 execution
15081 .driver
15082 .finish(&mut execution.runner.state)
15083 .await
15084 .unwrap();
15085 }
15086
15087 #[tokio::test]
15088 async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
15089 crate::run::pin_test_home();
15090 let mut runner = runner_at(RunStatus::Reviewing);
15091 runner.state.config.agents = vec![conductor()];
15092 runner.state.save().unwrap();
15093 let id = runner.state.id.clone();
15094 let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
15095 let mut resumed = Runner::resume(&id).unwrap();
15096 assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
15097 assert!(resumed.claim_driver().await.is_err());
15098 assert!(RunState::load(&id).unwrap().passes.is_empty());
15099 drop(lease);
15100 let mut execution = resumed.claim_driver().await.unwrap();
15101 execution
15102 .driver
15103 .finish(&mut execution.runner.state)
15104 .await
15105 .unwrap();
15106 }
15107
15108 #[tokio::test]
15109 async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
15110 crate::run::pin_test_home();
15111 let mut runner = runner_at(RunStatus::Reviewing);
15112 let state = &mut runner.state;
15113 state.config.agents = vec![conductor()];
15114 state.driver_claimed = None;
15115 state.driver_pid = Some(std::process::id());
15116 state.driver_started_at = None;
15117 state.begin_pass();
15118 state.save().unwrap();
15119 let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
15120 std::fs::write(
15121 state.dir().join("driver.json"),
15122 serde_json::to_vec(&serde_json::json!({
15123 "pid": state.driver_pid, "started_at": null,
15124 "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
15125 }))
15126 .unwrap(),
15127 )
15128 .unwrap();
15129 assert_eq!(state.liveness(false), crate::run::Liveness::Live);
15130 drop(lease);
15131 assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
15132 let mut resumed = Runner::resume(&state.id).unwrap();
15133 assert!(!resumed.state.driver_exited);
15134 let mut execution = resumed
15135 .claim_driver()
15136 .await
15137 .expect("lease proves previous execution ended");
15138 assert_eq!(execution.runner.state.passes.len(), 2);
15139 execution
15140 .driver
15141 .finish(&mut execution.runner.state)
15142 .await
15143 .unwrap();
15144 }
15145
15146 #[tokio::test]
15147 async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
15148 crate::run::pin_test_home();
15149 let mut runner = runner_at(RunStatus::Reviewing);
15150 let mut json = serde_json::to_value(&runner.state).unwrap();
15151 json.as_object_mut().unwrap().remove("driver_claimed");
15152 runner.state = serde_json::from_value(json).unwrap();
15153 assert_eq!(runner.state.driver_claimed, None);
15154 runner.state.save().unwrap();
15155 let mut execution = runner
15156 .claim_driver()
15157 .await
15158 .expect("the execution lease, not a missing marker, excludes a second driver");
15159 assert_eq!(execution.runner.state.passes.len(), 1);
15160 execution
15161 .driver
15162 .finish(&mut execution.runner.state)
15163 .await
15164 .unwrap();
15165 }
15166
15167 #[tokio::test]
15168 async fn driver_admission_refuses_a_proven_live_driver() {
15169 crate::run::pin_test_home();
15170 let mut runner = runner_at(RunStatus::Reviewing);
15171 let pid = std::process::id();
15172 runner.state.driver_pid = Some(pid);
15173 runner.state.driver_started_at = crate::proc::process_started_at(pid);
15174 assert!(runner.state.driver_started_at.is_some());
15175 runner.state.save().unwrap();
15176 let error = runner
15177 .claim_driver()
15178 .await
15179 .err()
15180 .expect("a live driver must refuse");
15181 assert!(error.to_string().contains("live driver"), "{error:#}");
15182 let saved = RunState::load(&runner.state.id).unwrap();
15183 assert_eq!(saved.driver_pid, Some(pid));
15184 assert_eq!(saved.status, RunStatus::Reviewing);
15185 assert!(saved.passes.is_empty());
15186 }
15187
15188 #[test]
15189 fn parking_resource_halt_persists_the_pass_once() {
15190 crate::run::pin_test_home();
15191 let mut runner = runner_at(RunStatus::Reviewing);
15192 runner.state.begin_pass();
15193 runner.state.resource_block = Some(crate::run::ResourceBlock {
15194 node: "fix".into(),
15195 kind: "shared build cache".into(),
15196 detail: "cancelled".into(),
15197 at: Timestamp::now(),
15198 cause: crate::run::ResourceBlockCause::Cancelled { park: None },
15199 });
15200 let cause = ParkCause::Upgrade {
15201 from: "old".into(),
15202 to: Some("new".into()),
15203 };
15204 runner.pause.park_for(cause.clone());
15205 runner.stop_for_resource().unwrap();
15208 assert!(runner.resource_halt().unwrap());
15209 assert!(runner.park_here().unwrap());
15210 let saved = RunState::load(&runner.state.id).unwrap();
15211 assert!(saved.parked);
15212 assert_eq!(saved.status, RunStatus::Reviewing);
15213 assert!(saved.passes[0].ended_at.is_some());
15214 assert_eq!(
15215 saved.passes[0].end,
15216 Some(crate::run::PassEnd::Parked {
15217 cause: cause.clone()
15218 })
15219 );
15220 assert_eq!(
15221 saved.resource_block.unwrap().cause,
15222 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
15223 );
15224 assert_eq!(
15225 saved
15226 .events
15227 .iter()
15228 .filter(|event| event.node == "park")
15229 .count(),
15230 1
15231 );
15232 }
15233
15234 #[test]
15242 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
15243 crate::run::pin_test_home();
15244 let mut runner = runner_at(RunStatus::Implementing);
15245 let shutdown = Pause::new();
15246 runner.on_pause(shutdown.clone());
15247 let interrupt = Pause::new();
15248 runner.watch_interrupt(interrupt.clone());
15249
15250 assert!(!runner.park_here().expect("park_here"));
15252 assert!(!runner.state.parked);
15253
15254 interrupt.park_because("test");
15256 assert!(!shutdown.parked());
15257 assert!(runner.park_here().expect("park_here"));
15258 }
15259
15260 #[tokio::test]
15274 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
15275 crate::run::pin_test_home();
15276 let mut runner = runner_at(RunStatus::Implementing);
15277 let interrupt = Pause::new();
15278 runner.watch_interrupt(interrupt.clone());
15279
15280 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
15281 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
15282
15283 let node = async move {
15287 started_tx.send(()).expect("send started");
15288 finish_rx.await.expect("recv finish");
15289 "node finished"
15290 };
15291
15292 let interrupter = async move {
15293 started_rx.await.expect("recv started");
15294 interrupt.park_because("higher-priority task waiting");
15296 tokio::task::yield_now().await;
15300 finish_tx.send(()).expect("send finish");
15301 };
15302
15303 let (node_result, ()) = tokio::join!(node, interrupter);
15304 assert_eq!(
15305 node_result, "node finished",
15306 "the in-flight call ran to completion"
15307 );
15308
15309 assert!(runner.park_here().expect("park_here"));
15312 assert!(runner.state.parked);
15313 }
15314
15315 #[test]
15321 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
15322 crate::run::pin_test_home();
15323 let mut runner = runner_at(RunStatus::Judging);
15324 runner.state.config.agents = vec![conductor()];
15328 runner.state.candidates = vec![Candidate {
15329 index: 0,
15330 label: 'A',
15331 agent: "alpha".to_owned(),
15332 branch: "magi/x/A".to_owned(),
15333 worktree: PathBuf::from("/nonexistent/worktree"),
15334 summary: "did the thing".to_owned(),
15335 stat: "1 file changed".to_owned(),
15336 files: 1,
15337 commits: 1,
15338 empty: false,
15339 failed: None,
15340 verified_noop: None,
15341 duration_ms: 1234,
15342 folded: false,
15343 }];
15344 let run_id = runner.state.id.clone();
15345
15346 let interrupt = Pause::new();
15347 runner.watch_interrupt(interrupt.clone());
15348 interrupt.park_because("task c3d4 asked to run first");
15349 assert!(runner.park_here().expect("park_here"));
15350
15351 let resumed = Runner::resume(&run_id).expect("resume");
15352 assert_eq!(resumed.state.candidates.len(), 1);
15353 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
15354 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
15355 assert_eq!(resumed.state.status, runner.state.status);
15356 assert!(
15357 resumed.state.parked,
15358 "still parked until `execute` actually walks the graph again"
15359 );
15360 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
15361 }
15362
15363 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
15365 let mut q = ask::Question::new(
15366 run.to_owned(),
15367 "implement".to_owned(),
15368 "impl-A".to_owned(),
15369 "Which storage backend should the cache use?".to_owned(),
15370 String::new(),
15371 vec!["SQLite".to_owned(), "Redis".to_owned()],
15372 );
15373 store.put(&mut q).unwrap();
15374 q
15375 }
15376
15377 #[test]
15378 fn a_failed_runs_open_question_is_abandoned() {
15379 ask_test_home();
15380 let store = ask::Questions::open();
15381 let mut runner = runner_at(RunStatus::Failed);
15382 let run = runner.state.id.clone();
15383 let q = ask_open_question(&store, &run);
15384
15385 runner.settle_questions();
15386
15387 let back = store.get(&q.id).unwrap();
15388 assert!(
15389 !back.status.open(),
15390 "the seat that asked died with the run; nobody is left to read an answer"
15391 );
15392 assert!(
15393 back.detail.contains(&run) && back.detail.contains("failed"),
15394 "the reason names what the run became, not just that it is gone: {}",
15395 back.detail
15396 );
15397 }
15398
15399 #[test]
15400 fn a_merged_runs_open_question_is_abandoned_too() {
15401 ask_test_home();
15402 let store = ask::Questions::open();
15403 for status in [RunStatus::Merged, RunStatus::Ready] {
15406 let mut runner = runner_at(status);
15407 let run = runner.state.id.clone();
15408 let q = ask_open_question(&store, &run);
15409
15410 runner.settle_questions();
15411
15412 let back = store.get(&q.id).unwrap();
15413 assert!(
15414 !back.status.open(),
15415 "{status:?} run's question must not outlive the run"
15416 );
15417 }
15418 }
15419
15420 #[test]
15421 fn a_still_resumable_runs_open_question_is_left_alone() {
15422 ask_test_home();
15423 let store = ask::Questions::open();
15424 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15430 let mut runner = runner_at(status);
15431 let run = runner.state.id.clone();
15432 let q = ask_open_question(&store, &run);
15433
15434 runner.settle_questions();
15435
15436 let back = store.get(&q.id).unwrap();
15437 assert!(
15438 back.status.open(),
15439 "{status:?} is still alive; the question must still be waiting"
15440 );
15441 }
15442 }
15443
15444 #[test]
15445 fn settle_questions_never_touches_an_already_answered_question() {
15446 ask_test_home();
15447 let store = ask::Questions::open();
15448 let mut runner = runner_at(RunStatus::Failed);
15449 let run = runner.state.id.clone();
15450 let mut q = ask_open_question(&store, &run);
15451 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15452 .unwrap();
15453 store.put(&mut q).unwrap();
15454
15455 runner.settle_questions();
15460 runner.settle_questions();
15461
15462 let back = store.get(&q.id).unwrap();
15463 assert_eq!(
15464 back.status,
15465 ask::QuestionStatus::Answered,
15466 "a real answer is a decision on record, never overwritten by a sweep"
15467 );
15468 }
15469
15470 #[tokio::test]
15481 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15482 crate::run::pin_test_home();
15483 let tmp = tempfile::tempdir().expect("tempdir");
15484 let repo = tmp.path().join("repo");
15485 std::fs::create_dir_all(&repo).unwrap();
15486 init_repo(&repo);
15487
15488 let mut config = Config::default();
15489 config.graph.worktree_root = Some(tmp.path().join("wt"));
15490
15491 let mut state = RunState::new(
15492 repo.clone(),
15493 "main".to_owned(),
15494 "deadbeef".to_owned(),
15495 "task".to_owned(),
15496 config,
15497 );
15498 let root = state.worktree_root();
15499 let wt_a = root.join("cand-A");
15500 let wt_b = root.join("cand-B");
15501 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15502 .await
15503 .expect("worktree A");
15504 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15505 .await
15506 .expect("worktree B");
15507
15508 state.candidates = vec![
15509 Candidate {
15510 index: 0,
15511 label: 'A',
15512 agent: "alpha".to_owned(),
15513 branch: "magi/x/A".to_owned(),
15514 worktree: wt_a.clone(),
15515 summary: String::new(),
15516 stat: String::new(),
15517 files: 0,
15518 commits: 0,
15519 empty: false,
15520 failed: None,
15521 verified_noop: None,
15522 duration_ms: 0,
15523 folded: false,
15524 },
15525 Candidate {
15526 index: 1,
15527 label: 'B',
15528 agent: "beta".to_owned(),
15529 branch: "magi/x/B".to_owned(),
15530 worktree: wt_b.clone(),
15531 summary: String::new(),
15532 stat: String::new(),
15533 files: 0,
15534 commits: 0,
15535 empty: false,
15536 failed: None,
15537 verified_noop: None,
15538 duration_ms: 0,
15539 folded: false,
15540 },
15541 ];
15542 state.tally = Some(Tally {
15543 first_choice: BTreeMap::from([('A', 1)]),
15544 borda: BTreeMap::new(),
15545 winner: 'A',
15546 rankings: 1,
15547 unanimous_initial: true,
15548 deliberated: false,
15549 changed_votes: 0,
15550 unanimous_final: true,
15551 tie_break: None,
15552 judges: 1,
15553 present: 1,
15554 quorum: 1,
15555 met_quorum: true,
15556 uncontested: None,
15557 });
15558 state.status = RunStatus::Ready;
15559
15560 fold_run(&mut state, false, &crate::run::home())
15561 .await
15562 .expect("fold_run");
15563
15564 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15565 assert!(
15566 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15567 "the unmerged winner's branch survives"
15568 );
15569 assert!(
15570 !state.candidates[0].folded,
15571 "the winner is not marked folded"
15572 );
15573
15574 assert!(!wt_b.exists(), "the loser's worktree is removed");
15575 assert!(
15576 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15577 "the loser's branch is removed"
15578 );
15579 assert!(state.candidates[1].folded, "the loser is marked folded");
15580 }
15581
15582 #[tokio::test]
15583 #[cfg(unix)]
15584 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15585 use std::os::fd::AsRawFd as _;
15586 use std::time::Duration;
15587
15588 let tmp = tempfile::tempdir().unwrap();
15589 let repo = tmp.path().join("repo");
15590 std::fs::create_dir_all(&repo).unwrap();
15591 init_repo(&repo);
15592 let common = repo.join(".git");
15593 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15594 let home = tmp.path().join("home");
15595 let mut config = Config::default();
15596 config.coordination = crate::config::test_coordination();
15597 config.graph.worktree_root = Some(tmp.path().join("wt"));
15598 config.graph.timeout_resource_wait = 1;
15599 let mut state = RunState::new(
15600 repo.clone(),
15601 "main".into(),
15602 "deadbeef".into(),
15603 "task".into(),
15604 config,
15605 );
15606 let root = state.worktree_root();
15607 let candidate = root.join("cand-A");
15608 let reviewer = root.join("review-1");
15609 let branch = state.branch_for('A');
15610 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15611 .await
15612 .unwrap();
15613 git::worktree_add_detached(&repo, &reviewer, "main")
15614 .await
15615 .unwrap();
15616 git::acquire_worktree_config(&repo).await.unwrap();
15617 state.enabled_worktree_config = true;
15618 state.status = RunStatus::Ready;
15619 state.candidates.push(Candidate {
15620 index: 0,
15621 label: 'A',
15622 agent: "implement".into(),
15623 branch: branch.clone(),
15624 worktree: candidate.clone(),
15625 summary: String::new(),
15626 stat: String::new(),
15627 files: 0,
15628 commits: 0,
15629 empty: false,
15630 failed: None,
15631 verified_noop: None,
15632 duration_ms: 0,
15633 folded: false,
15634 });
15635 state.save_under(&home).unwrap();
15636 let config_before = std::fs::read(common.join("config")).unwrap();
15637 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15638 .await
15639 .unwrap();
15640 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15643 assert_eq!(
15644 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15645 0
15646 );
15647 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15648 assert!(
15649 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15650 .await
15651 .is_err()
15652 );
15653 assert!(candidate.exists());
15654 assert!(reviewer.exists());
15655 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15656 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15657 assert_eq!(
15658 git::git(&repo, &["worktree", "list", "--porcelain"])
15659 .await
15660 .unwrap(),
15661 worktrees_before
15662 );
15663 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15664 .await
15665 .unwrap()
15666 .unwrap_err();
15667 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15668 drop(fold);
15669 assert!(candidate.exists() && reviewer.exists());
15670 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15671 assert_eq!(
15672 state.resource_block.as_ref().unwrap().kind,
15673 "repository coordination"
15674 );
15675 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15676 drop(external);
15677 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15678 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15679 .await
15680 .unwrap()
15681 .unwrap();
15682 drop(fold);
15683 assert!(removed.contains(&branch));
15684 assert!(!root.exists());
15685 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15686 assert!(state.candidates[0].folded);
15687 assert!(!state.enabled_worktree_config);
15688 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15689 let enabled = git::git_raw(
15690 &repo,
15691 &["config", "--local", "--get", "extensions.worktreeConfig"],
15692 )
15693 .await
15694 .unwrap();
15695 assert!(!enabled.ok(), "the last config reference was released");
15696 }
15697
15698 #[tokio::test]
15701 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15702 let tmp = tempfile::tempdir().expect("tempdir");
15703 let repo = tmp.path().join("repo");
15704 std::fs::create_dir_all(&repo).unwrap();
15705 init_repo(&repo);
15706 let home = tmp.path().join("home");
15707
15708 let mut config = Config::default();
15709 config.graph.worktree_root = Some(tmp.path().join("wt"));
15710 let mut state = RunState::new(
15711 repo.clone(),
15712 "main".to_owned(),
15713 "deadbeef".to_owned(),
15714 "task".to_owned(),
15715 config,
15716 );
15717 git::git(&repo, &["branch", "magi/x/A", "main"])
15719 .await
15720 .expect("branch");
15721 state.candidates = vec![Candidate {
15722 index: 0,
15723 label: 'A',
15724 agent: "alpha".to_owned(),
15725 branch: "magi/x/A".to_owned(),
15726 worktree: state.worktree_root().join("cand-A"),
15727 summary: String::new(),
15728 stat: String::new(),
15729 files: 0,
15730 commits: 0,
15731 empty: false,
15732 failed: None,
15733 verified_noop: None,
15734 duration_ms: 0,
15735 folded: true,
15736 }];
15737 state.released_to = Some("20260901-000000-new1".to_owned());
15738 state.released_branches = vec!["magi/x/A".to_owned()];
15739
15740 fold_run(&mut state, true, &home).await.expect("fold_run");
15741 state.status = RunStatus::Merged;
15742 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15743 state.merge = Some(crate::run::MergeOutcome {
15744 mode: MergeMode::Pr,
15745 ok: true,
15746 detail: "confirmed merged".into(),
15747 empty: false,
15748 });
15749 fold_run(&mut state, true, &home)
15750 .await
15751 .expect("merged fold");
15752
15753 assert!(
15754 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15755 "the handed-over branch survives a fold"
15756 );
15757 }
15758
15759 #[tokio::test]
15763 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15764 let tmp = tempfile::tempdir().expect("tempdir");
15765 let repo = tmp.path().join("repo");
15766 std::fs::create_dir_all(&repo).unwrap();
15767 init_repo(&repo);
15768 let run = |args: &[&str]| {
15769 let out = std::process::Command::new("git")
15770 .quiet()
15771 .args(args)
15772 .current_dir(&repo)
15773 .output()
15774 .expect("spawn git");
15775 assert!(out.status.success(), "git {args:?}");
15776 };
15777 run(&["branch", "magi/x/A"]);
15778 run(&["checkout", "-q", "-b", "magi/x/B"]);
15779 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15780 run(&["add", "-A"]);
15781 run(&["commit", "-q", "-m", "work"]);
15782 run(&["checkout", "-q", "main"]);
15783
15784 let probe = RunState::new(
15788 repo.clone(),
15789 "main".to_owned(),
15790 "deadbeef".to_owned(),
15791 "task".to_owned(),
15792 Config::default(),
15793 );
15794 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15795 let bare = tmp.path().join("origin.git");
15796 let out = std::process::Command::new("git")
15797 .quiet()
15798 .args(["init", "-q", "--bare"])
15799 .arg(&bare)
15800 .output()
15801 .expect("spawn git");
15802 assert!(out.status.success(), "git init --bare");
15803 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15804 run(&["push", "-q", "origin", "main"]);
15805
15806 let mut state = RunState::new(
15807 repo.clone(),
15808 "main".to_owned(),
15809 "deadbeef".to_owned(),
15810 "task".to_owned(),
15811 Config::default(),
15812 );
15813 state.seeds = vec![refs::Seed {
15814 token: "magi/27b2/A".to_owned(),
15815 kind: refs::SeedKind::Unresolved,
15816 sha: String::new(),
15817 branch: true,
15818 detail: "no branch or commit named magi/27b2/A".to_owned(),
15819 }];
15820
15821 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15822 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15823 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15824 let detail = empty_candidate_detail(&state, "main");
15825 assert!(detail.starts_with("empty candidate"), "{detail}");
15826 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15827 assert!(detail.contains("--start-from"), "{detail}");
15828 }
15829
15830 #[tokio::test]
15839 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15840 let tmp = tempfile::tempdir().expect("tempdir");
15841 let repo = tmp.path().join("repo");
15842 std::fs::create_dir_all(&repo).unwrap();
15843 init_repo(&repo);
15844
15845 let mut config = Config::default();
15846 config.merge.mode = MergeMode::Local;
15847
15848 let mut state = RunState::new(
15849 repo.clone(),
15850 "main".to_owned(),
15851 "deadbeef".to_owned(),
15852 "task".to_owned(),
15853 config,
15854 );
15855 state.candidates = vec![Candidate {
15856 index: 0,
15857 label: 'A',
15858 agent: "alpha".to_owned(),
15859 branch: "does-not-exist".to_owned(),
15860 worktree: repo.clone(),
15861 summary: String::new(),
15862 stat: String::new(),
15863 files: 0,
15864 commits: 0,
15865 empty: false,
15866 failed: None,
15867 verified_noop: None,
15868 duration_ms: 0,
15869 folded: false,
15870 }];
15871 state.tally = Some(Tally {
15872 first_choice: BTreeMap::from([('A', 1)]),
15873 borda: BTreeMap::new(),
15874 winner: 'A',
15875 rankings: 1,
15876 unanimous_initial: true,
15877 deliberated: false,
15878 changed_votes: 0,
15879 unanimous_final: true,
15880 tie_break: None,
15881 judges: 0,
15882 present: 0,
15883 quorum: 0,
15884 met_quorum: true,
15885 uncontested: Some("only candidate A produced a change".to_owned()),
15886 });
15887 state.reviews = vec![ReviewRound {
15888 round: 1,
15889 head: "deadbeef".to_owned(),
15890 verified_head: None,
15891 verified_at: None,
15892 reviews: Vec::new(),
15893 e2e: Vec::new(),
15894 fix: None,
15895 blocking: 0,
15896 answered: 0,
15897 expected: 0,
15898 clean: true,
15899 verify_retried: false,
15900 e2e_deferred: false,
15901 e2e_defer_reason: None,
15902 progressed: false,
15903 vote_split: false,
15904 reconsideration: Vec::new(),
15905 verdict: None,
15906 }];
15907 state.gate = vec![CommandOutcome {
15908 command: "test".to_owned(),
15909 code: Some(0),
15910 output_tail: String::new(),
15911 duration_ms: 0,
15912 resource_blocked: false,
15913 }];
15914 state.gate_ran = true;
15915 state.status = RunStatus::Ready;
15920 state.merge = Some(MergeOutcome {
15921 mode: MergeMode::Local,
15922 ok: false,
15923 detail: "already concluded".to_owned(),
15924 empty: false,
15925 });
15926
15927 let mut runner = Runner {
15928 state,
15929 roles: ResolvedRoles {
15930 implementers: Vec::new(),
15931 judges: Vec::new(),
15932 reviewers: Vec::new(),
15933 fixer: None,
15934 conductor: conductor(),
15935 implementer_roster: Vec::new(),
15936 judge_roster: Vec::new(),
15937 reviewer_roster: Vec::new(),
15938 },
15939 sem: Arc::new(Semaphore::new(1)),
15940 pause: Pause::new(),
15941 interrupt: Pause::new(),
15942 };
15943
15944 runner.merge().await.expect("merge");
15945
15946 assert_eq!(
15947 runner.state.status,
15948 RunStatus::Ready,
15949 "a concluded run's status must not change on reentry"
15950 );
15951 assert_eq!(
15952 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15953 Some("already concluded"),
15954 "merge must not run again once the node already recorded an outcome"
15955 );
15956 }
15957
15958 #[tokio::test]
15967 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15968 let tmp = tempfile::tempdir().expect("tempdir");
15969 let repo = tmp.path().join("repo");
15970 std::fs::create_dir_all(&repo).unwrap();
15971 init_repo(&repo);
15972
15973 let mut config = Config::default();
15974 config.merge.mode = MergeMode::Local;
15975
15976 let mut state = RunState::new(
15977 repo.clone(),
15978 "main".to_owned(),
15979 "deadbeef".to_owned(),
15980 "task".to_owned(),
15981 config,
15982 );
15983 state.candidates = vec![Candidate {
15984 index: 0,
15985 label: 'A',
15986 agent: "alpha".to_owned(),
15987 branch: "does-not-exist".to_owned(),
15988 worktree: repo.clone(),
15989 summary: String::new(),
15990 stat: String::new(),
15991 files: 0,
15992 commits: 0,
15993 empty: false,
15994 failed: None,
15995 verified_noop: None,
15996 duration_ms: 0,
15997 folded: false,
15998 }];
15999 state.tally = Some(Tally {
16000 first_choice: BTreeMap::from([('A', 1)]),
16001 borda: BTreeMap::new(),
16002 winner: 'A',
16003 rankings: 1,
16004 unanimous_initial: true,
16005 deliberated: false,
16006 changed_votes: 0,
16007 unanimous_final: true,
16008 tie_break: None,
16009 judges: 0,
16010 present: 0,
16011 quorum: 0,
16012 met_quorum: true,
16013 uncontested: Some("only candidate A produced a change".to_owned()),
16014 });
16015 state.reviews = vec![ReviewRound {
16016 round: 1,
16017 head: "deadbeef".to_owned(),
16018 verified_head: None,
16019 verified_at: None,
16020 reviews: Vec::new(),
16021 e2e: Vec::new(),
16022 fix: None,
16023 blocking: 0,
16024 answered: 0,
16025 expected: 0,
16026 clean: true,
16027 verify_retried: false,
16028 e2e_deferred: false,
16029 e2e_defer_reason: None,
16030 progressed: false,
16031 vote_split: false,
16032 reconsideration: Vec::new(),
16033 verdict: None,
16034 }];
16035 state.gate = Vec::new();
16037 state.gate_ran = false;
16038 state.status = RunStatus::Gating;
16039
16040 let mut runner = Runner {
16041 state,
16042 roles: ResolvedRoles {
16043 implementers: Vec::new(),
16044 judges: Vec::new(),
16045 reviewers: Vec::new(),
16046 fixer: None,
16047 conductor: conductor(),
16048 implementer_roster: Vec::new(),
16049 judge_roster: Vec::new(),
16050 reviewer_roster: Vec::new(),
16051 },
16052 sem: Arc::new(Semaphore::new(1)),
16053 pause: Pause::new(),
16054 interrupt: Pause::new(),
16055 };
16056
16057 runner.merge().await.expect("merge");
16058
16059 assert!(
16060 runner.state.merge.is_none(),
16061 "an empty gate must never be read as a passing one: {:?}",
16062 runner.state.merge
16063 );
16064 }
16065
16066 #[tokio::test]
16073 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
16074 let tmp = tempfile::tempdir().expect("tempdir");
16075 let repo = tmp.path().join("repo");
16076 std::fs::create_dir_all(&repo).unwrap();
16077 init_repo(&repo);
16078
16079 let config = Config::default();
16081
16082 let mut state = RunState::new(
16083 repo.clone(),
16084 "main".to_owned(),
16085 "deadbeef".to_owned(),
16086 "task".to_owned(),
16087 config,
16088 );
16089 state.candidates = vec![Candidate {
16090 index: 0,
16091 label: 'A',
16092 agent: "alpha".to_owned(),
16093 branch: "does-not-exist".to_owned(),
16094 worktree: repo.clone(),
16095 summary: String::new(),
16096 stat: String::new(),
16097 files: 0,
16098 commits: 0,
16099 empty: false,
16100 failed: None,
16101 verified_noop: None,
16102 duration_ms: 0,
16103 folded: false,
16104 }];
16105 state.tally = Some(Tally {
16106 first_choice: BTreeMap::from([('A', 1)]),
16107 borda: BTreeMap::new(),
16108 winner: 'A',
16109 rankings: 1,
16110 unanimous_initial: true,
16111 deliberated: false,
16112 changed_votes: 0,
16113 unanimous_final: true,
16114 tie_break: None,
16115 judges: 0,
16116 present: 0,
16117 quorum: 0,
16118 met_quorum: true,
16119 uncontested: Some("only candidate A produced a change".to_owned()),
16120 });
16121 state.reviews = vec![ReviewRound {
16122 round: 1,
16123 head: "deadbeef".to_owned(),
16124 verified_head: None,
16125 verified_at: None,
16126 reviews: Vec::new(),
16127 e2e: Vec::new(),
16128 fix: None,
16129 blocking: 0,
16130 answered: 0,
16131 expected: 0,
16132 clean: true,
16133 verify_retried: false,
16134 e2e_deferred: false,
16135 e2e_defer_reason: None,
16136 progressed: false,
16137 vote_split: false,
16138 reconsideration: Vec::new(),
16139 verdict: None,
16140 }];
16141
16142 let mut runner = Runner {
16143 state,
16144 roles: ResolvedRoles {
16145 implementers: Vec::new(),
16146 judges: Vec::new(),
16147 reviewers: Vec::new(),
16148 fixer: None,
16149 conductor: conductor(),
16150 implementer_roster: Vec::new(),
16151 judge_roster: Vec::new(),
16152 reviewer_roster: Vec::new(),
16153 },
16154 sem: Arc::new(Semaphore::new(1)),
16155 pause: Pause::new(),
16156 interrupt: Pause::new(),
16157 };
16158
16159 runner.gate().await.expect("gate");
16160 assert!(
16161 runner.state.gate_ran,
16162 "zero configured commands is still a real attempt, not an unrun gate"
16163 );
16164 assert!(runner.state.gate.is_empty());
16165 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
16166 assert_ne!(
16167 runner.state.status,
16168 RunStatus::Blocked,
16169 "a gate with nothing to check must not read as failed"
16170 );
16171
16172 runner.merge().await.expect("merge");
16173 assert_eq!(
16174 runner.state.status,
16175 RunStatus::Ready,
16176 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
16177 );
16178 }
16179
16180 #[tokio::test]
16191 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
16192 crate::run::pin_test_home();
16193 let home = crate::run::home();
16194
16195 let tmp = tempfile::tempdir().expect("tempdir");
16196 let repo = tmp.path().join("repo");
16197 std::fs::create_dir_all(&repo).unwrap();
16198 init_repo(&repo);
16199 let cache_dir = tmp.path().join("target");
16202
16203 let mut config = Config::default();
16204 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
16205 config.graph.timeout_verify = Some(2);
16208 config.graph.timeout_resource_wait = 2;
16209
16210 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16211 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16212 .expect("no io error acquiring directly")
16213 {
16214 crate::cache::AcquireOutcome::Acquired(g) => g,
16215 crate::cache::AcquireOutcome::Busy(b) => {
16216 panic!("expected the direct acquire to win the lease first: {b:?}")
16217 }
16218 };
16219
16220 let mut state = RunState::new(
16221 repo.clone(),
16222 "main".to_owned(),
16223 "deadbeef".to_owned(),
16224 "task".to_owned(),
16225 config,
16226 );
16227 state.candidates = vec![Candidate {
16228 index: 0,
16229 label: 'A',
16230 agent: "alpha".to_owned(),
16231 branch: "does-not-exist".to_owned(),
16232 worktree: repo.clone(),
16233 summary: String::new(),
16234 stat: String::new(),
16235 files: 0,
16236 commits: 0,
16237 empty: false,
16238 failed: None,
16239 verified_noop: None,
16240 duration_ms: 0,
16241 folded: false,
16242 }];
16243 state.tally = Some(Tally {
16244 first_choice: BTreeMap::from([('A', 1)]),
16245 borda: BTreeMap::new(),
16246 winner: 'A',
16247 rankings: 1,
16248 unanimous_initial: true,
16249 deliberated: false,
16250 changed_votes: 0,
16251 unanimous_final: true,
16252 tie_break: None,
16253 judges: 0,
16254 present: 0,
16255 quorum: 0,
16256 met_quorum: true,
16257 uncontested: Some("only candidate A produced a change".to_owned()),
16258 });
16259 state.reviews = vec![ReviewRound {
16260 round: 1,
16261 head: "deadbeef".to_owned(),
16262 verified_head: None,
16263 verified_at: None,
16264 reviews: Vec::new(),
16265 e2e: Vec::new(),
16266 fix: None,
16267 blocking: 0,
16268 answered: 0,
16269 expected: 0,
16270 clean: true,
16271 verify_retried: false,
16272 e2e_deferred: false,
16273 e2e_defer_reason: None,
16274 progressed: false,
16275 vote_split: false,
16276 reconsideration: Vec::new(),
16277 verdict: None,
16278 }];
16279
16280 let mut runner = Runner {
16281 state,
16282 roles: ResolvedRoles {
16283 implementers: Vec::new(),
16284 judges: Vec::new(),
16285 reviewers: Vec::new(),
16286 fixer: None,
16287 conductor: conductor(),
16288 implementer_roster: Vec::new(),
16289 judge_roster: Vec::new(),
16290 reviewer_roster: Vec::new(),
16291 },
16292 sem: Arc::new(Semaphore::new(1)),
16293 pause: Pause::new(),
16294 interrupt: Pause::new(),
16295 };
16296
16297 let started = std::time::Instant::now();
16298 runner.gate().await.expect("gate");
16299 assert!(
16300 started.elapsed() < Duration::from_secs(1),
16301 "a gate with nothing to run must never wait on a lease it never needed"
16302 );
16303 assert!(
16304 runner.state.gate_ran,
16305 "zero commands is still a real, immediate attempt"
16306 );
16307 assert!(runner.state.gate.is_empty());
16308 assert_ne!(
16309 runner.state.status,
16310 RunStatus::Blocked,
16311 "must not read as resource-blocked on a lease it never asked for"
16312 );
16313 }
16314
16315 #[cfg(unix)]
16318 #[tokio::test]
16319 async fn verification_is_standalone_inside_a_live_worker() {
16320 const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
16321 if std::env::var_os(CHILD).is_none() {
16322 let home = tempfile::tempdir().unwrap();
16323 let generation = crate::worker::pin_current(home.path()).unwrap();
16324 let output = std::process::Command::new(std::env::current_exe().unwrap())
16325 .args([
16326 "--exact",
16327 "graph::tests::verification_is_standalone_inside_a_live_worker",
16328 "--nocapture",
16329 ])
16330 .env(CHILD, "1")
16331 .env("MAGI_HOME", home.path())
16332 .env("MAGI_WORKER_HOME", home.path())
16333 .env("MAGI_WORKER_ID", "abcdef")
16334 .env("MAGI_WORKER_GENERATION", &generation.id)
16335 .env("MAGI_WORKER_EXE", &generation.executable)
16336 .env("MAGI_RUN", "verification-parent")
16337 .env("MAGI_NODE", "gate")
16338 .env("MAGI_SEAT", "worker-fixture")
16339 .env("MAGI_TURN_SCOPE", "fixture-scope")
16340 .env("MAGI_PROCESS_OWNER", "fixture-owner")
16341 .env("MAGI_NO_AUTOUPDATE", "0")
16342 .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
16343 .env("VERIFY_AUTH_TOKEN", "fixture-auth")
16344 .env(
16345 "VERIFY_EXPECT_TOOLCHAIN",
16346 std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
16347 )
16348 .env(
16349 "VERIFY_EXPECT_CACHE",
16350 std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
16351 )
16352 .output()
16353 .unwrap();
16354 assert!(
16355 output.status.success()
16356 && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16357 "{}\n{}",
16358 String::from_utf8_lossy(&output.stdout),
16359 String::from_utf8_lossy(&output.stderr)
16360 );
16361 return;
16362 }
16363 let home = crate::run::home();
16364 let directory = home.join("workers/jobs/abcdef");
16365 std::fs::create_dir_all(&directory).unwrap();
16366 let job = crate::worker::Job {
16367 id: "abcdef".into(),
16368 sequence: 1,
16369 kind: "task".into(),
16370 key: "task:verification".into(),
16371 payload: serde_json::json!({}),
16372 generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
16373 version: Some(env!("CARGO_PKG_VERSION").into()),
16374 status: crate::worker::Status::Running,
16375 result: None,
16376 pid: Some(std::process::id()),
16377 created_at: 0,
16378 finished_at: None,
16379 };
16380 std::fs::write(
16381 directory.join("job.json"),
16382 serde_json::to_vec(&job).unwrap(),
16383 )
16384 .unwrap();
16385 let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
16386 let assert_worker = || {
16387 assert_eq!(
16388 crate::worker::current_job(&home).unwrap().unwrap().id,
16389 job.id
16390 );
16391 assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
16392 assert_eq!(
16393 crate::worker::helper_executable(&home).unwrap(),
16394 PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
16395 );
16396 assert!(
16397 crate::updater::Checker::new(&crate::config::Update {
16398 mode: crate::config::UpdateMode::Notify,
16399 interval: None,
16400 auto_restart: false,
16401 })
16402 .is_none(),
16403 "the driver remains pinned, even while tests are standalone"
16404 );
16405 };
16406 assert_worker();
16407 let repo = home.join("repo");
16408 std::fs::create_dir_all(&repo).unwrap();
16409 init_repo(&repo);
16410 let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
16413 test -z "${MAGI_WORKER_GENERATION+x}" &&
16414 test -z "${MAGI_WORKER_HOME+x}" &&
16415 test -z "${MAGI_WORKER_EXE+x}" &&
16416 test "$MAGI_HOME/repo" -ef . &&
16417 test "$MAGI_RUN" = verification-parent &&
16418 test "$MAGI_NODE" = gate &&
16419 test "$MAGI_SEAT" = worker-fixture &&
16420 test "$MAGI_TURN_SCOPE" = fixture-scope &&
16421 test -n "$MAGI_PROCESS_OWNER" &&
16422 test "$PATH" = "$VERIFY_EXPECT_PATH" &&
16423 test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
16424 test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
16425 test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
16426 assert!(
16428 !std::process::Command::new("sh")
16429 .args(["-c", probe])
16430 .current_dir(&repo)
16431 .status()
16432 .unwrap()
16433 .success()
16434 );
16435 let unrelated_home = tempfile::tempdir().unwrap();
16436 assert!(
16437 crate::worker::current_job(unrelated_home.path()).is_err(),
16438 "an inherited worker id cannot be resolved in a unit fixture's home"
16439 );
16440 let mut runner = runner_at(RunStatus::Gating);
16441 runner.state.repo = repo.clone();
16442 runner.state.config.disk.min_free_bytes = 0;
16443 runner.state.config.coordination = crate::config::test_coordination();
16444 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16445 runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
16446 runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
16447 runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
16448 let winner = Candidate {
16449 index: 0,
16450 label: 'A',
16451 agent: "fixture".into(),
16452 branch: "main".into(),
16453 worktree: repo.clone(),
16454 summary: String::new(),
16455 stat: String::new(),
16456 files: 0,
16457 commits: 0,
16458 empty: false,
16459 failed: None,
16460 verified_noop: None,
16461 duration_ms: 0,
16462 folded: false,
16463 };
16464 let (e2e, retried, pids) = run_e2e_with_retry(
16465 &mut runner.state,
16466 &["sh".into(), "-c".into()],
16467 &[format!("{probe} && printf initial")],
16468 &repo,
16469 Duration::from_secs(10),
16470 "initial",
16471 )
16472 .await;
16473 assert!(!retried && pids.is_empty());
16474 assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
16475 assert_eq!(e2e[0].output_tail, "initial");
16476 let gate = runner.run_gate(&winner).await.unwrap();
16477 assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
16478 runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
16481 head: git::rev_parse(&repo, "HEAD").await.unwrap(),
16482 passed: false,
16483 outcomes: Vec::new(),
16484 });
16485 runner.state.save().unwrap();
16486 let mut resumed = RunState::load(&runner.state.id).unwrap();
16487 let wait = crate::repo_lock::WaitPolicy::default();
16488 assert!(matches!(
16489 check_gate_repair_e2e(&mut resumed, &winner, &wait)
16490 .await
16491 .unwrap(),
16492 GateFix::Retry
16493 ));
16494 let saved = RunState::load(&resumed.id).unwrap();
16495 let repair = saved.gate_repair_e2e.unwrap();
16496 assert!(repair.passed);
16497 assert_eq!(repair.outcomes.len(), 1);
16498 assert!(repair.outcomes[0].ok());
16499 let landing = run_landing_gate(&mut resumed, &winner, &wait)
16500 .await
16501 .unwrap();
16502 assert_eq!(landing.len(), 1);
16503 assert!(landing[0].ok(), "{landing:?}");
16504 assert_eq!(
16505 std::fs::read_to_string(repo.join("phases")).unwrap(),
16506 "pre\ngate\ne2e\npre\ngate\n"
16507 );
16508 assert_worker();
16509 }
16510
16511 #[cfg(unix)]
16512 #[tokio::test]
16513 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
16514 crate::run::pin_test_home();
16515 let tmp = tempfile::tempdir().unwrap();
16516 let mut config = Config::default();
16517 config.disk.min_free_bytes = 0;
16518 let mut state = RunState::new(
16519 tmp.path().to_owned(),
16520 "main".into(),
16521 "base".into(),
16522 "verification evidence".into(),
16523 config,
16524 );
16525 let (outcomes, pids) = run_commands(
16526 &mut state,
16527 "gate",
16528 "gate",
16529 1,
16530 &["sh".into(), "-c".into()],
16531 &[
16532 "printf before-timeout; exec sleep 30".into(),
16533 "kill -TERM $$".into(),
16534 "exit 7".into(),
16535 "printf success".into(),
16536 ],
16537 tmp.path(),
16538 Duration::from_secs(1),
16539 )
16540 .await;
16541 wait_for_timed_out_children_to_die(&pids).await;
16542 assert_eq!(
16543 pids.len(),
16544 1,
16545 "unleased timeout retains its death-check PID"
16546 );
16547 assert_eq!(outcomes.len(), 4);
16548 assert_eq!(outcomes[0].code, None);
16549 assert!(outcomes[0].output_tail.contains("before-timeout"));
16550 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
16551 assert_eq!(outcomes[1].code, None);
16552 assert!(outcomes[1].output_tail.contains("command terminated:"));
16553 assert!(!outcomes[1].output_tail.contains("timed out"));
16554 assert_eq!(outcomes[2].code, Some(7));
16555 assert!(!outcomes[2].output_tail.contains("timed out"));
16556 assert_eq!(outcomes[3].code, Some(0));
16557 assert_eq!(outcomes[3].output_tail, "success");
16558 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
16559 }
16560
16561 #[tokio::test]
16571 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
16572 crate::run::pin_test_home();
16573
16574 let tmp = tempfile::tempdir().expect("tempdir");
16575 let repo = tmp.path().join("repo");
16576 std::fs::create_dir_all(&repo).unwrap();
16577 init_repo(&repo);
16578
16579 let mut config = Config::default();
16580 config.verify.gate = vec![
16581 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
16582 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
16583 .to_owned(),
16584 ];
16585
16586 let mut state = RunState::new(
16587 repo.clone(),
16588 "main".to_owned(),
16589 "deadbeef".to_owned(),
16590 "task".to_owned(),
16591 config,
16592 );
16593 let run_id = state.id.clone();
16594 state.candidates = vec![Candidate {
16595 index: 0,
16596 label: 'A',
16597 agent: "alpha".to_owned(),
16598 branch: "does-not-exist".to_owned(),
16599 worktree: repo.clone(),
16600 summary: String::new(),
16601 stat: String::new(),
16602 files: 0,
16603 commits: 0,
16604 empty: false,
16605 failed: None,
16606 verified_noop: None,
16607 duration_ms: 0,
16608 folded: false,
16609 }];
16610 state.tally = Some(Tally {
16611 first_choice: BTreeMap::from([('A', 1)]),
16612 borda: BTreeMap::new(),
16613 winner: 'A',
16614 rankings: 1,
16615 unanimous_initial: true,
16616 deliberated: false,
16617 changed_votes: 0,
16618 unanimous_final: true,
16619 tie_break: None,
16620 judges: 0,
16621 present: 0,
16622 quorum: 0,
16623 met_quorum: true,
16624 uncontested: Some("only candidate A produced a change".to_owned()),
16625 });
16626 state.reviews = vec![ReviewRound {
16627 round: 1,
16628 head: "deadbeef".to_owned(),
16629 verified_head: None,
16630 verified_at: None,
16631 reviews: Vec::new(),
16632 e2e: Vec::new(),
16633 fix: None,
16634 blocking: 0,
16635 answered: 0,
16636 expected: 0,
16637 clean: true,
16638 verify_retried: false,
16639 e2e_deferred: false,
16640 e2e_defer_reason: None,
16641 progressed: false,
16642 vote_split: false,
16643 reconsideration: Vec::new(),
16644 verdict: None,
16645 }];
16646
16647 let mut runner = Runner {
16648 state,
16649 roles: ResolvedRoles {
16650 implementers: Vec::new(),
16651 judges: Vec::new(),
16652 reviewers: Vec::new(),
16653 fixer: None,
16654 conductor: conductor(),
16655 implementer_roster: Vec::new(),
16656 judge_roster: Vec::new(),
16657 reviewer_roster: Vec::new(),
16658 },
16659 sem: Arc::new(Semaphore::new(1)),
16660 pause: Pause::new(),
16661 interrupt: Pause::new(),
16662 };
16663
16664 let started_marker = repo.join("started.marker");
16665 let release_marker = repo.join("release.marker");
16666 let poller = tokio::spawn(async move {
16667 for _ in 0..100 {
16672 if started_marker.exists()
16673 && let Ok(s) = crate::run::RunState::load(&run_id)
16674 && let Some(a) = s.active.get("gate")
16675 {
16676 std::fs::write(&release_marker, b"go").expect("release marker");
16677 return Some(a.clone());
16678 }
16679 tokio::time::sleep(Duration::from_millis(50)).await;
16680 }
16681 None
16682 });
16683
16684 runner.gate().await.expect("gate");
16685 let captured = poller.await.expect("poller task");
16686 let captured = captured.expect(
16687 "the poller never saw a `gate` task entry in run.json while the command was \
16688 still blocked on its own release marker",
16689 );
16690
16691 assert_eq!(captured.task.as_deref(), Some("gate"));
16692 assert_eq!(captured.node, "gate");
16693 assert_eq!(captured.index, Some(1));
16694 assert_eq!(captured.total, Some(1));
16695 assert!(
16696 captured
16697 .command
16698 .as_deref()
16699 .is_some_and(|c| c.contains("started.marker")),
16700 "{captured:?}"
16701 );
16702
16703 assert!(
16704 runner.state.active.is_empty(),
16705 "the entry must be cleared once the command actually finished: {:?}",
16706 runner.state.active
16707 );
16708 assert!(runner.state.gate_ran);
16709 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16710 }
16711
16712 #[tokio::test]
16715 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16716 use crate::verdict::{Finding, ReviewVote, Severity};
16717 crate::run::pin_test_home();
16718 let tmp = tempfile::tempdir().expect("tempdir");
16719 let repo = tmp.path().join("repo");
16720 std::fs::create_dir_all(&repo).unwrap();
16721 init_repo(&repo);
16722
16723 for (severity, vote, expect) in [
16724 (Severity::Major, ReviewVote::Reject, true),
16725 (Severity::Minor, ReviewVote::Reject, false),
16726 (Severity::Major, ReviewVote::Approve, false),
16727 ] {
16728 let mut round = review_round(false, 1, 1, 1, false, true);
16729 round.reviews = vec![ReviewRecord {
16730 reviewer: 1,
16731 agent: "alpha".to_owned(),
16732 summary: String::new(),
16733 findings: vec![Finding {
16734 id: "R1-1-1".to_owned(),
16735 severity,
16736 file: None,
16737 line: None,
16738 title: "t".to_owned(),
16739 detail: String::new(),
16740 }],
16741 vote: Some(vote),
16742 failed: None,
16743 duration_ms: 0,
16744 attempts: 0,
16745 }];
16746 let mut state = RunState::new(
16747 repo.clone(),
16748 "main".to_owned(),
16749 "deadbeef".to_owned(),
16750 "task".to_owned(),
16751 Config::default(),
16752 );
16753 state.reviews = vec![round];
16754 let mut runner = Runner {
16755 state,
16756 roles: ResolvedRoles {
16757 implementers: Vec::new(),
16758 judges: Vec::new(),
16759 reviewers: Vec::new(),
16760 fixer: None,
16761 conductor: conductor(),
16762 implementer_roster: Vec::new(),
16763 judge_roster: Vec::new(),
16764 reviewer_roster: Vec::new(),
16765 },
16766 sem: Arc::new(Semaphore::new(1)),
16767 pause: Pause::new(),
16768 interrupt: Pause::new(),
16769 };
16770 let shell = runner.state.config.shell();
16771 runner
16772 .stop_reviewing("round budget spent", &shell, &repo)
16773 .await
16774 .expect("stop_reviewing");
16775 assert_eq!(runner.state.status, RunStatus::Gating);
16776 assert_eq!(
16777 runner.state.contested_handoff.is_some(),
16778 expect,
16779 "{severity:?} + {vote:?}"
16780 );
16781 }
16782 }
16783
16784 #[tokio::test]
16797 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16798 crate::run::pin_test_home();
16799 let home = crate::run::home();
16800
16801 let tmp = tempfile::tempdir().expect("tempdir");
16802 let repo = tmp.path().join("repo");
16803 std::fs::create_dir_all(&repo).unwrap();
16804 init_repo(&repo);
16805 let head = crate::git::rev_parse(&repo, "HEAD")
16806 .await
16807 .expect("rev-parse");
16808 let cache_dir = tmp.path().join("target");
16811
16812 let mut config = Config::default();
16813 config.verify.e2e = vec![format!(
16814 "CARGO_TARGET_DIR='{}' test -f README.md",
16815 cache_dir.display()
16816 )];
16817 config.graph.review_rounds = 1;
16818 config.graph.timeout_verify = Some(2);
16821 config.graph.timeout_resource_wait = 2;
16822
16823 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16824 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16825 .expect("no io error acquiring directly")
16826 {
16827 crate::cache::AcquireOutcome::Acquired(g) => g,
16828 crate::cache::AcquireOutcome::Busy(b) => {
16829 panic!("expected the direct acquire to win the lease first: {b:?}")
16830 }
16831 };
16832
16833 let mut state = RunState::new(
16834 repo.clone(),
16835 "main".to_owned(),
16836 head.clone(),
16837 "task".to_owned(),
16838 config,
16839 );
16840 state.candidates = vec![Candidate {
16841 index: 0,
16842 label: 'A',
16843 agent: "alpha".to_owned(),
16844 branch: "does-not-exist".to_owned(),
16845 worktree: repo.clone(),
16846 summary: String::new(),
16847 stat: String::new(),
16848 files: 0,
16849 commits: 0,
16850 empty: false,
16851 failed: None,
16852 verified_noop: None,
16853 duration_ms: 0,
16854 folded: false,
16855 }];
16856 state.tally = Some(Tally {
16857 first_choice: BTreeMap::from([('A', 1)]),
16858 borda: BTreeMap::new(),
16859 winner: 'A',
16860 rankings: 1,
16861 unanimous_initial: true,
16862 deliberated: false,
16863 changed_votes: 0,
16864 unanimous_final: true,
16865 tie_break: None,
16866 judges: 0,
16867 present: 0,
16868 quorum: 0,
16869 met_quorum: true,
16870 uncontested: Some("only candidate A produced a change".to_owned()),
16871 });
16872 state.reviews = vec![ReviewRound {
16876 round: 1,
16877 head: head.clone(),
16878 verified_head: None,
16879 verified_at: None,
16880 reviews: Vec::new(),
16881 e2e: Vec::new(),
16882 fix: None,
16883 blocking: 1,
16884 answered: 1,
16885 expected: 1,
16886 clean: false,
16887 verify_retried: false,
16888 e2e_deferred: true,
16889 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16890 progressed: false,
16891 vote_split: false,
16892 reconsideration: Vec::new(),
16893 verdict: None,
16894 }];
16895
16896 let mut runner = Runner {
16897 state,
16898 roles: ResolvedRoles {
16899 implementers: Vec::new(),
16900 judges: Vec::new(),
16901 reviewers: Vec::new(),
16902 fixer: None,
16903 conductor: conductor(),
16904 implementer_roster: Vec::new(),
16905 judge_roster: Vec::new(),
16906 reviewer_roster: Vec::new(),
16907 },
16908 sem: Arc::new(Semaphore::new(1)),
16909 pause: Pause::new(),
16910 interrupt: Pause::new(),
16911 };
16912
16913 let shell = runner.state.config.shell();
16914 runner
16915 .stop_reviewing("round budget spent", &shell, &repo)
16916 .await
16917 .expect("stop_reviewing");
16918
16919 let last = runner.state.reviews.last().expect("round record");
16920 assert_eq!(
16921 last.e2e_status(),
16922 E2eStatus::ResourceBlocked,
16923 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16924 failed: {last:?}"
16925 );
16926 assert_eq!(
16927 last.verified_head.as_deref(),
16928 Some(head.as_str()),
16929 "which commit this attempt targeted is known even though nothing finished checking \
16930 it"
16931 );
16932 let first_attempt_at = last
16933 .verified_at
16934 .expect("when this attempt ran is known too");
16935 assert_ne!(
16936 runner.state.status,
16937 RunStatus::Blocked,
16938 "contention is evidence about the machine, not the patch — it must not settle the \
16939 run as blocked: {:?}",
16940 runner.state.status
16941 );
16942 assert!(
16943 !runner
16944 .state
16945 .events
16946 .iter()
16947 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16948 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16949 runner.state.events
16950 );
16951
16952 runner
16957 .stop_reviewing("round budget spent", &shell, &repo)
16958 .await
16959 .expect("stop_reviewing retry");
16960 assert_eq!(
16961 runner.state.reviews.len(),
16962 1,
16963 "no new round was started: {:?}",
16964 runner.state.reviews
16965 );
16966 let last = runner.state.reviews.last().expect("round record");
16967 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16968 assert!(
16969 last.verified_at.expect("still known") > first_attempt_at,
16970 "a second reentry must be a fresh attempt, not a stale copy of the first"
16971 );
16972 assert_ne!(runner.state.status, RunStatus::Blocked);
16973
16974 held.release();
16975 }
16976
16977 #[cfg(unix)]
16978 #[tokio::test]
16979 async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
16980 ask_test_home();
16981 for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
16982 let tmp = tempfile::tempdir().unwrap();
16983 let repo = tmp.path().join("repo");
16984 std::fs::create_dir(&repo).unwrap();
16985 init_repo(&repo);
16986 let run_git = |args: &[&str]| {
16987 let out = std::process::Command::new("git")
16988 .args(args)
16989 .current_dir(&repo)
16990 .output()
16991 .unwrap();
16992 assert!(
16993 out.status.success(),
16994 "{args:?}: {}",
16995 String::from_utf8_lossy(&out.stderr)
16996 );
16997 };
16998 let origin = tmp.path().join("origin.git");
16999 run_git(&["init", "--bare", origin.to_str().unwrap()]);
17000 run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
17001 std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
17002 run_git(&["add", "spec.rs"]);
17003 run_git(&["commit", "-m", "initial spec"]);
17004 run_git(&["push", "origin", "main"]);
17005 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17006 run_git(&["checkout", "-b", "fix/review"]);
17007 std::fs::write(
17008 repo.join("check.rs"),
17009 "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
17010 )
17011 .unwrap();
17012 run_git(&["add", "check.rs"]);
17013 run_git(&["commit", "-m", "use spec"]);
17014 let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
17015 run_git(&["checkout", "main"]);
17016 std::fs::write(
17017 repo.join("spec.rs"),
17018 "struct AgentSpec { max_turn_secs: u64 }\n",
17019 )
17020 .unwrap();
17021 run_git(&["add", "spec.rs"]);
17022 run_git(&["commit", "-m", "extend spec"]);
17023 run_git(&["push", "origin", "main"]);
17024 run_git(&["checkout", "fix/review"]);
17025
17026 let mut runner = runner_at(RunStatus::Reviewing);
17027 runner.state.repo = repo.clone();
17028 runner.state.base_commit = base;
17029 runner.state.pr_head = Some("fix/review".into());
17030 runner.state.config.graph.review_rounds = max_rounds;
17031 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17032 runner.state.config.graph.sessions = false;
17033 runner.state.config.disk.min_free_bytes = 0;
17034 runner.state.config.verify.e2e =
17035 vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
17036 runner.state.candidates = vec![Candidate {
17037 index: 0,
17038 label: 'A',
17039 agent: "fixture".into(),
17040 branch: "fix/review".into(),
17041 worktree: repo.clone(),
17042 summary: String::new(),
17043 stat: String::new(),
17044 files: 1,
17045 commits: 1,
17046 empty: false,
17047 failed: None,
17048 verified_noop: None,
17049 duration_ms: 0,
17050 folded: false,
17051 }];
17052 runner.state.tally = Some(Tally {
17053 first_choice: BTreeMap::new(),
17054 borda: BTreeMap::new(),
17055 winner: 'A',
17056 rankings: 0,
17057 unanimous_initial: true,
17058 deliberated: false,
17059 changed_votes: 0,
17060 unanimous_final: true,
17061 tie_break: None,
17062 judges: 0,
17063 present: 0,
17064 quorum: 0,
17065 met_quorum: true,
17066 uncontested: Some("review-only".into()),
17067 });
17068 let mut last = review_round(false, 0, 1, 1, false, false);
17069 last.head = reviewed.clone();
17070 last.e2e_deferred = true;
17071 last.e2e[0].resource_blocked = true;
17072 last.e2e[0].output_tail = "waiting for the shared build cache".into();
17073 if max_rounds == 1 {
17074 last.e2e.clear();
17075 }
17076 runner.state.reviews = vec![last];
17077 let calls = tmp.path().join("fix-calls");
17078 let mut fixer = spec("repair");
17079 fixer
17080 .env
17081 .insert("FIX_CALLS".into(), calls.display().to_string());
17082 fixer.command = vec!["sh".into(), "-c".into(), r#"
17083 echo fix >> "$FIX_CALLS"
17084 grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
17085 grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
17086 printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
17087 printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
17088 "#.into()];
17089 let mut reviewer = spec("reviewer");
17090 reviewer.command = vec![
17091 "sh".into(),
17092 "-c".into(),
17093 "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
17094 .into(),
17095 ];
17096 runner.roles.fixer = Some(vec![fixer]);
17097 runner.roles.reviewers = vec![reviewer];
17098
17099 runner.sync_to_base().await.unwrap();
17100 let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
17101 assert_ne!(rebased, reviewed);
17102 assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
17103 if reentry {
17104 let shell = runner.state.config.shell();
17107 runner
17108 .stop_unresolved_review("deferred verification", &shell, &repo)
17109 .await
17110 .unwrap();
17111 assert!(runner.catchup_repairable());
17112 assert!(!calls.exists());
17113 runner.state = RunState::load(&runner.state.id).unwrap();
17114 assert!(runner.catchup_repairable());
17115 }
17116 runner.review_loop().await.unwrap();
17117 let first = &runner.state.reviews[0];
17118 assert_eq!(first.head, reviewed);
17119 assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
17120 assert!(
17121 first.verify_retried,
17122 "a reproduced build failure gets exactly one retry: {:?}",
17123 first.e2e
17124 );
17125 assert!(first.e2e[0].output_tail.contains("E0063"));
17126 assert!(!runner.catchup_repairable());
17127 if max_rounds == 2 {
17128 assert_eq!(
17129 runner.state.status,
17130 RunStatus::Gating,
17131 "{}",
17132 crate::report::run(&runner.state)
17133 );
17134 assert_eq!(runner.state.reviews.len(), 2);
17135 assert!(first.fix.as_ref().unwrap().committed);
17136 assert!(runner.state.reviews[1].clean);
17137 assert_eq!(
17138 runner
17139 .state
17140 .reviews
17141 .iter()
17142 .filter(|r| r.fix.is_some())
17143 .count(),
17144 1
17145 );
17146 assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
17147 assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
17148 } else {
17149 assert_eq!(runner.state.status, RunStatus::Blocked);
17150 assert!(first.fix.is_none());
17151 assert!(!calls.exists());
17152 let recovery = crate::recovery::assess(
17153 "prior",
17154 Some(&runner.state),
17155 &runner.landing_base(),
17156 crate::recovery::Relation::Diverged,
17157 );
17158 assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
17159 let reasons = recovery.reasons.join("\n");
17160 assert!(
17161 reasons.contains(&reviewed) && reasons.contains(&rebased),
17162 "{reasons}"
17163 );
17164 assert!(
17165 reasons.contains("base sync to")
17166 && reasons.contains("additional review is required"),
17167 "{reasons}"
17168 );
17169 }
17170 }
17171 }
17172
17173 #[tokio::test]
17185 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
17186 crate::run::pin_test_home();
17187 let home = crate::run::home();
17188
17189 let tmp = tempfile::tempdir().expect("tempdir");
17190 let repo = tmp.path().join("repo");
17191 std::fs::create_dir_all(&repo).unwrap();
17192 init_repo(&repo);
17193 let head = crate::git::rev_parse(&repo, "HEAD")
17194 .await
17195 .expect("rev-parse");
17196 let cache_dir = tmp.path().join("target");
17197
17198 let mut config = Config::default();
17199 config.verify.e2e = vec![format!(
17200 "CARGO_TARGET_DIR='{}' test -f README.md",
17201 cache_dir.display()
17202 )];
17203 config.graph.review_rounds = 1;
17204 config.graph.timeout_verify = Some(2);
17205 config.graph.timeout_resource_wait = 2;
17206
17207 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
17208 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
17209 .expect("no io error acquiring directly")
17210 {
17211 crate::cache::AcquireOutcome::Acquired(g) => g,
17212 crate::cache::AcquireOutcome::Busy(b) => {
17213 panic!("expected the direct acquire to win the lease first: {b:?}")
17214 }
17215 };
17216
17217 let mut state = RunState::new(
17218 repo.clone(),
17219 "main".to_owned(),
17220 head.clone(),
17221 "task".to_owned(),
17222 config,
17223 );
17224 state.candidates = vec![Candidate {
17225 index: 0,
17226 label: 'A',
17227 agent: "alpha".to_owned(),
17228 branch: "does-not-exist".to_owned(),
17229 worktree: repo.clone(),
17230 summary: String::new(),
17231 stat: String::new(),
17232 files: 0,
17233 commits: 0,
17234 empty: false,
17235 failed: None,
17236 verified_noop: None,
17237 duration_ms: 0,
17238 folded: false,
17239 }];
17240 state.tally = Some(Tally {
17241 first_choice: BTreeMap::from([('A', 1)]),
17242 borda: BTreeMap::new(),
17243 winner: 'A',
17244 rankings: 1,
17245 unanimous_initial: true,
17246 deliberated: false,
17247 changed_votes: 0,
17248 unanimous_final: true,
17249 tie_break: None,
17250 judges: 0,
17251 present: 0,
17252 quorum: 0,
17253 met_quorum: true,
17254 uncontested: Some("only candidate A produced a change".to_owned()),
17255 });
17256 state.reviews = vec![ReviewRound {
17260 round: 1,
17261 head: head.clone(),
17262 verified_head: Some(head.clone()),
17263 verified_at: Some(jiff::Timestamp::now()),
17264 reviews: Vec::new(),
17265 e2e: vec![CommandOutcome {
17266 command: format!(
17267 "CARGO_TARGET_DIR='{}' test -f README.md",
17268 cache_dir.display()
17269 ),
17270 code: None,
17271 output_tail: "waiting for the shared build cache".to_owned(),
17272 duration_ms: 0,
17273 resource_blocked: true,
17274 }],
17275 fix: None,
17276 blocking: 1,
17277 answered: 1,
17278 expected: 1,
17279 clean: false,
17280 verify_retried: false,
17281 e2e_deferred: false,
17282 e2e_defer_reason: None,
17283 progressed: false,
17284 vote_split: false,
17285 reconsideration: Vec::new(),
17286 verdict: None,
17287 }];
17288
17289 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
17290 let mut runner = Runner {
17291 state,
17292 roles: ResolvedRoles {
17293 implementers: Vec::new(),
17294 judges: Vec::new(),
17295 reviewers: Vec::new(),
17296 fixer: None,
17297 conductor: conductor(),
17298 implementer_roster: Vec::new(),
17299 judge_roster: Vec::new(),
17300 reviewer_roster: Vec::new(),
17301 },
17302 sem: Arc::new(Semaphore::new(1)),
17303 pause: Pause::new(),
17304 interrupt: Pause::new(),
17305 };
17306
17307 runner.review_loop().await.expect("review_loop");
17312
17313 assert_eq!(
17314 runner.state.reviews.len(),
17315 1,
17316 "no new round was started on top of the unresolved one: {:?}",
17317 runner.state.reviews
17318 );
17319 let last = &runner.state.reviews[0];
17320 assert_eq!(
17321 last.e2e_status(),
17322 E2eStatus::ResourceBlocked,
17323 "still contended: {last:?}"
17324 );
17325 assert!(
17326 last.verified_at.expect("still known") > first_attempt_at,
17327 "review_loop must have actually retried the check, not left it exactly as found"
17328 );
17329 assert_ne!(
17330 runner.state.status,
17331 RunStatus::Blocked,
17332 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
17333 runner.state.status
17334 );
17335
17336 held.release();
17337 }
17338
17339 #[tokio::test]
17340 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
17341 crate::run::pin_test_home();
17342 for status in [RunStatus::Blocked, RunStatus::Gating] {
17343 let mut runner = runner_at(status);
17344 runner.state.config.graph.land = true;
17345 runner.state.gate_ran = true;
17346 runner.state.merge = Some(MergeOutcome {
17347 mode: MergeMode::Pr,
17348 ok: false,
17349 detail: "the fixer produced no commit".into(),
17350 empty: false,
17351 });
17352 let url = "https://example.invalid/o/r/pull/1";
17353 runner.state.pr = Some(crate::run::PrRecord {
17354 url: url.into(),
17355 number: 1,
17356 state: "open".into(),
17357 checks: "red".into(),
17358 round: 1,
17359 rounds: 1,
17360 red_at_merge: Vec::new(),
17361 });
17362 runner.state.save().unwrap();
17363 runner.execute_graph().await.unwrap();
17367 assert_eq!(runner.state.status, RunStatus::Blocked);
17368 assert!(runner.state.land_sync_pending);
17369 assert!(runner.state.events.iter().any(|event|
17370 event.node == "land" && event.message == format!("watching {url}")));
17371 assert_eq!(
17372 runner.state.merge.as_ref().unwrap().detail,
17373 "the fixer produced no commit"
17374 );
17375 }
17376 }
17377
17378 #[tokio::test]
17379 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
17380 crate::run::pin_test_home();
17381 let tmp = tempfile::tempdir().expect("tempdir");
17382 let repo = tmp.path().join("repo");
17383 std::fs::create_dir_all(&repo).unwrap();
17384 init_repo(&repo);
17385
17386 let mut config = Config::default();
17387 config.merge.mode = MergeMode::Pr;
17388 config.graph.land = true;
17389 config.graph.land_approval = false;
17390
17391 let mut state = RunState::new(
17392 repo.clone(),
17393 "main".to_owned(),
17394 "deadbeef".to_owned(),
17395 "task".to_owned(),
17396 config,
17397 );
17398 state.candidates = vec![Candidate {
17399 index: 0,
17400 label: 'A',
17401 agent: "alpha".to_owned(),
17402 branch: "does-not-exist".to_owned(),
17403 worktree: repo.clone(),
17404 summary: String::new(),
17405 stat: String::new(),
17406 files: 0,
17407 commits: 0,
17408 empty: false,
17409 failed: None,
17410 verified_noop: None,
17411 duration_ms: 0,
17412 folded: false,
17413 }];
17414 state.tally = Some(Tally {
17415 first_choice: BTreeMap::from([('A', 1)]),
17416 borda: BTreeMap::new(),
17417 winner: 'A',
17418 rankings: 1,
17419 unanimous_initial: true,
17420 deliberated: false,
17421 changed_votes: 0,
17422 unanimous_final: true,
17423 tie_break: None,
17424 judges: 0,
17425 present: 0,
17426 quorum: 0,
17427 met_quorum: true,
17428 uncontested: Some("only candidate A produced a change".to_owned()),
17429 });
17430 state.reviews = vec![ReviewRound {
17431 round: 1,
17432 head: "deadbeef".to_owned(),
17433 verified_head: None,
17434 verified_at: None,
17435 reviews: Vec::new(),
17436 e2e: Vec::new(),
17437 fix: None,
17438 blocking: 0,
17439 answered: 0,
17440 expected: 0,
17441 clean: true,
17442 verify_retried: false,
17443 e2e_deferred: false,
17444 e2e_defer_reason: None,
17445 progressed: false,
17446 vote_split: false,
17447 reconsideration: Vec::new(),
17448 verdict: None,
17449 }];
17450 state.gate = vec![CommandOutcome {
17451 command: "test".to_owned(),
17452 code: Some(0),
17453 output_tail: String::new(),
17454 duration_ms: 0,
17455 resource_blocked: false,
17456 }];
17457 state.gate_ran = true;
17458 state.status = RunStatus::Landing;
17462 state.merge = Some(MergeOutcome {
17463 mode: MergeMode::Pr,
17464 ok: true,
17465 detail: "https://example.invalid/x/y/pull/1".to_owned(),
17466 empty: false,
17467 });
17468
17469 ask_test_home();
17473 state.save().expect("save landing run");
17476 let store = ask::Questions::open();
17477 let q = ask_open_question(&store, &state.id);
17478
17479 let mut runner = Runner {
17480 state,
17481 roles: ResolvedRoles {
17482 implementers: Vec::new(),
17483 judges: Vec::new(),
17484 reviewers: Vec::new(),
17485 fixer: None,
17486 conductor: conductor(),
17487 implementer_roster: Vec::new(),
17488 judge_roster: Vec::new(),
17489 reviewer_roster: Vec::new(),
17490 },
17491 sem: Arc::new(Semaphore::new(1)),
17492 pause: Pause::new(),
17493 interrupt: Pause::new(),
17494 };
17495
17496 runner.execute().await.expect("execute");
17501
17502 assert_eq!(
17503 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
17504 Some("https://example.invalid/x/y/pull/1"),
17505 "reentry must not push again or open a second pull request over the \
17506 one `land` is already watching"
17507 );
17508 assert_ne!(
17509 runner.state.status,
17510 RunStatus::Landing,
17511 "land could not actually reach the fake pull request, so it must \
17512 have given up rather than left the run silently parked forever"
17513 );
17514 assert_eq!(runner.state.status, RunStatus::Blocked);
17518 assert!(
17519 store.get(&q.id).unwrap().status.open(),
17520 "Blocked is still alive; settle_questions must have been a no-op here"
17521 );
17522 }
17523
17524 fn state_with_round(round: ReviewRound) -> RunState {
17525 let mut s = RunState::new(
17526 PathBuf::from("/repo"),
17527 "main".to_owned(),
17528 "abc1234".to_owned(),
17529 "add retries".to_owned(),
17530 Config::default(),
17531 );
17532 s.reviews = vec![round];
17533 s
17534 }
17535
17536 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
17537 crate::verdict::Finding {
17538 id: id.to_owned(),
17539 severity,
17540 file: None,
17541 line: None,
17542 title: title.to_owned(),
17543 detail: String::new(),
17544 }
17545 }
17546
17547 #[test]
17548 fn pr_body_names_open_findings_and_declined_ones() {
17549 let round = ReviewRound {
17550 round: 2,
17551 head: "deadbee".to_owned(),
17552 verified_head: None,
17553 verified_at: None,
17554 reviews: vec![ReviewRecord {
17555 attempts: 0,
17556 reviewer: 1,
17557 agent: "alpha".to_owned(),
17558 summary: String::new(),
17559 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
17560 vote: None,
17561 failed: None,
17562 duration_ms: 0,
17563 }],
17564 e2e: vec![CommandOutcome {
17565 command: "cargo test".to_owned(),
17566 code: Some(0),
17567 output_tail: String::new(),
17568 duration_ms: 0,
17569 resource_blocked: false,
17570 }],
17571 verify_retried: false,
17572 e2e_deferred: false,
17573 e2e_defer_reason: None,
17574 fix: Some(FixRecord {
17575 agent: "alpha".to_owned(),
17576 addressed: Vec::new(),
17577 rejected: vec![crate::verdict::Rejection {
17578 id: "R1-1-1".to_owned(),
17579 why: "not reachable from any caller".to_owned(),
17580 }],
17581 notes: String::new(),
17582 committed: true,
17583 failed: None,
17584 duration_ms: 0,
17585 continuation: None,
17586 }),
17587 blocking: 0,
17588 answered: 1,
17589 expected: 1,
17590 clean: false,
17591 progressed: true,
17592 vote_split: false,
17593 reconsideration: Vec::new(),
17594 verdict: None,
17595 };
17596 let state = state_with_round(round);
17597 let body = pr_message(&state, 'A').body;
17598
17599 assert!(body.contains("add retries"), "the task must still be there");
17600 assert!(body.contains("R2-1-1"), "{body}");
17601 assert!(!body.contains("unused import"), "{body}");
17602 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
17603 assert!(
17604 !body.contains("not reachable from any caller"),
17605 "the reason it was declined: {body}"
17606 );
17607 }
17608
17609 #[test]
17610 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
17611 let round = ReviewRound {
17612 round: 1,
17613 head: "deadbee".to_owned(),
17614 verified_head: None,
17615 verified_at: None,
17616 reviews: vec![ReviewRecord {
17617 attempts: 0,
17618 reviewer: 1,
17619 agent: "alpha".to_owned(),
17620 summary: String::new(),
17621 findings: Vec::new(),
17622 vote: None,
17623 failed: None,
17624 duration_ms: 0,
17625 }],
17626 e2e: Vec::new(),
17627 verify_retried: false,
17628 e2e_deferred: false,
17629 e2e_defer_reason: None,
17630 fix: None,
17631 blocking: 0,
17632 answered: 1,
17633 expected: 1,
17634 clean: true,
17635 progressed: false,
17636 vote_split: false,
17637 reconsideration: Vec::new(),
17638 verdict: None,
17639 };
17640 let state = state_with_round(round);
17641 let body = pr_message(&state, 'A').body;
17642 assert!(!body.contains("Open review findings"), "{body}");
17643 assert!(!body.contains("Declined"), "{body}");
17644 }
17645
17646 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
17647 let mut state = RunState::new(
17648 PathBuf::from("/repo"),
17649 "main".to_owned(),
17650 "abc1234".to_owned(),
17651 instruction.to_owned(),
17652 Config::default(),
17653 );
17654 state.candidates.push(Candidate {
17655 index: 0,
17656 label: 'A',
17657 agent: "alpha".to_owned(),
17658 branch: "magi/x/A".to_owned(),
17659 worktree: PathBuf::from("/wt"),
17660 summary: summary.to_owned(),
17661 stat: String::new(),
17662 files: 1,
17663 commits: 1,
17664 empty: false,
17665 failed: None,
17666 verified_noop: None,
17667 folded: false,
17668 duration_ms: 0,
17669 });
17670 state
17671 }
17672
17673 fn review_state(subjects: &[&str]) -> RunState {
17674 let mut state = state_with_summary(
17675 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
17676 "",
17677 );
17678 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17679 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
17680 state
17681 }
17682
17683 #[tokio::test]
17686 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
17687 ask_test_home();
17688 let tmp = tempfile::tempdir().unwrap();
17689 let repo = tmp.path().join("repo");
17690 std::fs::create_dir_all(&repo).unwrap();
17691 init_repo(&repo);
17692 let origin = tmp.path().join("origin.git");
17693 let g = |dir: &Path, args: &[&str]| {
17694 let out = std::process::Command::new("git")
17695 .args(args)
17696 .current_dir(dir)
17697 .quiet()
17698 .output()
17699 .expect("spawn git");
17700 assert!(
17701 out.status.success(),
17702 "git {args:?}: {}",
17703 String::from_utf8_lossy(&out.stderr)
17704 );
17705 };
17706 g(
17707 tmp.path(),
17708 &[
17709 "clone",
17710 "--bare",
17711 "-q",
17712 repo.to_str().unwrap(),
17713 origin.to_str().unwrap(),
17714 ],
17715 );
17716 g(
17717 &repo,
17718 &["remote", "add", "origin", origin.to_str().unwrap()],
17719 );
17720 let c1 = git::rev_parse(&repo, "main").await.unwrap();
17721
17722 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
17724 g(&repo, &["add", "-A"]);
17725 g(
17726 &repo,
17727 &["commit", "-q", "-m", "chore(deps): update a crate"],
17728 );
17729 g(&repo, &["push", "-q", "origin", "main"]);
17730 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
17731 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17732 g(&repo, &["add", "-A"]);
17733 g(
17734 &repo,
17735 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
17736 );
17737 g(&repo, &["checkout", "-q", "main"]);
17738
17739 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
17740 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
17741 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
17743 assert_eq!(fallback, c1);
17744
17745 let mut state = review_state(&[
17746 "chore(deps): update a crate",
17747 "fix(daemon): apply a chosen action",
17748 ]);
17749 state.repo = repo.clone();
17750 state.base_branch = "main".to_owned();
17751 state.base_commit = c1;
17752 refresh_reviewed_commits(&mut state, "feat/own").await;
17753 assert_eq!(
17754 state.reviewed_commits,
17755 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
17756 );
17757 assert_eq!(
17758 review_title(&state).as_deref(),
17759 Some("fix(daemon): apply a chosen action")
17760 );
17761 assert_eq!(
17762 leaked_subjects(&state, "feat/own").await,
17763 Some(vec!["chore(deps): update a crate".to_owned()])
17764 );
17765 state.config.merge.remote = "nowhere".to_owned();
17768 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
17769 }
17770
17771 struct SummaryFixture {
17775 _tmp: tempfile::TempDir,
17776 repo: PathBuf,
17777 reply: PathBuf,
17778 runner: Runner,
17779 }
17780
17781 fn summary_fixture() -> SummaryFixture {
17782 ask_test_home();
17783 let tmp = tempfile::tempdir().unwrap();
17784 let repo = tmp.path().join("repo");
17785 std::fs::create_dir_all(&repo).unwrap();
17786 init_repo(&repo);
17787 let origin = tmp.path().join("origin.git");
17788 let g = |args: &[&str]| {
17789 let out = std::process::Command::new("git")
17790 .args(args)
17791 .current_dir(&repo)
17792 .quiet()
17793 .output()
17794 .expect("spawn git");
17795 assert!(
17796 out.status.success(),
17797 "git {args:?}: {}",
17798 String::from_utf8_lossy(&out.stderr)
17799 );
17800 };
17801 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
17802 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
17803 g(&["push", "-q", "origin", "main"]);
17804 g(&["checkout", "-q", "-b", "feat/own"]);
17805 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
17806 g(&["add", "-A"]);
17807 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17808 let reply = tmp.path().join("reply.txt");
17809 std::fs::write(
17810 &reply,
17811 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17812 )
17813 .unwrap();
17814 let mut runner = runner_at(RunStatus::Gating);
17815 let mut state = review_state(&["feat(own): add the own file"]);
17816 state.repo = repo.clone();
17817 state.base_branch = "main".to_owned();
17818 state.candidates[0].branch = "feat/own".to_owned();
17819 state.candidates[0].worktree = repo.clone();
17820 let mut author = spec("alpha");
17821 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17822 author
17823 .env
17824 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17825 state.config.agents = vec![author];
17826 runner.state = state;
17827 SummaryFixture {
17828 _tmp: tmp,
17829 repo,
17830 reply,
17831 runner,
17832 }
17833 }
17834
17835 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17836 let winner = f.runner.state.candidates[0].clone();
17837 f.runner
17838 .guarded_pr_message(&winner, None, true)
17839 .await
17840 .unwrap()
17841 .expect("an English description never parks")
17842 }
17843
17844 fn clean_round_with_minor_findings() -> ReviewRound {
17845 let mut round = passing_round(2, None);
17846 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17847 round.reviews = vec![ReviewRecord {
17848 attempts: 0,
17849 reviewer: 1,
17850 agent: "alpha".into(),
17851 summary: "内部レビュー".into(),
17852 findings: vec![
17853 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17854 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17855 ],
17856 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17857 failed: None,
17858 duration_ms: 0,
17859 }];
17860 round
17861 }
17862
17863 #[tokio::test]
17864 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17865 for review_only in [false, true] {
17866 let mut f = summary_fixture();
17867 if !review_only {
17868 f.runner.state.candidates[0].agent = "alpha".into();
17869 }
17870 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17871 assert!(f.runner.state.reviews[0].clean);
17872 assert!(f.runner.state.reviews[0].fix.is_none());
17873 assert!(f.runner.state.open_findings().is_empty());
17874 assert_eq!(
17875 crate::pr_outcome::required_finding_ids(&f.runner.state),
17876 ["R2-1-1", "R2-1-2"]
17877 );
17878 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17879 let evidence = crate::pr_outcome::summary_evidence(
17880 &f.runner.state,
17881 crate::pr_outcome::FinalHead::At(&head),
17882 );
17883 assert!(
17884 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17885 );
17886 let omitted = posted(&mut f).await;
17888 assert!(omitted.body.contains("No description"));
17889 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17890 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17891 assert!(f.runner.state.change_summary.is_none());
17892 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17893 let described = posted(&mut f).await;
17894 assert!(described.body.contains("improve documentation for callers"));
17895 assert!(described.body.contains("remaining edge case"));
17896 assert!(described.body.contains("## Open review findings"));
17897 assert!(!described.body.contains("未修正"));
17898 }
17899 }
17900
17901 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17902 let mut f = summary_fixture();
17903 if !review_only {
17904 f.runner.state.candidates[0].agent = "alpha".into();
17905 }
17906 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17907 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17908 fixed
17909 .fix
17910 .as_mut()
17911 .unwrap()
17912 .rejected
17913 .push(crate::verdict::Rejection {
17914 id: "R1-1-1".into(),
17915 why: "呼び出されないコードのため辞退".into(),
17916 });
17917 let mut clean = clean_round_with_minor_findings();
17918 clean.verified_head = Some(head.clone());
17919 f.runner.state.reviews = vec![fixed, clean];
17920 ok_gate(&mut f.runner.state, Some(&head));
17921 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17922 f.runner.state.config.agents.push(scripted("beta", reply));
17923 f.runner.state.config.roles.language_judge =
17924 Some(crate::config::AgentChoice::One("beta".into()));
17925 f
17926 }
17927
17928 #[tokio::test]
17929 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17930 for review_only in [false, true] {
17931 let mut f = rewrite_fixture_with_findings(review_only,
17932 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17933 let m = posted(&mut f).await;
17934 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17935 let expected =
17936 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17937 assert!(m.body.contains(&expected));
17938 assert!(m.body.contains("## Review rounds"));
17939 assert!(
17940 m.body
17941 .contains("Gate (`verify.gate`): passed on the final commit")
17942 );
17943 assert!(
17944 m.body
17945 .contains("Tests (`verify.e2e`): passed on the final commit")
17946 );
17947 let c = f.runner.state.change_summary.as_ref().unwrap();
17948 assert!(
17949 m.body
17950 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17951 );
17952 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17953 assert!(crate::pr_outcome::covers_required_findings(
17954 &f.runner.state,
17955 &m.body
17956 ));
17957 assert!(m.body.contains("documentation remains incomplete"));
17958 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17959 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17960 assert!(f.runner.state.github_text.is_none());
17961 }
17962 }
17963
17964 #[tokio::test]
17965 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17966 for review_only in [false, true] {
17967 for reply in [
17968 "TITLE: fix: public description\nA short English description.",
17969 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17970 ] {
17971 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17972 let winner = f.runner.state.candidates[0].clone();
17973 let posted = f
17974 .runner
17975 .guarded_pr_message(&winner, None, true)
17976 .await
17977 .unwrap();
17978 assert!(posted.is_none());
17979 assert!(f.runner.state.parked);
17980 let withheld = f.runner.state.github_text.as_ref().unwrap();
17981 assert!(
17982 withheld.rewrite.is_none(),
17983 "an incomplete rewrite must not be selectable"
17984 );
17985 let q = ask::Questions::open()
17986 .get(withheld.question.as_ref().unwrap())
17987 .unwrap();
17988 assert!(
17989 !q.choices
17990 .contains(&crate::github_text::USE_REWRITE.to_owned())
17991 );
17992 assert!(
17993 q.detail.contains("R1-1-1")
17994 && q.detail.contains("R2-1-1")
17995 && q.detail.contains("R2-1-2")
17996 );
17997 }
17998 }
17999 }
18000
18001 #[tokio::test]
18002 async fn japanese_reports_stay_local_in_both_public_run_modes() {
18003 for review_only in [false, true] {
18004 let mut f = summary_fixture();
18005 if !review_only {
18006 f.runner.state.candidates[0].agent = "alpha".into();
18007 }
18008 f.runner.state.instruction = "内部タスクの説明".into();
18009 f.runner.state.candidates[0].summary = "内部の候補報告".into();
18010 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18011 f.runner.state.reviews = vec![
18012 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
18013 round_with_notes(2, Some("二回目の内部修正報告です。")),
18014 passing_round(3, Some(&head)),
18015 ];
18016 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
18017 ok_gate(&mut f.runner.state, Some(&head));
18018 let captured = f.repo.join("public-prompt.txt");
18019 let author = &mut f.runner.state.config.agents[0];
18020 author.command = vec![
18021 "sh".into(),
18022 "-c".into(),
18023 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18024 ];
18025 author.env.insert(
18026 "PROMPT_FILE".into(),
18027 captured.to_string_lossy().into_owned(),
18028 );
18029 let m = posted(&mut f).await;
18030 assert!(m.body.contains("Adds `own.txt`"));
18031 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
18032 assert!(m.body.contains("R1-1-1"));
18033 assert!(
18034 m.body
18035 .contains("Gate (`verify.gate`): passed on the final commit")
18036 );
18037 assert!(
18038 m.body
18039 .contains("Tests (`verify.e2e`): passed on the final commit")
18040 );
18041 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18042 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
18043 let prompt = std::fs::read_to_string(captured).unwrap();
18044 assert!(prompt.contains(&format!(
18045 "git diff {} {head}",
18046 f.runner.state.change_summary.as_ref().unwrap().base
18047 )));
18048 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
18049 assert_eq!(
18050 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
18051 "二回目の内部修正報告です。"
18052 );
18053 }
18054 }
18055
18056 #[tokio::test]
18057 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
18058 for review_only in [false, true] {
18059 let mut f = summary_fixture();
18060 if !review_only {
18061 f.runner.state.candidates[0].agent = "alpha".into();
18062 }
18063 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
18064 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
18065 std::fs::write(&f.reply, "no usable description").unwrap();
18066 let failed = posted(&mut f).await;
18067 assert!(
18068 failed
18069 .body
18070 .contains("No description of what this change does could be generated")
18071 );
18072 assert!(failed.body.is_ascii());
18073 assert!(f.runner.state.github_text.is_none());
18074 assert!(f.runner.state.change_summary.is_none());
18075 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18076 let evidence = crate::pr_outcome::summary_evidence(
18077 &f.runner.state,
18078 crate::pr_outcome::FinalHead::At(&head),
18079 );
18080 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
18081 assert!(
18082 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
18083 .exists()
18084 );
18085 f.runner.state =
18086 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
18087 std::fs::write(
18088 &f.reply,
18089 "TITLE: feat(own): add the own file\nAdds the committed own file.",
18090 )
18091 .unwrap();
18092 let retried = posted(&mut f).await;
18093 assert!(retried.body.contains("Adds the committed own file."));
18094 assert_eq!(
18095 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
18096 "日本語の修正報告"
18097 );
18098 }
18099 }
18100
18101 #[tokio::test]
18102 async fn same_head_verification_and_findings_invalidate_public_prose() {
18103 let mut f = summary_fixture();
18104 posted(&mut f).await;
18105 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18106 let original = f
18107 .runner
18108 .state
18109 .change_summary
18110 .as_ref()
18111 .unwrap()
18112 .evidence
18113 .clone();
18114 ok_gate(&mut f.runner.state, Some(&head));
18115 std::fs::write(
18116 &f.reply,
18117 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
18118 )
18119 .unwrap();
18120 let verified = posted(&mut f).await;
18121 assert!(verified.body.contains("now verified"));
18122 assert_ne!(
18123 f.runner.state.change_summary.as_ref().unwrap().evidence,
18124 original
18125 );
18126 let mut round = round_with_notes(1, Some("内部報告"));
18127 round.reviews.push(ReviewRecord {
18128 attempts: 0,
18129 reviewer: 1,
18130 agent: "alpha".into(),
18131 summary: "内部レビュー".into(),
18132 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
18133 vote: None,
18134 failed: None,
18135 duration_ms: 0,
18136 });
18137 round
18138 .fix
18139 .as_mut()
18140 .unwrap()
18141 .rejected
18142 .push(crate::verdict::Rejection {
18143 id: "R1-1-1".into(),
18144 why: "呼び出し元から到達できない".into(),
18145 });
18146 f.runner.state.reviews = vec![round];
18147 let omitted = posted(&mut f).await;
18149 assert!(omitted.body.contains("No description"));
18150 assert!(omitted.body.contains("R1-1-1"));
18151 assert!(!omitted.body.contains("到達"));
18152 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
18153 let described = posted(&mut f).await;
18154 assert!(described.body.contains("unreachable from callers"));
18155 assert!(described.body.contains("R1-1-1"));
18156 assert!(!described.body.contains("到達"));
18157 let evidence = f
18158 .runner
18159 .state
18160 .change_summary
18161 .as_ref()
18162 .unwrap()
18163 .evidence
18164 .clone();
18165 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
18166 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
18167 assert!(posted(&mut f).await.body.contains("rationale updated"));
18168 assert_ne!(
18169 f.runner.state.change_summary.as_ref().unwrap().evidence,
18170 evidence
18171 );
18172 f.runner
18174 .state
18175 .change_summary
18176 .as_mut()
18177 .unwrap()
18178 .evidence
18179 .clear();
18180 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
18181 assert!(posted(&mut f).await.body.contains("current specification"));
18182 }
18183
18184 #[tokio::test]
18185 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
18186 let mut f = summary_fixture();
18187 let m = posted(&mut f).await;
18188 assert_eq!(m.title, "feat(own): add the own file");
18189 assert!(
18190 m.body
18191 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
18192 "{}",
18193 m.body
18194 );
18195 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
18196 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
18197 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
18198 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
18199 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
18200 let kept = f.runner.state.change_summary.as_ref().unwrap();
18201 assert_eq!(
18202 (kept.head.as_str(), kept.base.as_str()),
18203 (head.as_str(), base.as_str())
18204 );
18205 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
18206 }
18207
18208 #[tokio::test]
18209 async fn review_only_summary_receives_the_configured_github_language() {
18210 let mut f = summary_fixture();
18211 f.runner.state.config.graph.github_language = "ja".into();
18212 let captured = f.repo.join("summary-prompt.txt");
18213 let author = &mut f.runner.state.config.agents[0];
18214 author.command = vec![
18215 "sh".into(),
18216 "-c".into(),
18217 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
18218 ];
18219 author.env.insert(
18220 "PROMPT_FILE".into(),
18221 captured.to_string_lossy().into_owned(),
18222 );
18223 std::fs::write(
18224 &f.reply,
18225 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
18226 )
18227 .unwrap();
18228 let m = posted(&mut f).await;
18229 let prompt = std::fs::read_to_string(captured).unwrap();
18230 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
18231 assert!(!prompt.contains("in English"));
18232 assert_eq!(m.title, "feat(own): ファイルを追加");
18233 assert!(m.body.contains("own.txt を追加します。"));
18234 }
18235
18236 #[tokio::test]
18237 async fn review_fixes_come_after_the_whole_change_summary() {
18238 let mut f = summary_fixture();
18239 f.runner.state.reviews = vec![
18240 round_with_notes(1, Some("handled the PATH case")),
18241 round_with_notes(2, None),
18242 ];
18243 let m = posted(&mut f).await;
18244 let summary = m.body.find("## Summary").expect("summary");
18245 let fixes = m.body.find("## Review rounds").expect("review rounds");
18246 assert!(summary < fixes, "{}", m.body);
18247 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
18248 assert!(m.body.contains("## Final verification"), "{}", m.body);
18249 }
18250
18251 #[tokio::test]
18252 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
18253 let mut f = summary_fixture();
18254 let first = posted(&mut f).await;
18255 let first_head = f.runner.state.change_summary.clone().unwrap().head;
18256 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
18258 let again = posted(&mut f).await;
18259 assert_eq!((again.title, again.body), (first.title, first.body));
18260
18261 let g = |args: &[&str]| {
18262 let out = std::process::Command::new("git")
18263 .args(args)
18264 .current_dir(&f.repo)
18265 .quiet()
18266 .output()
18267 .expect("spawn git");
18268 assert!(out.status.success(), "git {args:?}");
18269 };
18270 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
18271 g(&["add", "-A"]);
18272 g(&["commit", "-q", "-m", "fix(own): handle more"]);
18273 let second = posted(&mut f).await;
18274 let kept = f.runner.state.change_summary.clone().unwrap();
18275 assert_ne!(kept.head, first_head);
18276 assert!(second.body.contains("Not read yet."), "{}", second.body);
18277 assert!(!second.body.contains("so the first summary is read"));
18278 assert_eq!(second.title, "feat(own): other");
18279 }
18280
18281 #[tokio::test]
18282 async fn an_unavailable_summary_is_stated_and_never_invented() {
18283 let mut f = summary_fixture();
18284 std::fs::write(&f.reply, "no title line here").unwrap();
18285 let m = posted(&mut f).await;
18286 assert!(
18287 m.body
18288 .contains("No description of what this change does could be generated"),
18289 "{}",
18290 m.body
18291 );
18292 assert!(m.body.contains("internal reports remain in local records"));
18293 assert!(f.runner.state.change_summary.is_none());
18294 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
18295 let winner = f.runner.state.candidates[0].clone();
18297 let preview = f
18298 .runner
18299 .guarded_pr_message(&winner, None, false)
18300 .await
18301 .unwrap()
18302 .unwrap();
18303 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
18304 }
18305
18306 #[test]
18307 fn pr_message_review_single_commit_uses_its_subject() {
18308 let state = review_state(&["feat(nats): per-role user"]);
18309 let m = pr_message(&state, 'A');
18310 assert_eq!(m.title, "feat(nats): per-role user");
18311 assert!(!m.body.contains("Review the work already"), "{}", m.body);
18312 assert!(m.body.contains("## Commits under review"), "{}", m.body);
18313 }
18314
18315 #[test]
18316 fn pr_message_review_multi_commit_takes_the_oldest() {
18317 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
18318 let m = pr_message(&state, 'A');
18319 assert_eq!(m.title, "feat: the change");
18320 for s in ["feat: the change", "fix: typo", "fix: again"] {
18321 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
18322 }
18323 }
18324
18325 #[test]
18326 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
18327 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
18328 let mut state = review_state(&[first, "fix: later fixup"]);
18329 state.candidates[0].branch = "機能/ログイン".to_owned();
18330 let m = pr_message(&state, 'A');
18331 assert!(
18332 m.title.starts_with("chore: land candidate A of run"),
18333 "{}",
18334 m.title
18335 );
18336 }
18337 }
18338
18339 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
18340 BranchFacts {
18341 commits: commits
18342 .iter()
18343 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
18344 .collect(),
18345 stat: stat.to_owned(),
18346 }
18347 }
18348
18349 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
18350 let mut r = review_round(true, 0, 1, 1, true, true);
18351 r.round = round;
18352 r.fix = notes.map(|n| FixRecord {
18353 agent: "fixer".to_owned(),
18354 addressed: Vec::new(),
18355 rejected: Vec::new(),
18356 notes: n.to_owned(),
18357 committed: true,
18358 failed: None,
18359 duration_ms: 0,
18360 continuation: None,
18361 });
18362 r
18363 }
18364
18365 #[test]
18366 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
18367 let state = review_state(&["ignored"]);
18368 let f = facts(
18369 &[
18370 (
18371 "fix(login): resolve PATH on macOS",
18372 "Login shells skip rc files.",
18373 ),
18374 ("fix: address review", ""),
18375 ],
18376 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
18377 );
18378 let m = pr_message_with(&state, 'A', Some(&f));
18379 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
18380 assert!(
18381 m.body
18382 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
18383 "{}",
18384 m.body
18385 );
18386 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
18387 assert!(m.body.contains("## Diff stat"), "{}", m.body);
18388 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
18389 for banned in [
18390 "Review the work already",
18391 "no task statement",
18392 "Original task",
18393 ] {
18394 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
18395 }
18396 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
18397 }
18398
18399 #[test]
18400 fn pr_message_review_truncates_a_huge_first_commit_body() {
18401 let state = review_state(&["ignored"]);
18402 let f = facts(
18403 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
18404 "s",
18405 );
18406 let m = pr_message_with(&state, 'A', Some(&f));
18407 assert!(m.body.len() < 30_000, "{}", m.body.len());
18408 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
18409 assert!(
18410 m.body.contains("- ... 1 more commit(s)"),
18411 "{}",
18412 m.body.len()
18413 );
18414 assert!(m.body.ends_with("magi:candidate-a\n"));
18415 }
18416
18417 #[test]
18418 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
18419 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
18420 assert_eq!(m.title, "feat: x");
18421 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
18422 assert!(!m.body.contains("Diff stat"), "{}", m.body);
18423 assert!(!m.body.contains("no task statement"), "{}", m.body);
18424 }
18425
18426 #[test]
18427 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
18428 let mut state = review_state(&["日本語の件名"]);
18429 state.candidates[0].branch = "fix/macos-login-path".to_owned();
18430 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
18431 state.candidates[0].branch = "機能/ログイン".to_owned();
18432 assert!(
18433 pr_message(&state, 'A')
18434 .title
18435 .starts_with("chore: land candidate A")
18436 );
18437 }
18438
18439 #[test]
18440 fn pr_message_review_fixes_survive_a_clean_final_round() {
18441 let mut state = review_state(&["feat: x"]);
18442 state.reviews = vec![
18443 round_with_notes(1, Some("handled the PATH case")),
18444 round_with_notes(2, None),
18445 ];
18446 let body = pr_message(&state, 'A').body;
18447 assert!(
18448 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
18449 "{body}"
18450 );
18451 assert!(body.contains("### Round 1"), "{body}");
18452
18453 state.reviews = vec![
18454 round_with_notes(1, Some("first fix")),
18455 round_with_notes(2, Some("")),
18456 round_with_notes(3, Some("second fix")),
18457 round_with_notes(4, None),
18458 ];
18459 let body = pr_message(&state, 'A').body;
18460 assert!(
18461 body.contains("### Round 1") && !body.contains("first fix"),
18462 "{body}"
18463 );
18464 assert!(
18465 body.contains("### Round 3") && !body.contains("second fix"),
18466 "{body}"
18467 );
18468 assert!(body.contains("### Round 2"), "{body}");
18469 }
18470
18471 const FINAL: &str = "abcdef1234567890";
18472
18473 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
18474 let mut r = round_with_notes(round, None);
18475 r.verified_head = verified.map(str::to_owned);
18476 r
18477 }
18478
18479 fn ok_gate(state: &mut RunState, head: Option<&str>) {
18480 state.gate = vec![CommandOutcome {
18481 command: "gate".to_owned(),
18482 code: Some(0),
18483 output_tail: String::new(),
18484 duration_ms: 0,
18485 resource_blocked: false,
18486 }];
18487 state.gate_ran = true;
18488 state.gate_head = head.map(str::to_owned);
18489 }
18490
18491 fn body_at(state: &RunState) -> String {
18492 pr_message_raw(
18493 state,
18494 'A',
18495 None,
18496 crate::pr_outcome::FinalHead::At(FINAL),
18497 ChangeNote::NotAsked,
18498 )
18499 .body
18500 }
18501
18502 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
18503
18504 #[test]
18505 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
18506 let mut state = review_state(&["feat: x"]);
18507 state.reviews = vec![
18508 round_with_notes(1, Some(SANDBOXED)),
18509 passing_round(2, Some(FINAL)),
18510 ];
18511 ok_gate(&mut state, Some(FINAL));
18512 let body = body_at(&state);
18513 assert!(!body.contains("Handled the PATH case."), "{body}");
18514 assert!(!body.contains("could not commit"), "{body}");
18515 assert!(!body.contains("could not run"), "{body}");
18516 assert!(
18517 body.contains("The fix is committed on the branch"),
18518 "{body}"
18519 );
18520 assert!(!body.contains("sandbox"), "{body}");
18521 assert!(
18522 body.contains("Tests (`verify.e2e`): passed on the final commit"),
18523 "{body}"
18524 );
18525 assert!(
18526 body.contains("Gate (`verify.gate`): passed on the final commit"),
18527 "{body}"
18528 );
18529 assert!(
18531 state.reviews[0]
18532 .fix
18533 .as_ref()
18534 .unwrap()
18535 .notes
18536 .contains("could not commit")
18537 );
18538 }
18539
18540 #[test]
18541 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
18542 let mut state = review_state(&["feat: x"]);
18543 let mut r1 = round_with_notes(1, Some(SANDBOXED));
18544 r1.fix.as_mut().unwrap().committed = false;
18545 let mut r2 = passing_round(2, Some(FINAL));
18546 r2.e2e[0].code = Some(1);
18547 state.reviews = vec![r1, r2];
18548 let body = body_at(&state);
18549 assert!(!body.contains("could not commit"), "{body}");
18550 assert!(!body.contains("could not run"), "{body}");
18551 assert!(body.contains("This round produced no commit"), "{body}");
18552 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
18553 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
18554 assert!(!body.contains("passed"), "{body}");
18555 }
18556
18557 #[test]
18558 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
18559 let mut state = review_state(&["feat: x"]);
18560 state.reviews = vec![
18561 round_with_notes(1, Some(SANDBOXED)),
18562 passing_round(2, Some("0123456789abcdef")),
18563 ];
18564 ok_gate(&mut state, Some("0123456789abcdef"));
18565 let body = body_at(&state);
18566 assert!(!body.contains("HTTP tests could not run"), "{body}");
18567 assert!(!body.contains("could not commit"), "{body}");
18569 assert!(
18570 body.contains("passed on an earlier commit (`0123456`)"),
18571 "{body}"
18572 );
18573 assert!(!body.contains("passed on the final commit"), "{body}");
18574
18575 ok_gate(&mut state, None);
18577 let body = body_at(&state);
18578 assert!(
18579 body.contains("passed on a commit that was not recorded"),
18580 "{body}"
18581 );
18582 }
18583
18584 #[test]
18585 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
18586 let state = state_with_summary(
18587 "add retries to the client",
18588 "TITLE: feat: retries\n\nDid it.",
18589 );
18590 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
18591 assert_eq!(m.title, "feat: retries");
18592 assert!(
18593 m.body.contains("<summary>Original task</summary>"),
18594 "{}",
18595 m.body
18596 );
18597 assert!(!m.body.contains("Commits under review"), "{}", m.body);
18598 assert!(
18599 m.body
18600 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
18601 "{}",
18602 m.body
18603 );
18604 }
18605
18606 #[test]
18607 fn pr_message_review_bounds_a_long_english_subject() {
18608 let long = format!("feat: {}", "word ".repeat(100));
18609 let m = pr_message(&review_state(&[&long]), 'A');
18610 assert!(m.title.starts_with("feat: word"), "{}", m.title);
18611 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18612 }
18613
18614 #[test]
18615 fn pr_message_implementation_run_is_unchanged_by_review_support() {
18616 let state = state_with_summary("add retries\n\ndetails", "- did some things");
18617 let m = pr_message(&state, 'A');
18618 assert_eq!(m.title, "add retries");
18619 assert!(m.body.contains("<summary>Original task</summary>"));
18620 assert!(!m.body.contains("Commits under review"));
18621 assert_eq!(landing_subject_source(&state), state.instruction);
18622 }
18623
18624 #[test]
18625 fn review_run_squash_subject_is_the_change_not_the_prompt() {
18626 let state = review_state(&["feat: the change", "fix: typo"]);
18627 let source = landing_subject_source(&state);
18628 assert_eq!(land::merge_subject("", &source), "feat: the change");
18629 assert_eq!(
18630 land::merge_subject("magi: candidate A (uncommitted work)", &source),
18631 "feat: the change"
18632 );
18633 assert_eq!(
18635 land::merge_subject("feat: renamed by hand", &source),
18636 "feat: renamed by hand"
18637 );
18638 let blank = review_state(&["日本語"]);
18639 assert!(
18640 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
18641 );
18642 }
18643
18644 #[test]
18645 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
18646 let state = review_state(&["feat: the change"]);
18647 let source = landing_subject_source(&state);
18648 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
18649 assert_eq!(
18650 land::merge_subject(landing_title(&state, old), &source),
18651 "feat: the change"
18652 );
18653 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
18654 let task = state_with_summary("add retries", "");
18655 assert_eq!(landing_title(&task, old), old);
18656 }
18657
18658 #[test]
18659 fn pr_message_describes_the_change_not_the_task() {
18660 let state = state_with_summary(
18661 "今回やってほしいこと: results projector を直す",
18662 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
18663 );
18664 let m = pr_message(&state, 'A');
18665 assert_eq!(m.title, "fix(web): batch the runs list reads");
18666 assert!(
18667 m.body.starts_with("## Summary\n\n- reads run.json once"),
18668 "{}",
18669 m.body
18670 );
18671 assert!(!m.body.contains("TITLE:"), "{}", m.body);
18672 let task_at = m.body.find("今回やってほしいこと").unwrap();
18673 let details_at = m.body.find("<details>").unwrap();
18674 assert!(
18675 details_at < task_at,
18676 "the task lives inside <details>: {}",
18677 m.body
18678 );
18679 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
18680 assert!(m.body.contains("magi:candidate-a"));
18681 }
18682
18683 #[test]
18684 fn pr_message_falls_back_to_the_task_without_a_title_line() {
18685 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
18686 let m = pr_message(&state, 'A');
18687 assert_eq!(m.title, "add retries");
18688 assert!(
18689 m.body.contains("## Summary\n\n- did some things"),
18690 "{}",
18691 m.body
18692 );
18693
18694 let none = RunState::new(
18695 PathBuf::from("/repo"),
18696 "main".to_owned(),
18697 "abc1234".to_owned(),
18698 "add retries".to_owned(),
18699 Config::default(),
18700 );
18701 let m = pr_message(&none, 'A');
18702 assert_eq!(m.title, "add retries");
18703 assert!(!m.body.contains("## Summary"), "{}", m.body);
18704 }
18705
18706 #[test]
18707 fn pr_message_refuses_the_candidate_commit_subject() {
18708 for bad in [
18709 "TITLE: magi: candidate A (uncommitted work)",
18710 "TITLE: chore: stuff (uncommitted work)",
18711 "TITLE: ",
18712 ] {
18713 let state = state_with_summary("add retries", bad);
18714 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
18715 }
18716 }
18717
18718 #[test]
18719 fn pr_message_bounds_a_very_long_task_and_title() {
18720 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
18721 let state = state_with_summary(&long, "- nothing");
18722 let m = pr_message(&state, 'A');
18723 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18724 assert!(!m.title.contains('\n'));
18725
18726 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
18727 let m = pr_message(&state, 'A');
18728 assert!(m.title.starts_with("feat: "));
18729 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
18730 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
18731 }
18732
18733 fn long_title_of(instruction: &str) -> String {
18734 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
18735 }
18736
18737 #[test]
18738 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
18739 let first = "Make the landing path keep a readable title for long tasks";
18740 let line = format!(
18741 "{first}. {}",
18742 "And then keep going with more words ".repeat(20)
18743 );
18744 let t = long_title_of(&line);
18745 assert_eq!(t, first);
18746 assert!(!t.starts_with("chore: land"));
18747 }
18748
18749 #[test]
18750 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
18751 let line = "word ".repeat(200);
18752 let t = long_title_of(&line);
18753 assert!(t.ends_with("word..."), "{t}");
18754 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
18755 }
18756
18757 #[test]
18758 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
18759 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
18760 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
18761 }
18762 }
18763
18764 #[test]
18765 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
18766 let mut state = state_with_summary("再試行を追加する", "- nothing");
18767 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
18768 state.config.graph.github_language = "ja".into();
18769 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
18770 }
18771
18772 #[test]
18773 fn pr_message_title_limit_is_exact() {
18774 let at = "a".repeat(PR_TITLE_MAX);
18775 assert_eq!(long_title_of(&at), at);
18776 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
18777 assert!(over.ends_with("..."), "{over}");
18778 assert_eq!(over.chars().count(), PR_TITLE_MAX);
18779 }
18780
18781 #[test]
18782 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
18783 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
18784 let t = long_title_of(&line);
18785 assert!(t.ends_with("..."), "{t}");
18786 assert!(t.is_ascii(), "{t}");
18787 }
18788
18789 #[test]
18790 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
18791 let line = format!(
18792 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
18793 "plus more filler words ".repeat(20)
18794 );
18795 let t = long_title_of(&line);
18796 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
18797 }
18798
18799 #[test]
18800 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
18801 let line = format!("\n\n# {}", "title words ".repeat(40));
18802 let state = state_with_summary(&line, "- nothing");
18803 let m = pr_message(&state, 'A');
18804 assert!(m.title.starts_with("title words"), "{}", m.title);
18805 assert_eq!(
18806 land::merge_subject(&m.title, &landing_subject_source(&state)),
18807 m.title
18808 );
18809 assert_eq!(
18811 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18812 "feat: renamed by hand"
18813 );
18814 }
18815
18816 #[tokio::test]
18817 async fn github_text_rewrite_is_bounded_and_falls_back() {
18818 crate::run::pin_test_home();
18819 for (reply, accepted) in [
18820 (
18821 "TITLE: fix: retries\nAdd retries for failed requests.",
18822 true,
18823 ),
18824 (
18825 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18826 false,
18827 ),
18828 (
18829 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18830 true,
18831 ),
18832 (
18833 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18834 true,
18835 ),
18836 (
18837 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18838 true,
18839 ),
18840 (
18841 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18842 false,
18843 ),
18844 ("TITLE: fix: retries\n", false),
18845 ] {
18846 let dir = tempfile::tempdir().unwrap();
18847 let mut runner = runner_at(RunStatus::Gating);
18848 runner.state = state_with_summary(
18849 "add retries",
18850 "TITLE: fix: retries\n日本語の説明を書きます。",
18851 );
18852 runner.state.repo = dir.path().to_owned();
18853 runner.state.candidates[0].worktree = dir.path().to_owned();
18854 let mut author = spec("alpha");
18855 author.command = vec![
18856 "sh".into(),
18857 "-c".into(),
18858 "printf '%s' \"$REWRITE_REPLY\"".into(),
18859 ];
18860 author.env.insert("REWRITE_REPLY".into(), reply.into());
18861 runner.state.config.agents = vec![author];
18862 seed_public_summary(&mut runner);
18863 let winner = runner.state.candidates[0].clone();
18864 let first = runner
18865 .guarded_pr_message(&winner, None, true)
18866 .await
18867 .unwrap();
18868 let message = if accepted {
18869 first.expect("a passing rewrite never asks")
18870 } else {
18871 assert!(first.is_none());
18873 assert!(runner.state.parked);
18874 let store = ask::Questions::open();
18875 let asked: Vec<_> = store
18876 .list()
18877 .into_iter()
18878 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18879 .collect();
18880 assert_eq!(asked.len(), 1);
18881 assert!(!asked[0].detail.contains("secret"));
18882 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18883 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18884 assert_eq!(
18885 asked[0]
18886 .choices
18887 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18888 rewrite.is_some()
18889 );
18890 store
18893 .update(&asked[0].id, |q| {
18894 q.abandon("test");
18895 Ok(())
18896 })
18897 .unwrap();
18898 let again = runner
18899 .guarded_pr_message(&winner, None, true)
18900 .await
18901 .unwrap()
18902 .expect("an abandoned question falls back");
18903 let n = store
18904 .list()
18905 .into_iter()
18906 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18907 .count();
18908 assert_eq!(n, 1, "asked once per rejected text");
18909 again
18910 };
18911 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18912 assert!(!message.body.contains("secret"));
18913 assert!(!message.body.contains("prefix"));
18914 assert!(!message.title.contains("secret"));
18915 assert_eq!(
18916 message.body.contains("Add retries for failed requests."),
18917 accepted
18918 );
18919 assert_eq!(
18920 runner.state.seats["github-text-rewrite"].turns, 1,
18921 "only one rewrite invocation"
18922 );
18923 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18924 if accepted {
18925 assert!(runner.state.github_text.is_none());
18926 assert!(!runner.state.parked);
18927 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18928 assert!(message.body.contains("## Final verification"));
18929 assert!(message.body.contains("Evidence: final diff"));
18930 assert!(
18931 message
18932 .body
18933 .contains(&format!("magi:run/{}", runner.state.id))
18934 );
18935 }
18936 if !accepted {
18937 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18938 assert!(
18939 message
18940 .body
18941 .contains(&format!("magi:run/{}", runner.state.id))
18942 );
18943 }
18944 }
18945 }
18946
18947 fn scripted(id: &str, reply: &str) -> AgentSpec {
18948 let mut a = spec(id);
18949 a.command = vec![
18950 "sh".into(),
18951 "-c".into(),
18952 "printf '%s' \"$REWRITE_REPLY\"".into(),
18953 ];
18954 a.env.insert("REWRITE_REPLY".into(), reply.into());
18955 a
18956 }
18957
18958 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18959 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18960
18961 fn seed_public_summary(runner: &mut Runner) {
18963 let repo = &runner.state.repo;
18964 let g = |args: &[&str]| {
18965 let out = std::process::Command::new("git")
18966 .args(args)
18967 .current_dir(repo)
18968 .quiet()
18969 .output()
18970 .unwrap();
18971 assert!(out.status.success(), "git {args:?}");
18972 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18973 };
18974 g(&["init", "-q", "-b", "main"]);
18975 g(&[
18976 "-c",
18977 "user.name=Test",
18978 "-c",
18979 "user.email=test@example.invalid",
18980 "commit",
18981 "-q",
18982 "--allow-empty",
18983 "-m",
18984 "feat: fixture",
18985 ]);
18986 let head = g(&["rev-parse", "HEAD"]);
18987 g(&["update-ref", "refs/remotes/origin/main", &head]);
18988 let evidence = crate::pr_outcome::summary_evidence(
18989 &runner.state,
18990 crate::pr_outcome::FinalHead::At(&head),
18991 );
18992 let generated = &runner.state.candidates[0].summary;
18993 runner.state.change_summary = Some(crate::run::ChangeSummary {
18994 base: head.clone(),
18995 head,
18996 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18997 title: summary_title(generated).unwrap(),
18998 body: summary_without_title(generated),
18999 });
19000 }
19001
19002 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
19003 let mut runner = runner_at(RunStatus::Gating);
19004 runner.state = state_with_summary("add retries", JA_SUMMARY);
19005 if review_only {
19006 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
19007 }
19008 runner.state.repo = dir.to_owned();
19009 runner.state.candidates[0].worktree = dir.to_owned();
19010 runner.state.config.agents = agents;
19011 seed_public_summary(&mut runner);
19012 runner
19013 }
19014
19015 fn gate_questions(runner: &Runner) -> usize {
19016 ask::Questions::open()
19017 .list()
19018 .into_iter()
19019 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
19020 .count()
19021 }
19022
19023 #[tokio::test]
19024 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
19025 crate::run::pin_test_home();
19026 let dir = tempfile::tempdir().unwrap();
19027 let mut runner = rewrite_runner(
19028 dir.path(),
19029 true,
19030 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19031 );
19032 runner.state.config.roles.language_judge =
19033 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
19034 let winner = runner.state.candidates[0].clone();
19035 let msg = runner
19036 .guarded_pr_message(&winner, None, true)
19037 .await
19038 .unwrap()
19039 .expect("an English rewrite posts without asking");
19040 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
19041 assert!(msg.body.contains("<details>kept</details>"));
19042 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
19043 assert_eq!(gate_questions(&runner), 0);
19044 assert!(runner.state.github_text.is_none());
19045 assert!(!runner.state.seats.contains_key("impl-A"));
19046 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19047 }
19048
19049 #[tokio::test]
19050 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
19051 crate::run::pin_test_home();
19052 let dir = tempfile::tempdir().unwrap();
19053 let mut runner = rewrite_runner(
19054 dir.path(),
19055 false,
19056 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
19057 );
19058 runner.state.config.roles.language_judge =
19059 Some(crate::config::AgentChoice::One("beta".into()));
19060 runner.state.seats.insert(
19061 "impl-A".into(),
19062 SeatState::new("impl-A", "alpha", runner.state.seed),
19063 );
19064 let winner = runner.state.candidates[0].clone();
19065 let msg = runner
19066 .guarded_pr_message(&winner, None, true)
19067 .await
19068 .unwrap()
19069 .expect("rewritten");
19070 assert!(msg.body.contains("<details>kept</details>"));
19071 assert_eq!(runner.state.seats["impl-A"].turns, 0);
19072 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19073 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19074 }
19075
19076 #[tokio::test]
19077 async fn rewrite_reuses_a_resumable_implementer_session() {
19078 crate::run::pin_test_home();
19079 let dir = tempfile::tempdir().unwrap();
19080 let mut runner = rewrite_runner(
19081 dir.path(),
19082 false,
19083 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
19084 );
19085 runner.state.config.roles.language_judge =
19086 Some(crate::config::AgentChoice::One("beta".into()));
19087 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19088 seat.turns = 1;
19089 runner.state.seats.insert("impl-A".into(), seat);
19090 let winner = runner.state.candidates[0].clone();
19091 runner
19092 .guarded_pr_message(&winner, None, true)
19093 .await
19094 .unwrap()
19095 .expect("rewritten");
19096 assert_eq!(runner.state.seats["impl-A"].turns, 2);
19097 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
19098 }
19099
19100 #[tokio::test]
19101 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
19102 crate::run::pin_test_home();
19105 let dir = tempfile::tempdir().unwrap();
19106 let mut exhausted = spec("alpha");
19107 exhausted.command = vec!["false".into()];
19108 let mut runner = rewrite_runner(
19109 dir.path(),
19110 false,
19111 vec![exhausted, scripted("beta", EN_REWRITE)],
19112 );
19113 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19115 "alpha".into(),
19116 "beta".into(),
19117 ]));
19118 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
19119 seat.turns = 1;
19120 runner.state.seats.insert("impl-A".into(), seat);
19121 let winner = runner.state.candidates[0].clone();
19122 let msg = runner
19123 .guarded_pr_message(&winner, None, true)
19124 .await
19125 .unwrap()
19126 .expect("the role rewrites after the implementer fails");
19127 assert!(msg.body.contains("`src/net.rs`"));
19128 assert_eq!(
19129 runner.state.seats["impl-A"].turns, 2,
19130 "the implementer was tried once"
19131 );
19132 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
19133 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19134 }
19135
19136 #[tokio::test]
19137 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
19138 crate::run::pin_test_home();
19139 let dir = tempfile::tempdir().unwrap();
19140 let mut broken = spec("bad");
19141 broken.command = vec!["false".into()];
19142 let mut runner = rewrite_runner(
19143 dir.path(),
19144 true,
19145 vec![
19146 broken,
19147 scripted("noformat", "Add retries."),
19148 scripted("good", EN_REWRITE),
19149 ],
19150 );
19151 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
19152 "bad".into(),
19153 "noformat".into(),
19154 "good".into(),
19155 ]));
19156 let winner = runner.state.candidates[0].clone();
19157 let msg = runner
19158 .guarded_pr_message(&winner, None, true)
19159 .await
19160 .unwrap()
19161 .expect("the third agent rewrites");
19162 assert!(msg.body.contains("`src/net.rs`"));
19163 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
19164 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
19165 }
19166
19167 #[tokio::test]
19168 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
19169 crate::run::pin_test_home();
19170 for (agents, role, saved) in [
19172 (
19173 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
19174 Some("beta"),
19175 true,
19176 ),
19177 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
19178 (Vec::new(), None, false),
19179 ] {
19180 let dir = tempfile::tempdir().unwrap();
19181 let mut runner = rewrite_runner(dir.path(), true, agents);
19182 runner.state.config.roles.language_judge =
19183 role.map(|r| crate::config::AgentChoice::One(r.into()));
19184 let winner = runner.state.candidates[0].clone();
19185 let first = runner
19186 .guarded_pr_message(&winner, None, true)
19187 .await
19188 .unwrap();
19189 assert!(first.is_none(), "withheld text parks the run");
19190 assert!(runner.state.parked);
19191 assert_eq!(gate_questions(&runner), 1);
19192 let g = runner.state.github_text.as_ref().unwrap();
19193 assert_eq!(g.rewrite.is_some(), saved);
19194 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
19195 }
19196 }
19197
19198 #[tokio::test]
19199 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
19200 crate::run::pin_test_home();
19201 let dir = tempfile::tempdir().unwrap();
19202 let mut runner = runner_at(RunStatus::Gating);
19203 runner.state =
19204 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
19205 runner.state.repo = dir.path().to_owned();
19206 runner.state.candidates[0].worktree = dir.path().to_owned();
19207 let mut author = spec("alpha");
19208 author.command = vec![
19209 "sh".into(),
19210 "-c".into(),
19211 "printf '%s' \"$REWRITE_REPLY\"".into(),
19212 ];
19213 author.env.insert(
19214 "REWRITE_REPLY".into(),
19215 format!(
19216 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
19217 "English sentence. ".repeat(600)
19218 ),
19219 );
19220 runner.state.config.agents = vec![author];
19221 seed_public_summary(&mut runner);
19222 let winner = runner.state.candidates[0].clone();
19223 assert!(
19224 runner
19225 .guarded_pr_message(&winner, None, true)
19226 .await
19227 .unwrap()
19228 .is_none()
19229 );
19230 let record = runner.state.github_text.clone().unwrap();
19231 let rewrite = record.rewrite.clone().unwrap();
19232 assert_eq!(rewrite.failed, ["body-language"]);
19233 assert!(
19234 rewrite
19235 .body
19236 .ends_with(&format!("magi:run/{}", runner.state.id))
19237 );
19238 let store = ask::Questions::open();
19239 let q = store.get(record.question.as_ref().unwrap()).unwrap();
19240 assert!(
19241 q.choices
19242 .contains(&crate::github_text::USE_REWRITE.to_owned())
19243 );
19244 for label in [
19245 "ORIGINAL",
19246 "FALLBACK",
19247 "REWRITE",
19248 "Rules still failed: body-language",
19249 "truncated",
19250 ] {
19251 assert!(q.detail.contains(label), "{label}");
19252 }
19253 let artifact = std::fs::read_to_string(crate::run::artifact_path(
19254 &runner.state,
19255 &crate::github_text::artifact_name(&record.fingerprint),
19256 ))
19257 .unwrap();
19258 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
19259 runner.state =
19261 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
19262 store
19263 .update(&q.id, |q| {
19264 q.say("まだ日本語")?;
19265 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
19266 })
19267 .unwrap();
19268 assert!(
19269 runner
19270 .guarded_pr_message(&winner, None, true)
19271 .await
19272 .unwrap()
19273 .is_none()
19274 );
19275 let retry = runner.state.github_text.as_ref().unwrap();
19276 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
19277 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
19278 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
19279 assert!(
19280 q.posting_comparison
19281 .as_ref()
19282 .unwrap()
19283 .matches_detail(&q.detail)
19284 );
19285 store
19286 .update(&q.id, |q| {
19287 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19288 })
19289 .unwrap();
19290 let message = runner
19291 .guarded_pr_message(&winner, None, true)
19292 .await
19293 .unwrap()
19294 .unwrap();
19295 assert_eq!(message.title, rewrite.title);
19296 assert_eq!(message.body, rewrite.body);
19297 assert_eq!(
19298 runner.state.github_text.as_ref().unwrap().choice,
19299 Some(GithubTextChoice::Rewrite)
19300 );
19301 let again = runner
19302 .guarded_pr_message(&winner, None, true)
19303 .await
19304 .unwrap()
19305 .unwrap();
19306 assert_eq!(again.title, message.title);
19307 assert_eq!(again.body, message.body);
19308 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
19309 assert!(
19310 !runner
19311 .state
19312 .events
19313 .iter()
19314 .any(|e| e.message.contains(&rewrite.title)
19315 || e.message.contains("English sentence."))
19316 );
19317 }
19318
19319 #[tokio::test]
19320 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
19321 crate::run::pin_test_home();
19322 let dir = tempfile::tempdir().unwrap();
19323 let mut runner = runner_at(RunStatus::Gating);
19324 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
19325 runner.state.repo = dir.path().to_owned();
19326 runner.state.candidates[0].worktree = dir.path().to_owned();
19327 seed_public_summary(&mut runner);
19328 let winner = runner.state.candidates[0].clone();
19329 assert!(
19330 runner
19331 .guarded_pr_message(&winner, None, true)
19332 .await
19333 .unwrap()
19334 .is_none()
19335 );
19336 let record = runner.state.github_text.as_ref().unwrap();
19337 assert!(record.rewrite.is_none());
19338 let mut old = serde_json::to_value(record).unwrap();
19340 old.as_object_mut().unwrap().remove("rewrite");
19341 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
19342 assert!(old.rewrite.is_none());
19343 let id = record.question.clone().unwrap();
19344 ask::Questions::open()
19345 .update(&id, |q| {
19346 q.choices.push(crate::github_text::USE_REWRITE.into());
19347 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
19348 })
19349 .unwrap();
19350 let message = runner
19351 .guarded_pr_message(&winner, None, true)
19352 .await
19353 .unwrap()
19354 .unwrap();
19355 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
19356 assert_eq!(
19357 runner.state.github_text.as_ref().unwrap().choice,
19358 Some(GithubTextChoice::Fallback)
19359 );
19360 }
19361
19362 #[tokio::test]
19363 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
19364 crate::run::pin_test_home();
19365 let dir = tempfile::tempdir().unwrap();
19366 let mut runner = runner_at(RunStatus::Gating);
19367 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
19368 runner.state.repo = dir.path().to_owned();
19369 runner.state.candidates[0].worktree = dir.path().to_owned();
19370 seed_public_summary(&mut runner);
19371 let winner = runner.state.candidates[0].clone();
19372 assert!(
19373 runner
19374 .guarded_pr_message(&winner, None, true)
19375 .await
19376 .unwrap()
19377 .is_none()
19378 );
19379 let store = ask::Questions::open();
19380 let find = |run: &str| {
19381 store
19382 .list()
19383 .into_iter()
19384 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
19385 .unwrap()
19386 };
19387 let run = runner.state.id.clone();
19388 let q = find(&run);
19389 let fp = runner
19390 .state
19391 .github_text
19392 .as_ref()
19393 .unwrap()
19394 .fingerprint
19395 .clone();
19396 let name = crate::github_text::artifact_name(&fp);
19397 let path = crate::run::artifact_path(&runner.state, &name);
19398 assert!(q.detail.contains(&path.display().to_string()));
19399 assert!(q.detail.contains("title-language: title"));
19400 let saved = std::fs::read_to_string(&path).expect("artifact written");
19401 assert!(saved.contains("日本語のタイトル"));
19402 assert!(q.cwd.is_none());
19403 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
19404 let say = |id: &str, text: &str, choice: &str| {
19406 store
19407 .update(id, |q| {
19408 q.thread.push(ask::Turn {
19409 who: ask::Who::Operator,
19410 body: text.to_owned(),
19411 at: jiff::Timestamp::now(),
19412 note: None,
19413 });
19414 q.answer(ask::Answer::Choice(choice.to_owned()))
19415 })
19416 .unwrap();
19417 };
19418 say(&q.id, "まだ日本語", "use my text");
19419 assert!(
19420 runner
19421 .guarded_pr_message(&winner, None, true)
19422 .await
19423 .unwrap()
19424 .is_none()
19425 );
19426 let second = find(&run);
19427 assert_ne!(second.id, q.id);
19428 assert!(second.detail.contains("did not pass"));
19429 assert!(second.detail.contains("日本語のタイトル"));
19432 assert!(!second.detail.contains("まだ日本語"));
19433 assert!(second.detail.contains(&path.display().to_string()));
19434 say(&second.id, "fix: retry failed requests", "use my text");
19435 let message = runner
19436 .guarded_pr_message(&winner, None, true)
19437 .await
19438 .unwrap()
19439 .expect("a vetted replacement is posted");
19440 assert_eq!(message.title, "fix: retry failed requests");
19441 assert_eq!(
19442 runner
19443 .state
19444 .github_text
19445 .as_ref()
19446 .unwrap()
19447 .chosen_title
19448 .as_deref(),
19449 Some("fix: retry failed requests")
19450 );
19451 assert!(runner.state.github_text.as_ref().unwrap().resolved);
19452 let again = runner
19454 .guarded_pr_message(&winner, None, true)
19455 .await
19456 .unwrap()
19457 .unwrap();
19458 assert_eq!(again.title, message.title);
19459 assert_eq!(
19460 store
19461 .list()
19462 .into_iter()
19463 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
19464 .count(),
19465 2
19466 );
19467 }
19468
19469 #[test]
19470 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
19471 let mut state = state_with_summary(
19475 "add retries",
19476 "TITLE: fix(web): batch reads\n- reads run.json once",
19477 );
19478 state.config.graph.language = "ja".to_owned();
19479 let m = pr_message(&state, 'A');
19480 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
19481 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
19482
19483 let task = "今回やってほしいこと: results projector を直す";
19486 let mut state = state_with_summary(task, "- no title line");
19487 state.config.graph.language = "ja".to_owned();
19488 let m = pr_message(&state, 'A');
19489 assert_eq!(
19490 m.title,
19491 format!("chore: land candidate A of run {}", state.id)
19492 );
19493 assert!(
19494 m.body.contains(&format!(
19495 "<summary>Original task</summary>\n\n{task}\n\n</details>"
19496 )),
19497 "{}",
19498 m.body
19499 );
19500 }
19501
19502 #[test]
19503 fn pr_message_scrubs_home_paths_and_addresses() {
19504 let state = state_with_summary(
19505 "fix it in /Users/someone/src/x",
19506 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
19507 );
19508 let m = pr_message(&state, 'A');
19509 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
19510 assert!(!m.body.contains(leak), "{}", m.body);
19511 }
19512 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
19513 }
19514
19515 #[test]
19516 fn pr_message_survives_a_task_that_closes_details() {
19517 let state = state_with_summary("a </details> b", "TITLE: fix: x");
19518 let m = pr_message(&state, 'A');
19519 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
19520 }
19521
19522 #[test]
19523 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
19524 let cmd = manual_merge_command(
19525 MergeStyle::Squash,
19526 Path::new("/repo"),
19527 "b",
19528 "fix: \"quoted\" $(x) `y`\n\nbody",
19529 );
19530 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
19531 }
19532
19533 #[test]
19534 fn manual_merge_command_matches_the_configured_style() {
19535 let repo = Path::new("/repo");
19536 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
19537
19538 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
19539 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
19540
19541 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
19542 assert_eq!(
19543 squash,
19544 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
19545 \"Merge magi run 0832 (candidate A)\""
19546 );
19547
19548 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
19549 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
19550 }
19551
19552 #[test]
19553 fn a_nudge_gets_a_quarter_of_the_budget() {
19554 assert_eq!(retry_budget(secs(1200), true), secs(300));
19556 assert_eq!(retry_budget(secs(3600), true), secs(900));
19557 }
19558
19559 #[test]
19560 fn a_resent_prompt_keeps_the_whole_budget() {
19561 assert_eq!(retry_budget(secs(1200), false), secs(1200));
19564 assert_eq!(retry_budget(secs(60), false), secs(60));
19565 }
19566
19567 #[test]
19568 fn the_floor_never_exceeds_the_original_budget() {
19569 assert_eq!(retry_budget(secs(60), true), secs(60));
19573 assert_eq!(retry_budget(secs(480), true), secs(120));
19574 assert_eq!(retry_budget(secs(0), true), secs(0));
19575 }
19576
19577 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
19578 agent::CommandEvidence {
19579 id: "item1".to_owned(),
19580 description: "cargo test".to_owned(),
19581 exit_code,
19582 result_summary: String::new(),
19583 source: "codex".to_owned(),
19584 }
19585 }
19586
19587 #[test]
19588 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
19589 assert!(!has_unconfirmed_command(&[]));
19593 }
19594
19595 #[test]
19596 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
19597 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
19601 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
19602 assert!(!has_unconfirmed_command(&[
19603 evidence(Some(0)),
19604 evidence(Some(101))
19605 ]));
19606 }
19607
19608 #[test]
19609 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
19610 assert!(has_unconfirmed_command(&[
19611 evidence(Some(0)),
19612 evidence(None)
19613 ]));
19614 }
19615
19616 #[test]
19617 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
19618 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19619 assert_eq!(
19620 verified_noop_claim(true, &[], text).as_deref(),
19621 Some("already fixed by b32cfc4, on main.")
19622 );
19623 }
19624
19625 #[test]
19626 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
19627 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19630 assert!(verified_noop_claim(false, &[], text).is_none());
19631 }
19632
19633 #[test]
19634 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
19635 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
19636 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
19637 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
19639 }
19640
19641 #[test]
19642 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
19643 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
19644 }
19645
19646 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
19649 runner.state.candidates = shape
19650 .iter()
19651 .enumerate()
19652 .map(|(i, &(empty, verified))| Candidate {
19653 index: i,
19654 label: (b'A' + i as u8) as char,
19655 agent: "sonnet".to_owned(),
19656 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
19657 worktree: PathBuf::from(format!("/wt/{i}")),
19658 summary: String::new(),
19659 stat: String::new(),
19660 files: 0,
19661 commits: 0,
19662 empty,
19663 failed: None,
19664 verified_noop: verified.map(str::to_owned),
19665 duration_ms: 0,
19666 folded: false,
19667 })
19668 .collect();
19669 }
19670
19671 #[test]
19672 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
19673 ask_test_home();
19674 let mut runner = runner_at(RunStatus::Implementing);
19675 set_candidates(
19676 &mut runner,
19677 &[
19678 (true, Some("already on main at b32cfc4")),
19679 (true, Some("same fix, see the existing test")),
19680 ],
19681 );
19682
19683 runner
19684 .after_implement()
19685 .expect("a verified no-op is not an error");
19686
19687 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
19688 }
19689
19690 #[test]
19691 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
19692 ask_test_home();
19693 let mut runner = runner_at(RunStatus::Implementing);
19694 set_candidates(
19698 &mut runner,
19699 &[(true, Some("already on main at b32cfc4")), (true, None)],
19700 );
19701
19702 let err = runner
19703 .after_implement()
19704 .expect_err("an unverified empty candidate must still fail the run");
19705
19706 assert!(
19707 err.to_string().contains("no candidate produced a change"),
19708 "{err}"
19709 );
19710 assert_eq!(runner.state.status, RunStatus::Failed);
19711 }
19712
19713 #[test]
19714 fn after_implement_still_fails_an_ordinary_all_empty_run() {
19715 ask_test_home();
19716 let mut runner = runner_at(RunStatus::Implementing);
19717 set_candidates(&mut runner, &[(true, None), (true, None)]);
19718
19719 let err = runner
19720 .after_implement()
19721 .expect_err("no candidate declared anything; this is an ordinary failure");
19722
19723 assert!(
19724 err.to_string().contains("no candidate produced a change"),
19725 "{err}"
19726 );
19727 assert_eq!(runner.state.status, RunStatus::Failed);
19728 }
19729}
19730
19731#[derive(Debug)]
19734struct GraftRestoreFailed(String);
19735
19736impl std::fmt::Display for GraftRestoreFailed {
19737 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
19738 write!(
19739 f,
19740 "could not restore the winner to its pre-graft commit: {}",
19741 self.0
19742 )
19743 }
19744}
19745
19746impl std::error::Error for GraftRestoreFailed {}
19747
19748#[cfg(test)]
19752mod resource_wait_tests {
19753 use super::*;
19754 use std::collections::BTreeMap;
19755
19756 fn sh(id: &str, script: &str) -> AgentSpec {
19757 AgentSpec {
19758 id: id.to_owned(),
19759 kind: crate::config::AgentKind::Command,
19760 model: None,
19761 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
19762 extra_args: Vec::new(),
19763 env: BTreeMap::new(),
19764 prompt_delivery: None,
19765 max_turn_secs: None,
19766 }
19767 }
19768
19769 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
19770 SeatJob {
19771 seat: SeatState::new("impl-A", &spec.id, 1),
19772 spec,
19773 cwd: dir.to_path_buf(),
19774 prompt: "do it".to_owned(),
19775 timeout,
19776 allow_write: true,
19777 sessions: false,
19778 artifacts: dir.join("art"),
19779 stem: "impl-A".to_owned(),
19780 handover: None,
19781 }
19782 }
19783
19784 fn state(resource_wait_secs: u64) -> RunState {
19785 let mut config = Config::default();
19786 config.graph.timeout_resource_wait = resource_wait_secs;
19787 RunState::new(
19788 PathBuf::from("/repo"),
19789 "main".to_owned(),
19790 "abc1234".to_owned(),
19791 "task".to_owned(),
19792 config,
19793 )
19794 }
19795
19796 fn hold(cache: &Path) -> crate::cache::Guard {
19797 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
19798 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
19799 crate::cache::AcquireOutcome::Acquired(g) => g,
19800 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
19801 }
19802 }
19803
19804 async fn run(
19805 jobs: Vec<SeatJob>,
19806 sem: Arc<Semaphore>,
19807 cache: Option<&Path>,
19808 st: &mut RunState,
19809 ) -> Vec<(usize, SeatState, AgentOutcome)> {
19810 let prompts = Prompts::default();
19811 let ctx = WaveCtx {
19812 run: "r",
19813 node: "implement",
19814 prompts: &prompts,
19815 cache,
19816 round: None,
19817 carry_seats: false,
19818 pauses: None,
19819 };
19820 wave(jobs, sem, &ctx, st, 0).await
19821 }
19822
19823 #[tokio::test]
19824 async fn verification_cache_wait_preserves_the_command_budget() {
19825 crate::run::pin_test_home();
19826 let tmp = tempfile::tempdir().unwrap();
19827 let cache = tmp.path().join("target");
19828 let guard = hold(&cache);
19829 let release = tokio::spawn(async move {
19830 tokio::time::sleep(Duration::from_millis(100)).await;
19831 guard.release();
19832 });
19833 let mut st = state(10);
19834 let budget = Duration::from_millis(50);
19835 let wait = repository_wait(&st, None);
19836 let (outcomes, _) = with_cache_lease(
19837 &mut st,
19838 Some(&cache),
19839 "gate",
19840 "gate",
19841 tmp.path(),
19842 "abc1234",
19843 budget,
19844 "test gate",
19845 &wait,
19846 |_, available| async move {
19847 assert_eq!(
19848 available, budget,
19849 "contention must not shorten verification"
19850 );
19851 (Vec::new(), false, Vec::new())
19852 },
19853 )
19854 .await
19855 .unwrap();
19856 release.await.unwrap();
19857 assert!(outcomes.is_empty(), "{outcomes:?}");
19858 assert!(
19859 st.events
19860 .iter()
19861 .any(|event| event.message.contains("waiting for"))
19862 );
19863 assert!(st.resource_wait.is_none());
19864 hold(&cache).release();
19866 }
19867
19868 #[tokio::test]
19869 async fn verification_cache_wait_expires_without_starting_commands() {
19870 crate::run::pin_test_home();
19871 let tmp = tempfile::tempdir().unwrap();
19872 let cache = tmp.path().join("target");
19873 let _held = hold(&cache);
19874 let mut st = state(0);
19875 let wait = repository_wait(&st, None);
19876 let (outcomes, _) = with_cache_lease(
19877 &mut st,
19878 Some(&cache),
19879 "gate",
19880 "gate",
19881 tmp.path(),
19882 "abc1234",
19883 Duration::from_secs(60),
19884 "test gate",
19885 &wait,
19886 |_, _| async { panic!("commands must not start without the cache lease") },
19887 )
19888 .await
19889 .unwrap();
19890 assert_eq!(outcomes.len(), 1);
19891 assert!(outcomes[0].resource_blocked);
19892 assert!(st.resource_wait.is_none());
19893 assert!(matches!(
19894 st.resource_block.unwrap().cause,
19895 crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
19896 ));
19897 }
19898
19899 #[tokio::test]
19900 async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
19901 crate::run::pin_test_home();
19902 let tmp = tempfile::tempdir().unwrap();
19903 std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
19904 let cache = tmp.path().join("cache");
19905 let mut state = state(21600);
19906 let wait = repository_wait(&state, None);
19907 let (outcomes, _) = with_cache_lease(
19908 &mut state,
19909 Some(&cache),
19910 "gate",
19911 "gate",
19912 tmp.path(),
19913 "head",
19914 Duration::from_secs(60),
19915 "gate",
19916 &wait,
19917 |_, _| async { panic!("freshness failure must not start verification") },
19918 )
19919 .await
19920 .unwrap();
19921 assert_eq!(outcomes.len(), 1);
19922 assert!(outcomes[0].resource_blocked);
19923 let block = state.resource_block.unwrap();
19924 assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
19925 assert!(!block.description().contains("timed out"));
19926 }
19927
19928 #[tokio::test]
19929 async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
19930 crate::run::pin_test_home();
19931 let tmp = tempfile::tempdir().unwrap();
19932 let cache = tmp.path().join("cache");
19933 let _held = hold(&cache);
19934 let mut state = state(21600);
19935 let pause = Pause::new();
19936 let interrupt = Pause::new();
19937 let cause = ParkCause::Operator;
19938 pause.park_for(cause.clone());
19939 let prompts = Prompts::default();
19940 let ctx = WaveCtx {
19941 run: "test",
19942 node: "fix",
19943 prompts: &prompts,
19944 cache: Some(&cache),
19945 round: None,
19946 carry_seats: false,
19947 pauses: Some((&pause, &interrupt)),
19948 };
19949 let result = wave(
19950 vec![job(
19951 sh("writer", "exit 99"),
19952 tmp.path(),
19953 Duration::from_secs(1),
19954 )],
19955 Arc::new(Semaphore::new(1)),
19956 &ctx,
19957 &mut state,
19958 0,
19959 )
19960 .await;
19961 assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
19962 assert_eq!(result[0].1.turns, 0);
19963 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19964 assert_eq!(
19965 state.resource_block.unwrap().cause,
19966 crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19967 );
19968 assert!(state.active.is_empty());
19969 }
19970
19971 #[tokio::test]
19972 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19973 crate::run::pin_test_home();
19974 let tmp = tempfile::tempdir().unwrap();
19975 let cache = tmp.path().join("target-a");
19976 let guard = hold(&cache);
19977 tokio::spawn(async move {
19979 tokio::time::sleep(Duration::from_millis(1500)).await;
19980 guard.release();
19981 });
19982 let mut st = state(60);
19983 let results = run(
19984 vec![job(
19985 sh("alpha", "echo ok"),
19986 tmp.path(),
19987 Duration::from_secs(1),
19988 )],
19989 Arc::new(Semaphore::new(1)),
19990 Some(&cache),
19991 &mut st,
19992 )
19993 .await;
19994 assert!(
19995 matches!(results[0].2, AgentOutcome::Ok(_)),
19996 "the wait must not be charged to the agent"
19997 );
19998 assert!(st.resource_block.is_none());
19999 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20000 assert!(st.resource_wait.is_none() && st.active.is_empty());
20001 }
20002
20003 #[tokio::test]
20004 #[cfg(unix)]
20005 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
20006 use std::os::fd::AsRawFd as _;
20007 crate::run::pin_test_home();
20008 let tmp = tempfile::tempdir().unwrap();
20009 crate::git::git(tmp.path(), &["init"]).await.unwrap();
20010 let common = tmp.path().join(".git");
20011 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20012 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20013 assert_eq!(
20014 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20015 0
20016 );
20017 let mut st = state(1);
20018 st.config.coordination = crate::config::test_coordination();
20019 let mut reader = job(
20020 sh("reader", "echo must-not-run"),
20021 tmp.path(),
20022 Duration::from_secs(30),
20023 );
20024 reader.allow_write = false;
20025 let results = run(
20026 vec![reader.clone()],
20027 Arc::new(Semaphore::new(1)),
20028 None,
20029 &mut st,
20030 )
20031 .await;
20032 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20033 assert_eq!(results[0].1.turns, 0);
20034 assert!(FailClass::of(&results[0].2).is_none());
20035 assert_eq!(
20036 st.resource_block.as_ref().unwrap().kind,
20037 "repository coordination"
20038 );
20039 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20040 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20041 drop(external);
20042 let editor = crate::repo_lock::SeatLease::for_checkout(
20043 tmp.path(),
20044 true,
20045 crate::config::test_coordination().as_ref(),
20046 )
20047 .await
20048 .unwrap();
20049 let pause = Pause::new();
20050 let interrupt = Pause::new();
20051 let prompts = Prompts::default();
20052 let ctx = WaveCtx {
20053 run: "r",
20054 node: "review",
20055 prompts: &prompts,
20056 cache: None,
20057 round: None,
20058 carry_seats: false,
20059 pauses: Some((&pause, &interrupt)),
20060 };
20061 let (results, ()) = tokio::join!(
20062 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
20063 async {
20064 tokio::time::sleep(Duration::from_millis(50)).await;
20065 pause.park();
20066 }
20067 );
20068 assert!(
20069 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
20070 );
20071 assert_eq!(results[0].1.turns, 0);
20072 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
20073 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
20074 drop(editor);
20075 }
20076
20077 #[tokio::test]
20078 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
20079 crate::run::pin_test_home();
20080 let tmp = tempfile::tempdir().unwrap();
20081 let cache = tmp.path().join("target-b");
20082 let _held = hold(&cache);
20083 let mut st = state(1);
20084 let results = run(
20085 vec![job(
20086 sh("alpha", "echo ok"),
20087 tmp.path(),
20088 Duration::from_secs(1),
20089 )],
20090 Arc::new(Semaphore::new(1)),
20091 Some(&cache),
20092 &mut st,
20093 )
20094 .await;
20095 let (_, seat, out) = &results[0];
20096 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
20097 assert!(wave_blocked(&results));
20098 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
20099 assert_eq!(seat.turns, 0, "nothing was asked");
20100 assert!(
20101 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
20102 "no agent was spawned"
20103 );
20104 assert!(FailClass::of(out).is_none(), "not a failure class");
20105 let block = st.resource_block.as_ref().expect("recorded for the resume");
20106 assert_eq!(block.kind, "shared build cache");
20107 assert!(matches!(
20108 block.cause,
20109 crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
20110 ));
20111 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
20112 assert!(st.active.is_empty());
20113 }
20114
20115 #[tokio::test]
20116 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
20117 crate::run::pin_test_home();
20118 let tmp = tempfile::tempdir().unwrap();
20119 let mut st = state(1);
20120 let results = run(
20121 vec![job(
20122 sh("alpha", "echo ok"),
20123 tmp.path(),
20124 Duration::from_secs(1),
20125 )],
20126 Arc::new(Semaphore::new(0)),
20127 None,
20128 &mut st,
20129 )
20130 .await;
20131 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
20132 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
20133 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
20134 }
20135
20136 #[tokio::test]
20137 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
20138 crate::run::pin_test_home();
20139 let tmp = tempfile::tempdir().unwrap();
20140 let mut st = state(60);
20141 let results = run(
20142 vec![job(
20143 sh("alpha", "sleep 30"),
20144 tmp.path(),
20145 Duration::from_secs(1),
20146 )],
20147 Arc::new(Semaphore::new(1)),
20148 None,
20149 &mut st,
20150 )
20151 .await;
20152 let out = &results[0].2;
20153 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
20154 let class = FailClass::of(out).expect("a failure");
20155 assert_eq!(class, FailClass::Timeout);
20156 assert!(should_hand_over(None, &class), "existing handover policy");
20157 assert!(st.resource_block.is_none());
20158 }
20159}