1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
188}
189
190#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220 #[must_use]
222 pub fn new() -> Self {
223 Self::default()
224 }
225
226 pub fn park(&self) {
228 self.0.store(true, Ordering::SeqCst);
229 }
230
231 pub fn park_because(&self, reason: impl Into<String>) {
237 self.park_for(ParkCause::Interrupt {
238 reason: reason.into(),
239 });
240 }
241
242 pub fn park_for(&self, cause: ParkCause) {
245 let mut reason_guard = self
246 .1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner);
249 if reason_guard.is_none() {
250 *reason_guard = Some(cause);
251 }
252 drop(reason_guard);
253 self.park();
254 }
255
256 #[must_use]
258 pub fn parked(&self) -> bool {
259 self.0.load(Ordering::SeqCst)
260 }
261
262 #[must_use]
264 pub fn reason(&self) -> Option<String> {
265 self.cause().map(|c| c.label())
266 }
267
268 #[must_use]
270 pub fn cause(&self) -> Option<ParkCause> {
271 self.1
272 .lock()
273 .unwrap_or_else(std::sync::PoisonError::into_inner)
274 .clone()
275 }
276}
277
278pub struct Runner {
280 pub state: RunState,
282 roles: ResolvedRoles,
283 sem: Arc<Semaphore>,
284 pause: Pause,
288 interrupt: Pause,
294}
295
296async fn review_base(
301 repo: &Path,
302 remote: &str,
303 base_branch: &str,
304 base_commit: &str,
305 branch: &str,
306) -> String {
307 review_base_checked(repo, remote, base_branch, base_commit, branch)
308 .await
309 .0
310}
311
312async fn review_base_checked(
317 repo: &Path,
318 remote: &str,
319 base_branch: &str,
320 base_commit: &str,
321 branch: &str,
322) -> (String, bool) {
323 let tracking = format!("{remote}/{base_branch}");
324 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325 if git::rev_exists(repo, &tracking).await
326 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327 && !mb.is_empty()
328 {
329 return (mb, fresh);
330 }
331 let mb = git::merge_base(repo, base_commit, branch)
332 .await
333 .ok()
334 .filter(|mb| !mb.is_empty())
335 .unwrap_or_else(|| base_commit.to_owned());
336 (mb, false)
337}
338
339pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343 if !is_review_run(state) {
344 return;
345 }
346 let base = review_base(
347 &state.repo,
348 &state.config.merge.remote,
349 &state.base_branch,
350 &state.base_commit,
351 branch,
352 )
353 .await;
354 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355 && !subjects.is_empty()
356 && state.reviewed_commits.as_ref() != Some(&subjects)
357 {
358 state.reviewed_commits = Some(subjects);
359 state.save().ok();
360 }
361}
362
363async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367 let (base, trusted) = review_base_checked(
368 &state.repo,
369 &state.config.merge.remote,
370 &state.base_branch,
371 &state.base_commit,
372 branch,
373 )
374 .await;
375 if !trusted {
376 return None;
377 }
378 git::subjects(&state.repo, &state.base_commit, &base)
379 .await
380 .ok()
381}
382
383pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388 if is_leak(computed) {
389 return false;
390 }
391 let cur = current.trim();
392 cur.is_empty()
393 || cur.starts_with(REVIEW_PROMPT_OPENING)
394 || cur.starts_with("chore: land candidate")
395 || cur.starts_with("magi: candidate")
396 || is_leak(cur)
397}
398
399async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428 let tracking = format!("{remote}/{branch}");
429 let fetched = git::fetch(repo, remote, branch).await;
430 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431 let local_exists = git::branch_exists(repo, branch).await?;
432 if !fresh {
433 if !local_exists {
434 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435 }
436 tracing::warn!(
437 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438 );
439 return Ok(());
440 }
441 let remote_sha = git::rev_parse(repo, &tracking).await?;
442 if !local_exists {
443 git::git(repo, &["branch", branch, &tracking]).await?;
444 return Ok(());
445 }
446 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448 return Ok(());
449 }
450 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457 .await?
458 {
459 crate::reconcile::Reconciliation::Pushed => {
460 tracing::warn!(
461 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462 short(&local_sha),
463 short(&remote_sha)
464 );
465 return Ok(());
466 }
467 crate::reconcile::Reconciliation::Placeholder => {}
468 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469 }
470 }
471 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472 if !out.ok() {
473 bail!(
474 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475 short(&local_sha),
476 short(&remote_sha),
477 out.stderr
478 );
479 }
480 tracing::warn!(
481 "local `{branch}` was stale: fast-forwarded {} -> {}",
482 short(&local_sha),
483 short(&remote_sha)
484 );
485 Ok(())
486}
487
488async fn resolve_base(
489 repo: &Path,
490 base_branch: &str,
491 remote: &str,
492 coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494 let _lease = crate::repo_lock::SeatLease::operation_with(
495 repo,
496 coordination,
497 &crate::repo_lock::WaitPolicy::current(),
498 )
499 .await?;
500 let tracking = format!("{remote}/{base_branch}");
501 let fetched = git::fetch(repo, remote, base_branch).await;
502 if let Ok(out) = &fetched
503 && out.ok()
504 && git::rev_exists(repo, &tracking).await
505 {
506 return Ok(crate::run::FetchedBase {
507 remote: remote.to_owned(),
508 branch: base_branch.to_owned(),
509 commit: git::rev_parse(repo, &tracking).await?,
510 fetched_at: jiff::Timestamp::now(),
511 });
512 }
513 let why = match &fetched {
514 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515 Ok(_) => format!("{remote} has no {base_branch}"),
516 Err(e) => e.to_string(),
517 };
518 bail!(
521 "cannot read {tracking} ({why}); refusing to branch off the local \
522 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523 base / remote in magi.toml. This is launcher preparation: check the \
524 launcher's Git administration access, network and repository coordination \
525 before retrying; an unattended seat cannot approve those permissions"
526 )
527}
528
529struct FixClaim {
546 path: PathBuf,
547}
548
549impl FixClaim {
550 fn acquire(dir: &Path) -> Result<Self> {
551 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552 let path = dir.join("fix.lock");
553 match Self::create(&path) {
554 Ok(claim) => Ok(claim),
555 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
556 if Self::reclaim_if_dead(&path) {
557 Self::create(&path).with_context(|| format!("lock {}", path.display()))
558 } else {
559 bail!(
560 "another repair or `magi fix` is already running for this run ({} exists)",
561 path.display()
562 )
563 }
564 }
565 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
566 }
567 }
568
569 fn create(path: &Path) -> std::io::Result<Self> {
570 let mut f = std::fs::OpenOptions::new()
571 .write(true)
572 .create_new(true)
573 .open(path)?;
574 use std::io::Write as _;
575 writeln!(f, "{}", std::process::id())?;
577 Ok(Self {
578 path: path.to_owned(),
579 })
580 }
581
582 fn reclaim_if_dead(path: &Path) -> bool {
586 let dead = std::fs::read_to_string(path)
587 .ok()
588 .and_then(|body| body.trim().parse::<u32>().ok())
589 .is_some_and(|pid| !crate::proc::pid_alive(pid));
590 dead && std::fs::remove_file(path).is_ok()
591 }
592}
593
594impl Drop for FixClaim {
595 fn drop(&mut self) {
596 let _ = std::fs::remove_file(&self.path);
597 }
598}
599
600struct DriverExecution<'a> {
603 runner: &'a mut Runner,
604 driver: crate::driver::DriverGuard,
605}
606
607impl Drop for DriverExecution<'_> {
608 fn drop(&mut self) {
609 self.driver.release_local(&mut self.runner.state);
610 }
611}
612
613impl Runner {
614 pub async fn start(
616 repo: &Path,
617 instruction: String,
618 config: Config,
619 origin: Origin,
620 ) -> Result<Self> {
621 Self::start_naming(repo, instruction, "", config, origin).await
622 }
623
624 pub async fn start_naming(
628 repo: &Path,
629 instruction: String,
630 also_scan: &str,
631 config: Config,
632 origin: Origin,
633 ) -> Result<Self> {
634 let repo = git::toplevel(repo).await?;
635 let missing = agent::missing_programs(&config.agents);
636 if !missing.is_empty() {
637 bail!(
638 "these agent programs are not on PATH: {}. Fix the roster in \
639 magi.toml or install them.",
640 missing.join(", ")
641 );
642 }
643 let base_branch =
644 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
645 .await?;
646 let fetched_base = resolve_base(
647 &repo,
648 &base_branch,
649 &config.merge.remote,
650 config.coordination.as_ref(),
651 )
652 .await?;
653 let base_commit = fetched_base.commit.clone();
654 let roles = config.resolve_roles()?;
655 let max_parallel = config.graph.max_parallel.max(1);
656 let seeds = refs::resolve(
659 &repo,
660 &base_commit,
661 &config.merge.remote,
662 &format!("{also_scan}\n{instruction}"),
663 )
664 .await;
665 refs::plan(&repo, &seeds).await?;
666 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
667 state.fetched_base = Some(fetched_base);
671 state.origin = Some(origin);
672 for seed in &seeds {
673 state.event(
674 "seed",
675 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
676 );
677 }
678 state.seeds = seeds;
679 state.event("start", format!("run {} created", state.id));
680 state.save()?;
681 Ok(Self {
682 state,
683 roles,
684 sem: Arc::new(Semaphore::new(max_parallel)),
685 pause: Pause::new(),
686 interrupt: Pause::new(),
687 })
688 }
689
690 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
704 Self::review_taking_over(repo, branch, config, None, origin).await
705 }
706
707 pub async fn review_taking_over(
714 repo: &Path,
715 branch: &str,
716 config: Config,
717 takeover: Option<crate::handover::Takeover>,
718 origin: Origin,
719 ) -> Result<Self> {
720 let repo = git::toplevel(repo).await?;
721 let missing = agent::missing_programs(&config.agents);
722 if !missing.is_empty() {
723 bail!(
724 "these agent programs are not on PATH: {}. Fix the roster in \
725 magi.toml or install them.",
726 missing.join(", ")
727 );
728 }
729 let base_branch =
730 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
731 .await?;
732 if base_branch == branch {
733 bail!("`{branch}` is the base branch; there is nothing to review against");
734 }
735 let fetched_base = resolve_base(
736 &repo,
737 &base_branch,
738 &config.merge.remote,
739 config.coordination.as_ref(),
740 )
741 .await?;
742 let base_commit = fetched_base.commit.clone();
743
744 let roles = config.resolve_roles()?;
745 let max_parallel = config.graph.max_parallel.max(1);
746 let mut state = RunState::new(
747 repo.clone(),
748 base_branch,
749 base_commit.clone(),
750 String::new(),
751 config,
752 );
753 state.fetched_base = Some(fetched_base);
754 state.origin = Some(origin);
755
756 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
763 earlier: Vec::new(),
764 home: crate::run::home(),
765 choice: None,
766 task_instruction: None,
767 });
768 let wait = repository_wait(&state, None);
769 let coordination_config = state.config.coordination.clone();
770 let coordination =
771 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
772 .await?;
773 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
774 if let Some(released) = &released {
775 state.event(
776 "release",
777 format!(
778 "took `{branch}` over from run {}: its worktree was released: {}",
779 crate::run::short_of(&released.old_id),
780 released.audit
781 ),
782 );
783 }
784 if let Some(choice) = takeover.choice.as_ref()
788 && let Err(e) =
789 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
790 .await
791 {
792 if let Some(released) = &released {
793 released.restore(&repo, branch).await;
794 }
795 return Err(e.context("applying the owner's answer about the diverged branch"));
796 }
797 for id in takeover.earlier.iter().rev() {
803 if let Ok(old) = RunState::load_under(id, &takeover.home)
804 && let Some(head) = old.pr_head.clone()
805 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
806 {
807 state.pr_head = Some(head);
808 break;
809 }
810 }
811 drop(coordination);
812 let opened = Self::open_review(
813 &repo,
814 branch,
815 state,
816 roles,
817 max_parallel,
818 base_commit,
819 &takeover,
820 )
821 .await;
822 if opened.is_err()
823 && let Some(released) = &released
824 {
825 let _coordination = crate::repo_lock::SeatLease::operation_with(
826 &repo,
827 coordination_config.as_ref(),
828 &wait,
829 )
830 .await?;
831 released.restore(&repo, branch).await;
832 }
833 opened
834 }
835
836 pub async fn recover_pr(
838 repo: &Path,
839 task: &str,
840 selection: crate::pr_recovery::Selection,
841 mut config: Config,
842 ) -> Result<Self> {
843 selection.configure(&mut config)?;
844 selection.check_task(task, &selection.instruction)?;
845 let base_branch =
846 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
847 .await?;
848 let fetched_base = resolve_base(
849 repo,
850 &base_branch,
851 &config.merge.remote,
852 config.coordination.as_ref(),
853 )
854 .await?;
855 let base_commit = fetched_base.commit.clone();
856 selection
857 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
858 .await?;
859 Box::pin(selection.verify_source(repo)).await?;
860 if let Some(proof) = &selection.source_proof {
861 anyhow::ensure!(
862 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
863 "source rebase base is outside the fetched landing base"
864 );
865 }
866 let roles = config.resolve_roles()?;
867 let max_parallel = config.graph.max_parallel.max(1);
868 let mut state = RunState::new(
869 repo.to_path_buf(),
870 base_branch,
871 base_commit.clone(),
872 selection.instruction.clone(),
873 config,
874 );
875 state.fetched_base = Some(fetched_base);
876 state.origin = Some(crate::run::Origin::queue(task));
877 state.pr_head = Some(selection.branch.clone());
878 state.pr_recovery = Some(selection.clone());
879 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
880 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
881 }
882
883 async fn open_selected(
884 repo: &Path,
885 state: RunState,
886 roles: ResolvedRoles,
887 max_parallel: usize,
888 base_commit: String,
889 selection: crate::pr_recovery::Selection,
890 ) -> Result<Self> {
891 let mut state = state;
892 let branch = state.branch_for('A');
893 let wait = repository_wait(&state, None);
894 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
895 git::git(repo, &["branch", &branch, selection.source()]).await?;
896 drop(coordination);
897 state.event(
898 "recovery",
899 format!(
900 "selected implementation {} for {} (public lease {})",
901 selection.source(),
902 selection.url,
903 selection.head
904 ),
905 );
906 let takeover = crate::handover::Takeover {
907 earlier: Vec::new(),
908 home: crate::run::home(),
909 choice: None,
910 task_instruction: Some(selection.instruction.clone()),
911 };
912 state.save()?;
913 let id = state.id.clone();
914 let opened = Self::open_review(
915 repo,
916 &branch,
917 state,
918 roles,
919 max_parallel,
920 base_commit,
921 &takeover,
922 )
923 .await;
924 match opened {
925 Ok(runner) => Ok(runner),
926 Err(e) => {
927 if let Ok(mut partial) = RunState::load(&id) {
928 partial.status = RunStatus::Blocked;
929 partial.driver_exited = true;
930 partial.event(
931 "recovery",
932 format!("startup failed; retained source and candidate {branch}: {e:#}"),
933 );
934 partial.save()?;
935 }
936 Err(crate::pr_recovery::StartupFailed {
937 run: id,
938 cause: format!("{e:#}"),
939 }
940 .into())
941 }
942 }
943 }
944
945 async fn inherit(
950 repo: &Path,
951 branch: &str,
952 base_commit: &str,
953 takeover: &crate::handover::Takeover,
954 ) -> Option<crate::recovery::RecoveryRecord> {
955 use crate::recovery::{Relation, assess};
956 let newest = takeover.earlier.last()?;
957 let mut chosen = None;
958 for id in takeover.earlier.iter().rev() {
959 if let Ok(old) = RunState::load_under(id, &takeover.home)
960 && !old.reviews.is_empty()
961 && (old.winner().is_some_and(|w| w.branch == branch)
962 || old.pr_head.as_deref() == Some(branch))
963 {
964 chosen = Some(old);
965 break;
966 }
967 }
968 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
969 .await
970 .unwrap_or_default();
971 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
972 Some(last) if last.head == tip => Relation::Same,
973 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
974 _ => Relation::Diverged,
975 };
976 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
977 Some(assess(from, chosen.as_ref(), base_commit, relation))
978 }
979
980 async fn open_review(
983 repo: &Path,
984 branch: &str,
985 mut state: RunState,
986 roles: ResolvedRoles,
987 max_parallel: usize,
988 base_commit: String,
989 takeover: &crate::handover::Takeover,
990 ) -> Result<Self> {
991 let wait = repository_wait(&state, None);
992 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
993 if state.pr_recovery.is_none() {
994 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
995 }
996 let start = review_base(
999 repo,
1000 &state.config.merge.remote,
1001 &state.base_branch,
1002 &base_commit,
1003 branch,
1004 )
1005 .await;
1006 let log = git::log_oneline(repo, &start, branch)
1007 .await
1008 .unwrap_or_default();
1009 let log_text = if log.trim().is_empty() {
1010 "(no commit messages)"
1011 } else {
1012 log.trim()
1013 };
1014 let mut instruction = format!(
1015 "Review the work already on branch `{branch}`. There is no task \
1016 statement: what the change claims to do is whatever its commits \
1017 say.\n\n{log_text}"
1018 );
1019 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1023 if let Some(task) = takeover
1024 .task_instruction
1025 .as_deref()
1026 .filter(|t| !t.trim().is_empty())
1027 && rec.reviewed_head.is_some()
1028 {
1029 instruction = format!(
1030 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1031 task.trim()
1032 );
1033 rec.criteria_inherited = true;
1034 }
1035 state.event("recovery", rec.summary());
1036 state.recovery = Some(rec);
1037 }
1038 if let Some(selected) = &state.pr_recovery {
1039 instruction = selected.instruction.clone();
1040 }
1041 state.task_instruction = takeover.task_instruction.clone();
1042 state.instruction = instruction;
1043 state.reviewed_commits = Some(
1044 git::subjects(repo, &start, branch)
1045 .await
1046 .unwrap_or_default(),
1047 );
1048
1049 let worktree = state.worktree_root().join("under-review");
1052 if let Some(parent) = worktree.parent() {
1053 tokio::fs::create_dir_all(parent).await.ok();
1054 }
1055 let path = worktree.to_string_lossy().to_string();
1056 git::git(repo, &["worktree", "add", &path, branch])
1057 .await
1058 .with_context(|| {
1059 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1060 })?;
1061
1062 let setup = match ensure_setup_config(&mut state, repo).await {
1063 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1064 Err(e) => Err(e),
1065 };
1066 if let Err(e) = setup {
1067 git::worktree_remove_clean(repo, &worktree).await.ok();
1068 return Err(e);
1069 }
1070 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1071 .await
1072 .unwrap_or(0);
1073 if commits == 0 {
1074 git::worktree_remove_clean(repo, &worktree).await.ok();
1075 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1076 }
1077 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1078 .await
1079 .map(|f| f.len())
1080 .unwrap_or(0);
1081 if files == 0
1082 && let (Ok(head_tree), Ok(base_tree)) = (
1083 git::tree_of(&worktree, "HEAD").await,
1084 git::tree_of(&worktree, &base_commit).await,
1085 )
1086 && head_tree == base_tree
1087 {
1088 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1089 git::worktree_remove_clean(repo, &worktree).await.ok();
1090 bail!(
1091 "`{branch}` at {} has a tree identical to base {}; this usually means \
1092 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1093 against `{}/{branch}`) rather than an empty change",
1094 short(&head),
1095 short(&base_commit),
1096 state.config.merge.remote
1097 );
1098 }
1099 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1100 .await
1101 .unwrap_or_default();
1102
1103 state.candidates.push(Candidate {
1104 index: 0,
1105 label: 'A',
1106 agent: EXISTING_BRANCH.to_owned(),
1109 branch: branch.to_owned(),
1110 worktree,
1111 summary: String::new(),
1112 stat,
1113 files,
1114 commits,
1115 empty: false,
1116 failed: None,
1117 verified_noop: None,
1118 duration_ms: 0,
1119 folded: false,
1120 });
1121 state.tally = Some(Tally {
1122 first_choice: BTreeMap::from([('A', 0)]),
1123 borda: BTreeMap::new(),
1124 winner: 'A',
1125 rankings: 0,
1126 unanimous_initial: false,
1127 deliberated: false,
1128 changed_votes: 0,
1129 unanimous_final: false,
1130 tie_break: None,
1131 judges: 0,
1135 present: 0,
1136 quorum: 0,
1137 met_quorum: true,
1138 uncontested: Some("review-only run: nothing competed".to_owned()),
1139 });
1140 state.status = RunStatus::Reviewing;
1141 state.prep_complete = Some(true);
1142 state.event(
1143 "start",
1144 format!(
1145 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1146 state.id
1147 ),
1148 );
1149 state.save()?;
1150 Ok(Self {
1151 state,
1152 roles,
1153 sem: Arc::new(Semaphore::new(max_parallel)),
1154 pause: Pause::new(),
1155 interrupt: Pause::new(),
1156 })
1157 }
1158
1159 pub fn resume(id: &str) -> Result<Self> {
1161 let state = RunState::load(id)?;
1162 crate::pr_head::ensure_owner(&state)?;
1163 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1164 bail!(
1165 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1166 );
1167 }
1168 if let Some(to) = &state.released_to {
1169 bail!(
1170 "run {} cannot be resumed: its worktree was released to run {}",
1171 state.short(),
1172 crate::run::short_of(to)
1173 );
1174 }
1175 let roles = state.config.resolve_roles()?;
1176 if let Some(id) = &state.config.graph.selected_implementer
1177 && state.candidates.iter().any(|c| c.agent != *id)
1178 {
1179 bail!(
1180 "replacement implementer pin does not match the saved candidates; task must stay held"
1181 );
1182 }
1183 let max_parallel = state.config.graph.max_parallel.max(1);
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub async fn execute(&mut self) -> Result<()> {
1200 if let Some(selection) = &self.state.pr_recovery
1201 && (selection.source_proof.is_some() || selection.source_run.is_some())
1202 {
1203 Box::pin(selection.verify_source(&self.state.repo)).await?;
1204 }
1205 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1208 || self.state.land_approval.is_some()
1209 || self.state.merge_decision.is_some()
1210 || !self.state.merge_repairs.is_empty()
1211 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1212 {
1213 let claim = FixClaim::acquire(&self.state.dir())?;
1214 let disk = RunState::load(&self.state.id)?;
1215 let checkpoint = |s: &RunState| {
1216 (
1217 s.merge_repairs.len(),
1218 s.merge_repairs
1219 .last()
1220 .map(|r| (r.question.clone(), r.phase)),
1221 s.land_approval
1222 .as_ref()
1223 .map(|a| (a.question.clone(), a.head.clone())),
1224 )
1225 };
1226 if checkpoint(&disk) != checkpoint(&self.state) {
1227 bail!(
1228 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1229 );
1230 }
1231 Some(claim)
1232 } else {
1233 None
1234 };
1235 let mut execution = self.claim_driver()?;
1236 let runner = &mut *execution.runner;
1237 let wait = runner.repository_wait();
1238 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1241 let result = match result {
1242 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1243 record_repository_block(&mut runner.state, &e);
1244 runner.park_here()?;
1245 Ok(())
1246 }
1247 other => other,
1248 };
1249 execution.driver.finish(&mut runner.state)?;
1250 let ended = if result.is_err() {
1251 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1252 } else {
1253 crate::notices::run_ended(&runner.state)
1254 };
1255 if let Some(notice) = ended {
1256 crate::notices::raise(notice);
1257 }
1258 result
1259 }
1260
1261 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1262 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1263 }
1264
1265 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1266 let wait = self.repository_wait();
1267 coordinate_operation(&mut self.state, cwd, &wait).await
1268 }
1269
1270 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1272 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1273 Ok(owner) => owner,
1274 Err(error) => {
1275 let detail = format!("worktree maintenance or ownership update: {error:#}");
1276 self.state.resource_block = Some(crate::run::ResourceBlock {
1280 node: "start".into(),
1281 kind: "worktree maintenance".into(),
1282 detail: detail.clone(),
1283 at: Timestamp::now(),
1284 });
1285 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1286 }
1287 };
1288 let disk = RunState::load(&self.state.id)?;
1289 crate::pr_head::ensure_owner(&disk)?;
1290 if disk.liveness(false) == crate::run::Liveness::Live {
1291 bail!("run already has a live driver; wait for its execution to end");
1292 }
1293 if let Some(to) = &disk.released_to {
1294 bail!(
1295 "run {} cannot continue: its worktree was released to run {}",
1296 self.state.short(),
1297 crate::run::short_of(to)
1298 );
1299 }
1300 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1301 let mut execution = DriverExecution {
1302 runner: self,
1303 driver,
1304 };
1305 let _owner = owner;
1308 let state = &mut execution.runner.state;
1309 state.parked = false;
1314 state.resource_wait = None;
1315 state.resource_block = None;
1316 state.clear_active();
1323 let pid = std::process::id();
1338 state.driver_pid = Some(pid);
1339 state.driver_started_at = crate::proc::process_started_at(pid);
1340 state.driver_exited = false;
1341 state.begin_pass();
1342 execution.driver.bind(state);
1343 state.save()?;
1344 execution.driver.record(state)?;
1345 Ok(execution)
1346 }
1347
1348 async fn execute_graph(&mut self) -> Result<()> {
1349 self.state.configure_recovery_resume()?;
1350 self.persist_candidates().await?;
1353 if self
1354 .state
1355 .merge_decision
1356 .as_ref()
1357 .is_some_and(|decision| !decision.closed)
1358 && self.state.status != RunStatus::Landing
1359 {
1360 if crate::merge_decision::resumable_hold(&self.state) {
1361 self.state.merge = Some(MergeOutcome {
1362 mode: MergeMode::Pr,
1363 ok: false,
1364 detail: self
1365 .state
1366 .merge_decision
1367 .as_ref()
1368 .unwrap()
1369 .target
1370 .pr
1371 .clone(),
1372 empty: false,
1373 });
1374 self.run_land().await?;
1375 self.settle_questions();
1376 return Ok(());
1377 }
1378 bail!(
1379 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1380 );
1381 }
1382 if self.state.config.graph.selected_implementer.is_some()
1383 && self.state.status == RunStatus::Blocked
1384 && self.state.tally.is_none()
1385 && self.state.candidate_preservation.is_empty()
1386 && self.state.interrupted_implementations.is_empty()
1387 {
1388 bail!(
1389 "selected implementation is held; explicitly release the owning task to retry it"
1390 );
1391 }
1392
1393 if self.state.merge_repairs.last().is_some_and(|r| {
1394 matches!(
1395 r.phase,
1396 crate::run::MergeRepairPhase::Requested
1397 | crate::run::MergeRepairPhase::Fixing
1398 | crate::run::MergeRepairPhase::Reviewing
1399 )
1400 }) {
1401 if self.repair_merge_review().await? {
1402 self.run_land().await?;
1403 }
1404 self.settle_questions();
1405 return Ok(());
1406 }
1407 if self.state.pr_publication_pending {
1410 self.merge().await?;
1411 if self.resource_halt()? {
1412 return Ok(());
1413 }
1414 self.state.save()?;
1415 self.settle_questions();
1416 return Ok(());
1417 }
1418 if self.state.status == RunStatus::Stalled {
1431 if self.recover_stall().await? {
1432 self.finish_after_tally().await?;
1433 } else {
1434 self.state.save()?;
1436 }
1437 return Ok(());
1438 }
1439 if self.state.status == RunStatus::Landing
1449 || crate::handover::landing_checkpoint(&self.state)
1450 {
1451 self.state.land_sync_pending = true;
1452 self.state.save()?;
1453 self.run_land().await?;
1454 self.settle_questions();
1459 return Ok(());
1460 }
1461 if self.state.github_text.is_some()
1466 && self.state.merge.is_none()
1467 && self.state.status == RunStatus::Gating
1468 {
1469 self.merge().await?;
1470 if self.resource_halt()? {
1471 return Ok(());
1472 }
1473 self.state.save()?;
1474 return Ok(());
1475 }
1476 if let Err(e) = self.prep().await {
1477 if e.is::<crate::repo_lock::ResourceBlocked>() {
1478 return Err(e);
1479 }
1480 self.state.status = RunStatus::Blocked;
1481 self.state
1482 .prep_failure
1483 .get_or_insert_with(|| format!("{e:#}"));
1484 self.state
1485 .event("prep", format!("preparation stopped: {e:#}"));
1486 self.state.save()?;
1487 return Err(e);
1488 }
1489 if self.park_here()? {
1490 return Ok(());
1491 }
1492 self.advise().await?;
1493 if self.park_here()? {
1494 return Ok(());
1495 }
1496 self.implement().await?;
1497 if self.park_here()? {
1498 return Ok(());
1499 }
1500 if self.state.status == RunStatus::VerifiedNoop {
1504 return Ok(());
1505 }
1506 self.judge().await?;
1507 if self.park_here()? {
1508 return Ok(());
1509 }
1510 self.deliberate().await?;
1511 if self.park_here()? {
1512 return Ok(());
1513 }
1514 self.vote().await?;
1515 if self.park_here()? {
1516 return Ok(());
1517 }
1518 self.tally()?;
1519 if self.state.status == RunStatus::Stalled {
1524 self.state.save()?;
1528 return Ok(());
1529 }
1530 self.finish_after_tally().await?;
1531 Ok(())
1532 }
1533
1534 fn stop_for_resource(&mut self) -> Result<()> {
1539 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1541 {
1542 self.state.status = RunStatus::Blocked;
1543 }
1544 self.state.save()
1545 }
1546
1547 fn resource_halt(&mut self) -> Result<bool> {
1549 if self.state.resource_block.is_none() {
1550 return Ok(false);
1551 }
1552 self.stop_for_resource()?;
1553 Ok(true)
1554 }
1555
1556 fn park_here(&mut self) -> Result<bool> {
1563 let parking = self.pause.parked() || self.interrupt.parked();
1564 if !parking && self.state.resource_block.is_some() {
1565 self.stop_for_resource()?;
1566 return Ok(true);
1567 }
1568 if !self.pause.parked() && !self.interrupt.parked() {
1573 return Ok(false);
1574 }
1575 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1576 let why = match cause.as_ref().map(ParkCause::label) {
1577 Some(reason) => format!(
1578 "parked after `{}` ({reason}) — resume to carry on from here",
1579 self.state.status.as_str()
1580 ),
1581 None => format!(
1582 "parked after `{}` — resume to carry on from here",
1583 self.state.status.as_str()
1584 ),
1585 };
1586 self.state.event("park", why);
1587 self.state
1588 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1589 self.state.parked = true;
1590 self.state.save()?;
1591 Ok(true)
1592 }
1593
1594 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1596 if self.state.prep_failure.is_none() {
1597 return Ok(());
1598 }
1599 let failed_at = self
1600 .state
1601 .events
1602 .iter()
1603 .rev()
1604 .find(|event| event.node == "prep")
1605 .map_or(self.state.updated_at, |event| event.at);
1606 anyhow::ensure!(
1607 approval_at > failed_at,
1608 "preparation retry requires a new explicit release after the failure"
1609 );
1610 self.state.prep_failure = None;
1611 self.state.prep_complete = Some(false);
1612 self.state
1613 .event("prep-retry", "operator approved retry of preparation");
1614 Ok(())
1615 }
1616
1617 pub fn on_pause(&mut self, pause: Pause) {
1619 self.pause = pause;
1620 }
1621
1622 pub fn watch_interrupt(&mut self, pause: Pause) {
1628 self.interrupt = pause;
1629 }
1630
1631 fn settle_questions(&mut self) {
1651 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1652 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1653 }
1654 }
1655
1656 async fn finish_after_tally(&mut self) -> Result<()> {
1659 self.graft().await?;
1662 if self.resource_halt()? {
1663 return Ok(());
1664 }
1665 self.fold_losers().await?;
1666 self.sync_to_base().await?;
1671 if self.resource_halt()? {
1672 return Ok(());
1673 }
1674 if self.state.status == RunStatus::AlreadyInBase {
1675 return Ok(());
1676 }
1677 self.review_loop().await?;
1678 if self.resource_halt()? {
1679 return Ok(());
1680 }
1681 self.sync_to_base().await?;
1682 if self.resource_halt()? {
1683 return Ok(());
1684 }
1685 if self.state.status == RunStatus::AlreadyInBase {
1686 return Ok(());
1687 }
1688 self.gate().await?;
1689 self.merge().await?;
1690 if self.resource_halt()? {
1691 return Ok(());
1692 }
1693 self.state.save()?;
1694 Ok(())
1695 }
1696
1697 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1704 let result = match ensure_setup_config(&mut self.state, repo).await {
1705 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1706 Err(e) => Err(e),
1707 };
1708 if let Err(e) = result {
1709 git::worktree_remove_clean(repo, wt).await.ok();
1710 if in_prep {
1711 self.rollback_prep(repo).await;
1712 }
1713 self.state.status = RunStatus::Blocked;
1714 self.state.event(
1715 "setup",
1716 format!("worktree setup failed in {}: {e:#}", wt.display()),
1717 );
1718 self.state.save()?;
1719 return Err(e);
1720 }
1721 Ok(())
1722 }
1723
1724 async fn rollback_prep(&mut self, repo: &Path) {
1726 let root = self.state.worktree_root();
1727 for c in std::mem::take(&mut self.state.candidates) {
1728 if matches!(
1729 git::worktree_remove_clean(repo, &c.worktree).await,
1730 Ok(true)
1731 ) {
1732 git::branch_delete(repo, &c.branch).await.ok();
1733 } else {
1734 self.state.candidates.push(c);
1735 }
1736 }
1737 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1738 for j in 1..=seats {
1739 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1740 .await
1741 .ok();
1742 }
1743 for k in 1..=self.state.config.graph.advisors {
1744 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1745 .await
1746 .ok();
1747 }
1748 }
1749
1750 async fn prep(&mut self) -> Result<()> {
1751 if let Some(reason) = &self.state.prep_failure {
1752 bail!(
1753 "candidate preparation previously failed: {reason}; correct the source before retrying"
1754 );
1755 }
1756 if self
1757 .state
1758 .prep_complete
1759 .unwrap_or(!self.state.candidates.is_empty())
1760 {
1761 return Ok(());
1762 }
1763 let repo = self.state.repo.clone();
1764 let _coordination = self.coordinate(&repo).await?;
1767 self.rollback_prep(&repo).await;
1768 if !self.state.candidates.is_empty() {
1769 bail!(
1770 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1771 );
1772 }
1773 self.state.prep_complete = Some(false);
1774 self.state.status = RunStatus::Prep;
1775 self.state.save()?;
1776 let repo = self.state.repo.clone();
1777 let base = self.state.base_commit.clone();
1778 let plan = refs::plan(&repo, &self.state.seeds).await?;
1779 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1780 let root = self.state.worktree_root();
1781 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1782
1783 let hooks_dir = self.state.dir().join("hooks");
1786 if self.state.config.blind.commit_msg_hook {
1787 std::fs::create_dir_all(&hooks_dir)
1788 .with_context(|| format!("create {}", hooks_dir.display()))?;
1789 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1790 let path = hooks_dir.join("commit-msg");
1791 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1792 make_executable(&path)?;
1793 git::acquire_worktree_config(&repo).await?;
1801 self.state.enabled_worktree_config = true;
1802 }
1803
1804 ensure_setup_config(&mut self.state, &repo).await?;
1807
1808 for (index, (spec, label)) in self
1809 .roles
1810 .implementers
1811 .clone()
1812 .into_iter()
1813 .zip(labels)
1814 .enumerate()
1815 {
1816 let branch = self.state.branch_for(label);
1817 let worktree = root.join(format!("cand-{label}"));
1818 self.state.event(
1822 "prep-intent",
1823 serde_json::json!({
1824 "branch": branch,
1825 "worktree": worktree,
1826 "start": start,
1827 "picks": plan.picks,
1828 })
1829 .to_string(),
1830 );
1831 self.state.save()?;
1832 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1833 if self.state.config.blind.commit_msg_hook {
1834 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1835 }
1836 git::local_exclude(&worktree, "/.magi/").await?;
1837 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1838 git::worktree_remove_clean(&repo, &worktree).await.ok();
1841 git::branch_delete(&repo, &branch).await.ok();
1842 self.rollback_prep(&repo).await;
1843 self.state.status = RunStatus::Blocked;
1844 self.state
1845 .event("prep", format!("worktree setup failed: {e:#}"));
1846 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1847 self.state.save()?;
1848 return Err(e);
1849 }
1850 self.state.candidates.push(Candidate {
1851 index,
1852 label,
1853 agent: spec.id.clone(),
1854 branch,
1855 worktree,
1856 summary: String::new(),
1857 stat: String::new(),
1858 files: 0,
1859 commits: 0,
1860 empty: false,
1861 failed: None,
1862 verified_noop: None,
1863 duration_ms: 0,
1864 folded: false,
1865 });
1866 self.state.save()?;
1867 for pick in &plan.picks {
1868 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1869 {
1870 self.state.status = RunStatus::Blocked;
1871 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1872 self.state
1873 .event("prep", format!("cannot apply referenced commit: {e}"));
1874 self.state.save()?;
1875 return Err(e);
1876 }
1877 }
1878 }
1879
1880 for j in 1..=self.roles.judges.len() {
1881 let wt = root.join(format!("judge-{j}"));
1882 if !wt.exists() {
1883 git::worktree_add_detached(&repo, &wt, &base).await?;
1884 self.setup_seat_worktree(&repo, &wt, true).await?;
1885 }
1886 }
1887
1888 if self.state.config.graph.advise {
1896 for k in 1..=self.state.config.graph.advisors {
1897 let wt = root.join(format!("advisor-{k}"));
1898 if !wt.exists() {
1899 git::worktree_add_detached(&repo, &wt, &base).await?;
1900 self.setup_seat_worktree(&repo, &wt, true).await?;
1901 }
1902 }
1903 }
1904
1905 let authors: Vec<&str> = self
1910 .roles
1911 .implementers
1912 .iter()
1913 .map(|a| a.id.as_str())
1914 .collect();
1915 let overlap: Vec<String> = self
1916 .roles
1917 .judges
1918 .iter()
1919 .enumerate()
1920 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1921 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1922 .collect();
1923 if !overlap.is_empty() {
1924 let note = format!(
1925 "{} also authored a candidate; blind, but the panel is less \
1926 independent than {} distinct agents would be",
1927 overlap.join(", "),
1928 self.roles.judges.len()
1929 );
1930 self.state.event("prep", note);
1931 }
1932
1933 self.state.event(
1934 "prep",
1935 format!(
1936 "{} candidates, {} judges, base {} ({})",
1937 self.state.candidates.len(),
1938 self.roles.judges.len(),
1939 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1940 self.state.base_branch
1941 ),
1942 );
1943 self.state.prep_complete = Some(true);
1944 self.state.status = RunStatus::Implementing;
1945 self.state.save()?;
1946 Ok(())
1947 }
1948
1949 async fn advise(&mut self) -> Result<()> {
1982 let implement_untouched = self
1983 .state
1984 .candidates
1985 .iter()
1986 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1987 if !self.state.config.graph.advise || self.state.advise_attempted {
1988 return Ok(());
1989 }
1990 if !implement_untouched {
1991 self.state.event(
1992 "advise",
1993 "skipping the design-deliberation stage: at least one \
1994 candidate already shows implementation progress, so this \
1995 run is past the point the stage exists to run before"
1996 .to_owned(),
1997 );
1998 self.state.advise_attempted = true;
1999 self.state.save()?;
2000 return Ok(());
2001 }
2002 let run_id = self.state.id.clone();
2003 let prompts = self.state.config.prompts.clone();
2004 let instruction = self.state.instruction.clone();
2005 let language = self.state.config.graph.language.clone();
2006 let root = self.state.worktree_root();
2007 let n = self.state.config.graph.advisors;
2008 let where_recorded = self.state.dir().join("run.json");
2009
2010 let seats = match self.state.config.advisors() {
2011 Ok(seats) if !seats.is_empty() => seats,
2012 Ok(_) => {
2013 self.state.event(
2014 "advise",
2015 format!(
2016 "[graph] advisors is 0; skipping the design-deliberation \
2017 stage and continuing without a synthesis brief (see {})",
2018 where_recorded.display()
2019 ),
2020 );
2021 self.state.advise_attempted = true;
2022 self.state.save()?;
2023 return Ok(());
2024 }
2025 Err(e) => {
2026 self.state.event(
2027 "advise",
2028 format!(
2029 "could not resolve advisor seats ({e:#}); continuing \
2030 without a design-deliberation brief (see {})",
2031 where_recorded.display()
2032 ),
2033 );
2034 self.state.advise_attempted = true;
2035 self.state.save()?;
2036 return Ok(());
2037 }
2038 };
2039
2040 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2041 let artifacts = agent::artifacts_dir(&self.state.dir());
2042 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2043
2044 let mut jobs = Vec::new();
2045 for (i, spec) in seats.iter().cloned().enumerate() {
2046 let seat_key = format!("advisor-{}", i + 1);
2047 let seat = self.seat(&seat_key, &spec.id);
2048 jobs.push(SeatJob {
2049 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2050 spec,
2051 seat,
2052 cwd: worktrees[i % worktrees.len()].clone(),
2053 timeout,
2054 allow_write: false,
2055 sessions: false,
2056 artifacts: artifacts.clone(),
2057 stem: seat_key,
2058 handover: None,
2059 });
2060 }
2061
2062 self.state.event(
2063 "advise",
2064 format!(
2065 "{} advisor seat(s) sketching a design in parallel",
2066 jobs.len()
2067 ),
2068 );
2069 let mut quota_losses = Vec::new();
2070 let cache = self.state.config.cache_dir();
2071 let ctx = WaveCtx {
2072 carry_seats: false,
2073 pauses: Some((&self.pause, &self.interrupt)),
2074 run: &run_id,
2075 node: "advise",
2076 prompts: &prompts,
2077 cache: cache.as_deref(),
2078 round: None,
2079 };
2080 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2081 let results = ask_json_wave::<Proposal>(
2082 jobs,
2083 Arc::clone(&self.sem),
2084 self.state.config.graph.retries,
2085 &advisor_roster,
2086 &ctx,
2087 &mut quota_losses,
2088 &mut self.state,
2089 &|p: &Proposal| p.validate(),
2090 )
2091 .await;
2092 if self.state.resource_block.is_some() {
2093 return self.stop_for_resource();
2094 }
2095 self.state.quota.extend(quota_losses);
2096
2097 let mut records = Vec::with_capacity(results.len());
2098 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2099 let agent_id = seat.agent.clone();
2100 self.state.seats.insert(seat.key.clone(), seat);
2101 match res {
2102 Ok((proposal, out)) => {
2103 self.state
2104 .event("advise", format!("advisor-{} proposed a design", i + 1));
2105 records.push(advise::AdvisorRecord::proposed(
2106 i + 1,
2107 agent_id,
2108 proposal,
2109 out.duration_ms,
2110 ));
2111 }
2112 Err(e) => {
2113 self.state.event(
2114 "advise",
2115 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2116 );
2117 records.push(advise::AdvisorRecord::failed(
2118 i + 1,
2119 agent_id,
2120 e.to_string(),
2121 ));
2122 }
2123 }
2124 }
2125
2126 let mut advice = advise::Advice {
2127 records,
2128 synthesis: None,
2129 };
2130 if advice.proposals().is_empty() {
2131 self.state.event(
2132 "advise",
2133 "no advisor produced a usable proposal; continuing without a \
2134 synthesis brief"
2135 .to_owned(),
2136 );
2137 } else {
2138 match self
2139 .synthesize_brief(
2140 &advice,
2141 &instruction,
2142 &language,
2143 &worktrees[0],
2144 &artifacts,
2145 &run_id,
2146 &prompts,
2147 cache.as_deref(),
2148 )
2149 .await
2150 {
2151 Ok(Some(text)) => {
2152 self.state.event(
2153 "advise",
2154 "synthesized a design brief for the implementer".to_owned(),
2155 );
2156 advice.synthesis = Some(text);
2157 }
2158 Ok(None) => {
2159 self.state.event(
2160 "advise",
2161 "the synthesis seat produced nothing usable; continuing \
2162 without a design brief"
2163 .to_owned(),
2164 );
2165 }
2166 Err(e) => {
2167 if e.is::<crate::repo_lock::ResourceBlocked>() {
2168 return Err(e);
2169 }
2170 self.state.event(
2171 "advise",
2172 format!("could not synthesize a design brief: {e:#}"),
2173 );
2174 }
2175 }
2176 }
2177 advise::apply_reflection(&mut advice);
2178
2179 self.state.advice = Some(advice);
2180 self.state.advise_attempted = true;
2181 self.state.save()?;
2182 Ok(())
2183 }
2184
2185 #[allow(clippy::too_many_arguments)]
2197 async fn synthesize_brief(
2198 &mut self,
2199 advice: &advise::Advice,
2200 instruction: &str,
2201 language: &str,
2202 cwd: &Path,
2203 artifacts: &Path,
2204 run_id: &str,
2205 prompts: &Prompts,
2206 cache: Option<&Path>,
2207 ) -> Result<Option<String>> {
2208 let chain = agent::pick_chain(
2209 &self.state.config.agents,
2210 self.state.config.roles.synthesizer.as_ref(),
2211 &agent::installed,
2212 "synthesizer",
2213 )?;
2214 let proposals = advice.proposals();
2215 let mut prompt = prompt::with_overlay(
2216 prompt::synthesize_brief(instruction, &proposals, language),
2217 prompts.overlay("advise"),
2218 );
2219 if cache.is_some() {
2220 prompt.push('\n');
2225 prompt.push_str(&prompt::build_cache_note("advise", false));
2226 }
2227 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2228 let mut last = None;
2233 for (n, spec) in chain.iter().enumerate() {
2234 if n > 0 {
2235 self.state
2236 .event("advise", format!("synthesis falling back to {}", spec.id));
2237 }
2238 let mut seat = self.seat("advise-synthesis", &spec.id);
2239 let wait = self.repository_wait();
2240 let outcome = wait
2241 .scope(agent::invoke(
2242 spec,
2243 &mut seat,
2244 &Invocation {
2245 scope: crate::agent::InvocationScope::Repository,
2246 coordination: self.state.config.coordination.as_ref(),
2247 fetched_base: None,
2248 cwd,
2249 prompt: &prompt,
2250 timeout,
2251 allow_write: false,
2252 unsandboxed: false,
2253 sessions: false,
2254 artifacts,
2255 stem: &if n == 0 {
2256 "advise-synthesis".to_owned()
2257 } else {
2258 format!("advise-synthesis-{}", spec.id)
2259 },
2260 run: run_id,
2261 node: "advise",
2262 cache_dir: None,
2263 attachments: &[],
2264 writable: &[],
2265 },
2266 ))
2267 .await;
2268 if outcome.is_ok() {
2269 self.state.seats.insert(seat.key.clone(), seat);
2270 }
2271 let advance = agent::chain_advances(&outcome);
2272 last = Some(outcome);
2273 if !advance {
2274 break;
2275 }
2276 }
2277 let out = last.expect("a chain holds at least one agent")?;
2280 if !out.usable() {
2281 return Ok(None);
2282 }
2283 let text =
2284 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2285 Ok((!text.trim().is_empty()).then_some(text))
2286 }
2287
2288 async fn implement(&mut self) -> Result<()> {
2291 self.persist_candidates().await?;
2292 let run_id = self.state.id.clone();
2297 let prompts = self.state.config.prompts.clone();
2298 let todo: Vec<usize> = self
2299 .state
2300 .candidates
2301 .iter()
2302 .enumerate()
2303 .filter(|(_, c)| {
2304 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2305 && c.failed.is_none()
2306 && !c.empty
2307 })
2308 .map(|(i, _)| i)
2309 .collect();
2310 if todo.is_empty() {
2311 return self.after_implement();
2312 }
2313 self.state.status = RunStatus::Implementing;
2314
2315 let language = self.state.config.graph.language.clone();
2316 let github_language = self.state.config.graph.github_language.clone();
2317 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2318 let sessions = self.state.config.graph.sessions;
2319 let artifacts = agent::artifacts_dir(&self.state.dir());
2320 let brief = self
2324 .state
2325 .advice
2326 .as_ref()
2327 .and_then(|a| a.synthesis.as_deref())
2328 .map(str::to_owned);
2329 let attachments = self.state.attachments.clone();
2330
2331 let mut jobs = Vec::new();
2332 for &i in &todo {
2333 let (index, label, worktree) = {
2334 let c = &self.state.candidates[i];
2335 (c.index, c.label, c.worktree.clone())
2336 };
2337 let spec = self.roles.implementers[index].clone();
2338 let seat_key = format!("impl-{label}");
2339 let seat = self.seat(&seat_key, &spec.id);
2340 let instruction = seeded_instruction(&self.state);
2341 jobs.push(SeatJob {
2342 spec,
2343 seat,
2344 prompt: prompt::implement_in(
2345 &instruction,
2346 &worktree.to_string_lossy(),
2347 &language,
2348 &github_language,
2349 brief.as_deref(),
2350 &attachments,
2351 ),
2352 cwd: worktree,
2353 timeout,
2354 allow_write: true,
2355 sessions,
2356 artifacts: artifacts.clone(),
2357 stem: format!("impl-{label}"),
2358 handover: None,
2359 });
2360 }
2361
2362 self.state.event(
2363 "implement",
2364 format!("{} candidates in parallel", jobs.len()),
2365 );
2366 let mut sent = jobs.clone();
2372 let cache = self.state.config.cache_dir();
2373 let ctx = WaveCtx {
2374 carry_seats: false,
2375 pauses: Some((&self.pause, &self.interrupt)),
2376 run: &run_id,
2377 node: "implement",
2378 prompts: &prompts,
2379 cache: cache.as_deref(),
2380 round: None,
2381 };
2382 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2383 if !wave_blocked(&results) {
2384 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2385 .await;
2386 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2387 .await?;
2388 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2389 .await;
2390 }
2391 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2392
2393 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2394 && results
2395 .iter()
2396 .any(|(_, _, out)| FailClass::of(out).is_some());
2397 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2398 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2399 if self
2403 .state
2404 .candidate_preservation
2405 .contains_key(&self.state.candidates[i].label)
2406 {
2407 self.state.seats.insert(seat.key.clone(), seat);
2408 }
2409 continue;
2410 }
2411 let seat_key = seat.key.clone();
2412 let agent = seat.agent.clone();
2421 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2422 self.state.seats.insert(seat.key.clone(), seat);
2423 let label = self.state.candidates[i].label;
2424 self.state.interrupted_implementations.remove(&label);
2425
2426 let (summary, duration, failed, verified_claim) = match out {
2427 AgentOutcome::Ok(o) => {
2428 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2429 let failed = (!o.usable()).then(|| {
2430 if o.timed_out {
2431 "agent timed out".to_owned()
2432 } else {
2433 format!(
2434 "agent exited with {}",
2435 crate::notices::exit_status(o.exit_code)
2436 )
2437 }
2438 });
2439 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2440 (text, o.duration_ms, failed, verified_claim)
2441 }
2442 AgentOutcome::Dropped(o) => {
2448 let why = o
2449 .dropped
2450 .as_ref()
2451 .map(|d| d.why.as_str())
2452 .unwrap_or("the CLI ended the stream without delivering its answer");
2453 (
2454 String::new(),
2455 o.duration_ms,
2456 Some(format!("the CLI dropped the stream ({why})")),
2457 None,
2458 )
2459 }
2460 AgentOutcome::Quota(o) => {
2461 self.state.quota.push(QuotaLoss {
2462 seat: seat_key,
2463 node: "implement".to_owned(),
2464 at: Timestamp::now(),
2465 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2466 });
2467 (
2468 String::new(),
2469 o.duration_ms,
2470 Some("rate limited (quota); produced no change".to_owned()),
2471 None,
2472 )
2473 }
2474 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2475 (String::new(), 0, Some(e), None)
2476 }
2477 };
2478
2479 let c = &mut self.state.candidates[i];
2482 if !exhausted_the_fallback_chain {
2483 c.agent = agent;
2484 }
2485 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2486 c.duration_ms = duration;
2487 c.failed = failed;
2488 c.verified_noop = verified_claim;
2489 self.state
2490 .candidate_preservation
2491 .insert(label, "awaiting persistence".to_owned());
2492 }
2493 self.state.save()?;
2494 self.persist_candidates().await?;
2495
2496 if resource_blocked {
2497 return self.stop_for_resource();
2498 }
2499 if selected_failed {
2500 self.state.status = RunStatus::Blocked;
2501 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2502 self.state.save()?;
2503 bail!(
2504 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2505 );
2506 }
2507 self.after_implement()
2508 }
2509
2510 async fn persist_candidates(&mut self) -> Result<()> {
2512 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2513 for label in labels {
2514 let i = self
2515 .state
2516 .candidates
2517 .iter()
2518 .position(|c| c.label == label)
2519 .context("pending candidate is missing")?;
2520 let c = self.state.candidates[i].clone();
2521 let interrupted = self.state.interrupted_implementations.contains(&label);
2522 let result = async {
2523 let wait = crate::repo_lock::WaitPolicy::new(
2524 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2525 || false,
2526 );
2527 let _coordination =
2528 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2529 if let Err(e) = git::snapshot_uncommitted(
2532 &c.worktree,
2533 &self
2534 .state
2535 .dir()
2536 .join("artifacts")
2537 .join(format!("candidate-{label}-recovery")),
2538 )
2539 .await
2540 {
2541 self.state.event(
2542 "preserve",
2543 format!("candidate {label} snapshot incomplete: {e:#}"),
2544 );
2545 }
2546 let rescued = git::rescue_commit(
2547 &c.worktree,
2548 &format!("magi: candidate {label} (uncommitted work)"),
2549 )
2550 .await?;
2551 self.state.note_withheld("implement", &rescued.withheld);
2552 let base = &self.state.base_commit;
2553 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2554 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2555 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2556 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2557 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2558 if c.empty && commits > 0 && !patch.trim().is_empty()
2559 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2560 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2561 || self.state.tally.is_some())
2562 {
2563 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2567 }
2568 let c = &mut self.state.candidates[i];
2569 c.commits = commits;
2570 c.stat = stat;
2571 c.files = files;
2572 let no_changes = commits == 0 || patch.trim().is_empty();
2573 c.empty = no_changes && !interrupted;
2574 if interrupted && no_changes {
2575 c.failed = None;
2576 c.verified_noop = None;
2577 }
2578 if !c.empty {
2579 c.failed = None;
2580 c.verified_noop = None;
2581 }
2582 self.state.event(
2583 "preserve",
2584 format!("candidate {label}: {files} files, {commits} commits"),
2585 );
2586 Ok::<_, anyhow::Error>(())
2587 }
2588 .await;
2589 match result {
2590 Ok(()) => {
2591 self.state.candidate_preservation.remove(&label);
2592 if !interrupted || self.state.candidates[i].files > 0 {
2593 self.state.interrupted_implementations.remove(&label);
2594 }
2595 }
2596 Err(e) => {
2597 let reason = format!("candidate {label} preservation failed: {e:#}");
2598 self.state.event("preserve", &reason);
2599 self.state.candidate_preservation.insert(label, reason);
2600 }
2601 }
2602 if self.state.candidate_preservation.is_empty()
2603 && self.state.status == RunStatus::Blocked
2604 && self.state.candidates.iter().all(|c| c.failed.is_none())
2605 {
2606 self.state.status = RunStatus::Implementing;
2610 self.state.event(
2611 "preserve",
2612 "candidate persistence recovered; implementation can continue",
2613 );
2614 }
2615 self.state.save()?;
2616 }
2617 if !self.state.candidate_preservation.is_empty() {
2618 self.state.status = RunStatus::Blocked;
2619 self.state.save()?;
2620 bail!(
2621 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2622 );
2623 }
2624 Ok(())
2625 }
2626
2627 async fn resume_undelivered(
2655 &mut self,
2656 results: &mut [(usize, SeatState, AgentOutcome)],
2657 sent: &[SeatJob],
2658 prompts: &Prompts,
2659 run_id: &str,
2660 ) {
2661 for (wi, seat, out) in results.iter_mut() {
2662 let Some(dropped) = (match &*out {
2663 AgentOutcome::Dropped(o) => o.dropped.clone(),
2664 _ => None,
2665 }) else {
2666 continue;
2667 };
2668 let Some(job) = sent.get(*wi) else { continue };
2669 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2671 self.state.event(
2672 "implement",
2673 format!(
2674 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2675 work is in the tree",
2676 seat.key, dropped.output_tokens, dropped.why
2677 ),
2678 );
2679 continue;
2680 }
2681 if !has_context(&job.spec, seat, job.sessions) {
2689 self.state.event(
2690 "implement",
2691 format!(
2692 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2693 is no session left to resume",
2694 seat.key, dropped.output_tokens, dropped.why
2695 ),
2696 );
2697 continue;
2698 }
2699 self.state.event(
2700 "implement",
2701 format!(
2702 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2703 conversation",
2704 seat.key, dropped.output_tokens, dropped.why
2705 ),
2706 );
2707 let mut retry = job.clone();
2708 retry.seat = seat.clone();
2709 retry.prompt = prompt::resume_after_drop(&dropped.why);
2710 retry.timeout = retry_budget(job.timeout, true);
2711 retry.stem = format!("{}-resume", job.stem);
2712 let cache = self.state.config.cache_dir();
2713 let ctx = WaveCtx {
2714 carry_seats: false,
2715 pauses: Some((&self.pause, &self.interrupt)),
2716 run: run_id,
2717 node: "implement",
2718 prompts,
2719 cache: cache.as_deref(),
2720 round: None,
2721 };
2722 let (resumed_seat, resumed) =
2723 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2724 *seat = resumed_seat;
2725 *out = resumed;
2726 }
2727 }
2728
2729 async fn resume_seat_handovers(
2794 &mut self,
2795 results: &mut [(usize, SeatState, AgentOutcome)],
2796 sent: &mut [SeatJob],
2797 prompts: &Prompts,
2798 run_id: &str,
2799 ) -> Result<()> {
2800 let instruction = seeded_instruction(&self.state);
2801 let language = self.state.config.graph.language.clone();
2802 let github_language = self.state.config.graph.github_language.clone();
2803 let brief = self
2804 .state
2805 .advice
2806 .as_ref()
2807 .and_then(|a| a.synthesis.as_deref())
2808 .map(str::to_owned);
2809 let attachments = self.state.attachments.clone();
2810 for (wi, seat, out) in results.iter_mut() {
2811 let others: BTreeSet<String> = sent
2814 .iter()
2815 .enumerate()
2816 .filter(|(j, _)| j != wi)
2817 .map(|(_, j)| j.spec.id.clone())
2818 .collect();
2819 let Some(job) = sent.get_mut(*wi) else {
2820 continue;
2821 };
2822 let start = self
2827 .roles
2828 .implementer_roster
2829 .iter()
2830 .position(|s| s.id == job.spec.id)
2831 .unwrap_or(0);
2832 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2833 let mut fallback_attempt = 0usize;
2834 let mut prev: Option<FailClass> = None;
2835 while let Some(cur) = FailClass::of(&*out) {
2836 if !should_hand_over(prev.as_ref(), &cur) {
2837 break;
2838 }
2839 let Some(next) =
2840 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2841 .cloned()
2842 else {
2843 break;
2844 };
2845 tried.insert(next.id.clone());
2846 fallback_attempt += 1;
2847
2848 let coordination = self.coordinate(&job.cwd).await?;
2849 if let Ok(r) = git::rescue_commit(
2850 &job.cwd,
2851 &format!(
2852 "magi: candidate {} (uncommitted work before {} fallback)",
2853 seat.key,
2854 if cur == FailClass::Quota {
2855 "quota"
2856 } else {
2857 "handover"
2858 }
2859 ),
2860 )
2861 .await
2862 {
2863 self.state.note_withheld("implement", &r.withheld);
2864 }
2865 drop(coordination);
2866
2867 record_handover(
2868 &mut self.state,
2869 "implement",
2870 &seat.key,
2871 &seat.agent,
2872 &next.id,
2873 &cur,
2874 &fail_reason(&*out),
2875 );
2876 prev = Some(cur.clone());
2877
2878 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2879 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2880 job.spec = next.clone();
2888 let mut retry = job.clone();
2889 retry.seat = new_seat;
2890 retry.prompt = prompt::implement_in(
2891 &instruction,
2892 &job.cwd.to_string_lossy(),
2893 &language,
2894 &github_language,
2895 brief.as_deref(),
2896 &attachments,
2897 );
2898 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2899 let cache = self.state.config.cache_dir();
2900 let ctx = WaveCtx {
2901 carry_seats: false,
2902 pauses: Some((&self.pause, &self.interrupt)),
2903 run: run_id,
2904 node: "implement",
2905 prompts,
2906 cache: cache.as_deref(),
2907 round: None,
2908 };
2909 let (fallback_seat, fallback_out) = run_one(
2910 retry,
2911 Arc::clone(&self.sem),
2912 &ctx,
2913 &mut self.state,
2914 fallback_attempt,
2915 )
2916 .await;
2917 *seat = fallback_seat;
2918 *out = fallback_out;
2919 }
2920 }
2921 Ok(())
2922 }
2923
2924 async fn resume_unconfirmed_commands(
2948 &mut self,
2949 results: &mut [(usize, SeatState, AgentOutcome)],
2950 sent: &[SeatJob],
2951 prompts: &Prompts,
2952 run_id: &str,
2953 ) {
2954 for (wi, seat, out) in results.iter_mut() {
2955 let AgentOutcome::Ok(o) = &*out else {
2956 continue;
2957 };
2958 if !has_unconfirmed_command(&o.commands) {
2959 continue;
2960 }
2961 let Some(job) = sent.get(*wi) else { continue };
2962 if !has_context(&job.spec, seat, job.sessions) {
2963 self.state.event(
2964 "implement",
2965 format!(
2966 "{}: the reply named a command whose own CLI never confirmed the exit \
2967 status of, but there is no session left to resume",
2968 seat.key
2969 ),
2970 );
2971 continue;
2972 }
2973 self.state.event(
2974 "implement",
2975 format!(
2976 "{}: the reply named a command whose own CLI never confirmed the exit \
2977 status of; resuming the conversation",
2978 seat.key
2979 ),
2980 );
2981 let mut retry = job.clone();
2982 retry.seat = seat.clone();
2983 retry.prompt = prompt::resume_incomplete(
2984 "a command in your last reply had no confirmed exit status",
2985 );
2986 retry.timeout = retry_budget(job.timeout, true);
2987 retry.stem = format!("{}-confirm", job.stem);
2988 let cache = self.state.config.cache_dir();
2989 let ctx = WaveCtx {
2990 carry_seats: false,
2991 pauses: Some((&self.pause, &self.interrupt)),
2992 run: run_id,
2993 node: "implement",
2994 prompts,
2995 cache: cache.as_deref(),
2996 round: None,
2997 };
2998 let (resumed_seat, resumed) =
2999 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3000 *seat = resumed_seat;
3001 *out = resumed;
3002 }
3003 }
3004
3005 async fn continue_fix_report(
3026 &mut self,
3027 mut seat: SeatState,
3028 parse_err: String,
3029 job: &SeatJob,
3030 prompts: &Prompts,
3031 run_id: &str,
3032 round: usize,
3033 ) -> (
3034 SeatState,
3035 Option<FixReport>,
3036 Option<String>,
3037 ContinuationRecord,
3038 ) {
3039 let mut last_err = parse_err;
3040 let mut cumulative_wait_ms = 0u64;
3041 let mut attempts = 0usize;
3042 loop {
3043 if !has_context(&job.spec, &seat, job.sessions) {
3044 self.state.event(
3045 "fix",
3046 format!(
3047 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3048 session left to resume into"
3049 ),
3050 );
3051 let outcome = if attempts == 0 {
3052 ContinuationOutcome::NoSession
3053 } else {
3054 ContinuationOutcome::Exhausted
3055 };
3056 return (
3057 seat,
3058 None,
3059 Some(format!("unparsable fix report: {last_err}")),
3060 ContinuationRecord {
3061 attempts,
3062 cumulative_wait_ms,
3063 outcome,
3064 },
3065 );
3066 }
3067 if attempts >= MAX_FIX_CONTINUATIONS {
3068 self.state.event(
3069 "fix",
3070 format!(
3071 "round {round}: fixer's reply still had no adoption report after \
3072 {attempts} continuation(s) ({last_err}); giving up"
3073 ),
3074 );
3075 return (
3076 seat,
3077 None,
3078 Some(format!(
3079 "unparsable fix report after {attempts} continuation(s): {last_err}"
3080 )),
3081 ContinuationRecord {
3082 attempts,
3083 cumulative_wait_ms,
3084 outcome: ContinuationOutcome::Exhausted,
3085 },
3086 );
3087 }
3088 attempts += 1;
3089 self.state.event(
3090 "fix",
3091 format!(
3092 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3093 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3094 ),
3095 );
3096 let mut retry = job.clone();
3097 retry.seat = seat.clone();
3098 retry.prompt = prompt::resume_incomplete(&last_err);
3099 retry.timeout = retry_budget(job.timeout, true);
3100 retry.stem = format!("{}-continue{attempts}", job.stem);
3101 let cache = self.state.config.cache_dir();
3102 let ctx = WaveCtx {
3103 carry_seats: false,
3104 pauses: Some((&self.pause, &self.interrupt)),
3105 run: run_id,
3106 node: "fix",
3107 prompts,
3108 cache: cache.as_deref(),
3109 round: Some(round),
3110 };
3111 let (resumed_seat, resumed_out) = run_one(
3112 retry,
3113 Arc::clone(&self.sem),
3114 &ctx,
3115 &mut self.state,
3116 attempts,
3117 )
3118 .await;
3119 seat = resumed_seat;
3120 match resumed_out {
3121 AgentOutcome::Ok(o) => {
3122 cumulative_wait_ms += o.duration_ms;
3123 match verdict::extract_json::<FixReport>(&o.text) {
3124 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3125 self.state.event(
3126 "fix",
3127 format!(
3128 "round {round}: fixer's adoption report recovered after \
3129 {attempts} continuation(s)"
3130 ),
3131 );
3132 return (
3133 seat,
3134 Some(report),
3135 None,
3136 ContinuationRecord {
3137 attempts,
3138 cumulative_wait_ms,
3139 outcome: ContinuationOutcome::Resumed,
3140 },
3141 );
3142 }
3143 Ok(_) => {
3151 last_err = "the reply parsed, but it reported a command whose own CLI \
3152 never confirmed an exit status"
3153 .to_owned();
3154 }
3155 Err(e) => last_err = e.to_string(),
3156 }
3157 }
3158 AgentOutcome::Quota(o) => {
3159 cumulative_wait_ms += o.duration_ms;
3160 self.state.quota.push(QuotaLoss {
3161 seat: seat.key.clone(),
3162 node: "fix".to_owned(),
3163 at: Timestamp::now(),
3164 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3165 });
3166 self.state.event(
3167 "fix",
3168 format!(
3169 "round {round}: continuation rate limited (quota); not retrying now"
3170 ),
3171 );
3172 return (
3173 seat,
3174 None,
3175 Some("rate limited (quota) while recovering the fix report".to_owned()),
3176 ContinuationRecord {
3177 attempts,
3178 cumulative_wait_ms,
3179 outcome: ContinuationOutcome::QuotaLost,
3180 },
3181 );
3182 }
3183 AgentOutcome::Dropped(o) => {
3184 cumulative_wait_ms += o.duration_ms;
3185 let why = o
3186 .dropped
3187 .as_ref()
3188 .map(|d| d.why.as_str())
3189 .unwrap_or("the CLI ended the stream without delivering its answer");
3190 last_err = format!("the CLI dropped the stream ({why})");
3191 }
3192 AgentOutcome::Failed(e) => last_err = e,
3193 AgentOutcome::ResourceBlocked(e) => {
3196 return (
3197 seat,
3198 None,
3199 Some(format!("the continuation was not started: {e}")),
3200 ContinuationRecord {
3201 attempts: attempts.saturating_sub(1),
3202 cumulative_wait_ms,
3203 outcome: ContinuationOutcome::NoSession,
3204 },
3205 );
3206 }
3207 }
3208 }
3209 }
3210
3211 fn after_implement(&mut self) -> Result<()> {
3212 if !self.state.candidate_preservation.is_empty() {
3213 bail!("candidate preservation incomplete; judging is unsafe");
3214 }
3215 if self.state.leaks.is_empty() {
3217 let cfg = self.state.config.blind.clone();
3218 let mut leaks = Vec::new();
3219 for c in &self.state.candidates {
3220 let Some(patch) =
3221 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3222 else {
3223 continue;
3224 };
3225 leaks.extend(blind::scan(
3226 &format!("candidate {} patch", c.label),
3227 &patch,
3228 &cfg.vendor_tokens,
3229 ));
3230 }
3231 if !leaks.is_empty() {
3232 let summary = leaks
3233 .iter()
3234 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3235 .collect::<Vec<_>>()
3236 .join(", ");
3237 match cfg.on_leak {
3238 LeakPolicy::Fail => {
3239 self.state.status = RunStatus::Failed;
3240 self.state
3241 .event("blind", format!("vendor text in a patch: {summary}"));
3242 self.state.leaks = leaks;
3243 self.state.save()?;
3244 self.settle_questions();
3245 bail!(
3246 "blind.on_leak = \"fail\" and vendor text reached a \
3247 judged patch: {summary}"
3248 );
3249 }
3250 LeakPolicy::Redact => self.state.event(
3251 "blind",
3252 format!("redacting vendor text for judging: {summary}"),
3253 ),
3254 LeakPolicy::Warn => self.state.event(
3255 "blind",
3256 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3257 ),
3258 }
3259 self.state.leaks = leaks;
3260 }
3261 }
3262
3263 if self.state.viable().is_empty() {
3264 if self.state.all_candidates_verified_noop() {
3265 self.state.status = RunStatus::VerifiedNoop;
3276 self.state.save()?;
3277 self.settle_questions();
3278 return Ok(());
3279 }
3280 self.state.status = RunStatus::Failed;
3281 self.state.save()?;
3282 self.settle_questions();
3283 bail!("no candidate produced a change; nothing to judge");
3284 }
3285 self.state.status = RunStatus::Judging;
3286 self.state.save()?;
3287 Ok(())
3288 }
3289
3290 async fn judge(&mut self) -> Result<()> {
3293 let run_id = self.state.id.clone();
3298 let prompts = self.state.config.prompts.clone();
3299 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3300 return Ok(());
3301 }
3302 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3303 if viable.len() == 1 {
3304 self.state.judge_skipped = true;
3311 self.state.event(
3312 "judge",
3313 format!(
3314 "only candidate {} produced a change; judging skipped",
3315 viable[0].label
3316 ),
3317 );
3318 self.state.save()?;
3319 return Ok(());
3320 }
3321 self.state.status = RunStatus::Judging;
3322
3323 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3324 let language = self.state.config.graph.language.clone();
3325 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3326 let sessions = self.state.config.graph.sessions;
3327 let artifacts = agent::artifacts_dir(&self.state.dir());
3328 let root = self.state.worktree_root();
3329 let base_short = short(&self.state.base_commit);
3330
3331 let mut jobs = Vec::new();
3332 let mut orders = Vec::new();
3333 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3334 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3335 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3336 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3337 let seat_key = format!("judge-{}", j + 1);
3338 let seat = self.seat(&seat_key, &spec.id);
3339 jobs.push(SeatJob {
3340 prompt: prompt::judge(
3341 &self.state.instruction,
3342 &views,
3343 self.roles.judges.len(),
3344 &base_short,
3345 &language,
3346 ),
3347 spec,
3348 seat,
3349 cwd: root.join(format!("judge-{}", j + 1)),
3350 timeout,
3351 allow_write: false,
3352 sessions,
3353 artifacts: artifacts.clone(),
3354 stem: format!("judge-{}", j + 1),
3355 handover: None,
3356 });
3357 }
3358
3359 self.state.event(
3360 "judge",
3361 format!(
3362 "{} judges ranking {} candidates blind",
3363 jobs.len(),
3364 viable.len()
3365 ),
3366 );
3367 let labels_for_check = labels.clone();
3368 let mut quota_losses = Vec::new();
3369 let cache = self.state.config.cache_dir();
3370 let ctx = WaveCtx {
3371 carry_seats: false,
3372 pauses: Some((&self.pause, &self.interrupt)),
3373 run: &run_id,
3374 node: "judge",
3375 prompts: &prompts,
3376 cache: cache.as_deref(),
3377 round: None,
3378 };
3379 let results = ask_json_wave::<Ranking>(
3380 jobs,
3381 Arc::clone(&self.sem),
3382 self.state.config.graph.retries,
3383 &self.roles.judge_roster,
3384 &ctx,
3385 &mut quota_losses,
3386 &mut self.state,
3387 &move |r: &Ranking| r.validate(&labels_for_check),
3388 )
3389 .await;
3390 if self.state.resource_block.is_some() {
3391 return self.stop_for_resource();
3392 }
3393 self.state.quota.extend(quota_losses);
3394
3395 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3396 let agent_id = seat.agent.clone();
3397 self.state.seats.insert(seat.key.clone(), seat);
3398 let mut record = Judgement {
3399 judge: j + 1,
3400 seat: format!("judge-{}", j + 1),
3401 agent: agent_id,
3402 ranking: Vec::new(),
3403 reasons: BTreeMap::new(),
3404 confidence: None,
3405 strengths: BTreeMap::new(),
3406 order: orders[j].clone(),
3407 failed: None,
3408 duration_ms: 0,
3409 };
3410 match res {
3411 Ok((ranking, out)) => {
3412 record.ranking = ranking.normalized();
3413 record.reasons = ranking.reasons;
3414 record.strengths = ranking.strengths;
3415 record.confidence = ranking.confidence;
3416 record.duration_ms = out.duration_ms;
3417 self.state.event(
3418 "judge",
3419 format!(
3420 "judge {} ranked {}",
3421 j + 1,
3422 record.ranking.iter().collect::<String>()
3423 ),
3424 );
3425 }
3426 Err(e) => {
3427 record.failed = Some(e.to_string());
3428 self.state
3429 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3430 }
3431 }
3432 self.state.judgements.push(record);
3433 self.state.save()?;
3434 }
3435 Ok(())
3436 }
3437
3438 async fn deliberate(&mut self) -> Result<()> {
3441 let run_id = self.state.id.clone();
3446 let prompts = self.state.config.prompts.clone();
3447 if !self.state.deliberation.is_empty() {
3448 return Ok(());
3449 }
3450 let tops: Vec<char> = self
3451 .state
3452 .judgements
3453 .iter()
3454 .filter_map(|j| j.ranking.first().copied())
3455 .collect();
3456 let rounds = self.state.config.graph.deliberate_rounds;
3457 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3458 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3459 self.state.event(
3460 "deliberate",
3461 format!("judges agreed on {} outright; no deliberation", tops[0]),
3462 );
3463 }
3464 self.state.status = RunStatus::Voting;
3465 self.state.save()?;
3466 return Ok(());
3467 }
3468
3469 self.state.status = RunStatus::Deliberating;
3470 self.state.event(
3471 "deliberate",
3472 format!(
3473 "split: first choices were {} — opening {rounds} round(s)",
3474 tops.iter().collect::<String>()
3475 ),
3476 );
3477
3478 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3479 let language = self.state.config.graph.language.clone();
3480 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3481 let sessions = self.state.config.graph.sessions;
3482 let artifacts = agent::artifacts_dir(&self.state.dir());
3483 let root = self.state.worktree_root();
3484 let base_short = short(&self.state.base_commit);
3485
3486 for round in 1..=rounds {
3490 let mut turns: Vec<DeliberationTurn> = Vec::new();
3491 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3492 if self.state.judgements[j].failed.is_some() {
3493 continue;
3494 }
3495 let seat_key = format!("judge-{}", j + 1);
3496 let spec = self.occupant(&seat_key, spec);
3497 let mut seat = self.seat(&seat_key, &spec.id);
3498 let transcript = self.transcript(&turns, j);
3499 let build = |context: Option<&str>| {
3500 prompt::deliberate(
3501 &self.state.instruction,
3502 context,
3503 &transcript,
3504 round,
3505 rounds,
3506 &language,
3507 )
3508 };
3509 let block = self.candidate_block(&viable, &base_short);
3510 let full = build(Some(&block));
3511 let text = if has_context(&spec, &seat, sessions) {
3512 build(None)
3513 } else {
3514 full.clone()
3515 };
3516 let job = SeatJob {
3517 spec,
3518 seat: seat.clone(),
3519 prompt: text,
3520 cwd: root.join(format!("judge-{}", j + 1)),
3521 timeout,
3522 allow_write: false,
3523 sessions,
3524 artifacts: artifacts.clone(),
3525 stem: format!("delib-{round}-judge-{}", j + 1),
3526 handover: Some(full),
3527 };
3528 let cache = self.state.config.cache_dir();
3529 let ctx = WaveCtx {
3530 carry_seats: false,
3531 pauses: Some((&self.pause, &self.interrupt)),
3532 run: &run_id,
3533 node: "deliberate",
3534 prompts: &prompts,
3535 cache: cache.as_deref(),
3536 round: None,
3537 };
3538 let mut losses = Vec::new();
3542 let mut results = ask_wave_with::<String>(
3543 vec![job],
3544 Arc::clone(&self.sem),
3545 0,
3546 &self.roles.judge_roster,
3547 &ctx,
3548 &mut losses,
3549 &mut self.state,
3550 &|text: &str| {
3551 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3552 },
3553 )
3554 .await;
3555 self.state.quota.extend(losses);
3556 let (updated, res, _) = results.pop().expect("one job in, one result out");
3557 seat = updated;
3558 let agent_id = seat.agent.clone();
3559 self.state.seats.insert(seat.key.clone(), seat);
3560 let body = match res {
3561 Ok((body, _)) => body,
3562 Err(e) => {
3565 self.state
3566 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3567 continue;
3568 }
3569 };
3570 let tentative = verdict::extract_json::<Position>(&body)
3571 .ok()
3572 .and_then(|p| p.tentative)
3573 .and_then(|s| s.trim().chars().next())
3574 .map(|c| c.to_ascii_uppercase());
3575 self.state.event(
3576 "deliberate",
3577 format!(
3578 "round {round}: judge {} now favours {}",
3579 j + 1,
3580 tentative.map_or("—".to_owned(), |c| c.to_string())
3581 ),
3582 );
3583 turns.push(DeliberationTurn {
3584 judge: j + 1,
3585 agent: agent_id,
3586 body: blind::sanitize_prose(&body, &self.state.config.blind),
3587 tentative,
3588 });
3589 }
3590 self.state
3591 .deliberation
3592 .push(DeliberationRound { round, turns });
3593 self.state.save()?;
3594 }
3595
3596 self.state.status = RunStatus::Voting;
3597 self.state.save()?;
3598 Ok(())
3599 }
3600
3601 async fn vote(&mut self) -> Result<()> {
3604 let run_id = self.state.id.clone();
3609 let prompts = self.state.config.prompts.clone();
3610 if !self.state.votes.is_empty() {
3611 return Ok(());
3612 }
3613 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3614 if viable.len() == 1 {
3615 return Ok(());
3616 }
3617 self.state.status = RunStatus::Voting;
3618
3619 let language = self.state.config.graph.language.clone();
3620 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3621 let sessions = self.state.config.graph.sessions;
3622 let artifacts = agent::artifacts_dir(&self.state.dir());
3623 let root = self.state.worktree_root();
3624 let base_short = short(&self.state.base_commit);
3625 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3626
3627 let mut jobs = Vec::new();
3628 let mut seats_at = Vec::new();
3629 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3630 if self
3631 .state
3632 .judgements
3633 .get(j)
3634 .is_some_and(|r| r.failed.is_some())
3635 {
3636 continue;
3637 }
3638 let seat_key = format!("judge-{}", j + 1);
3639 let spec = self.occupant(&seat_key, spec);
3640 let seat = self.seat(&seat_key, &spec.id);
3641 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3642 let text = if has_context(&spec, &seat, sessions) {
3643 prompt::final_vote(&viable, &language)
3644 } else {
3645 full.clone()
3646 };
3647 jobs.push(SeatJob {
3648 spec,
3649 seat,
3650 prompt: text,
3651 cwd: root.join(format!("judge-{}", j + 1)),
3652 timeout,
3653 allow_write: false,
3654 sessions,
3655 artifacts: artifacts.clone(),
3656 stem: format!("vote-judge-{}", j + 1),
3657 handover: Some(full),
3658 });
3659 seats_at.push(j);
3660 }
3661
3662 self.state.event(
3663 "vote",
3664 format!(
3665 "collecting {} final votes one by one, privately",
3666 jobs.len()
3667 ),
3668 );
3669 let allowed = viable.clone();
3670 let mut quota_losses = Vec::new();
3671 let cache = self.state.config.cache_dir();
3672 let ctx = WaveCtx {
3673 carry_seats: false,
3674 pauses: Some((&self.pause, &self.interrupt)),
3675 run: &run_id,
3676 node: "vote",
3677 prompts: &prompts,
3678 cache: cache.as_deref(),
3679 round: None,
3680 };
3681 let results = ask_json_wave::<FinalVote>(
3682 jobs,
3683 Arc::clone(&self.sem),
3684 self.state.config.graph.retries,
3685 &self.roles.judge_roster,
3686 &ctx,
3687 &mut quota_losses,
3688 &mut self.state,
3689 &move |v: &FinalVote| match v.label() {
3690 Some(c) if allowed.contains(&c) => Ok(()),
3691 other => bail!("vote {other:?} is not one of {allowed:?}"),
3692 },
3693 )
3694 .await;
3695 if self.state.resource_block.is_some() {
3696 return self.stop_for_resource();
3697 }
3698 self.state.quota.extend(quota_losses);
3699
3700 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3701 let agent_id = seat.agent.clone();
3702 self.state.seats.insert(seat.key.clone(), seat);
3703 let initial = self
3704 .state
3705 .judgements
3706 .get(j)
3707 .and_then(|r| r.ranking.first().copied());
3708 let mut record = VoteRecord {
3709 judge: j + 1,
3710 agent: agent_id,
3711 vote: None,
3712 reason: String::new(),
3713 changed: false,
3714 };
3715 match res {
3716 Ok((v, _)) => {
3717 record.vote = v.label();
3718 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3719 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3720 self.state.event(
3721 "vote",
3722 format!(
3723 "judge {} voted {}{}",
3724 j + 1,
3725 record.vote.unwrap_or('?'),
3726 if record.changed { " (changed)" } else { "" }
3727 ),
3728 );
3729 }
3730 Err(e) => {
3731 self.state
3732 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3733 }
3734 }
3735 self.state.votes.push(record);
3736 self.state.save()?;
3737 }
3738 Ok(())
3739 }
3740
3741 fn tally(&mut self) -> Result<()> {
3744 if self.state.tally.is_some() {
3745 return Ok(());
3746 }
3747 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3748 let tops: Vec<char> = self
3749 .state
3750 .judgements
3751 .iter()
3752 .filter_map(|j| j.ranking.first().copied())
3753 .collect();
3754 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3755
3756 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3759 let mut cast: Vec<char> = Vec::new();
3760 for (i, j) in self.state.judgements.iter().enumerate() {
3761 let vote = self
3762 .state
3763 .votes
3764 .iter()
3765 .find(|v| v.judge == i + 1)
3766 .and_then(|v| v.vote)
3767 .or_else(|| j.ranking.first().copied());
3768 if let Some(v) = vote {
3769 *first_choice.entry(v).or_insert(0) += 1;
3770 cast.push(v);
3771 }
3772 }
3773
3774 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3775 for j in &self.state.judgements {
3776 let n = j.ranking.len();
3777 for (pos, label) in j.ranking.iter().enumerate() {
3778 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3779 }
3780 }
3781
3782 let best = first_choice.values().copied().max().unwrap_or(0);
3783 let mut leaders: Vec<char> = first_choice
3784 .iter()
3785 .filter(|(_, v)| **v == best)
3786 .map(|(k, _)| *k)
3787 .collect();
3788 let mut tie_break = None;
3789 if leaders.len() > 1 {
3790 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3791 let borda_leaders: Vec<char> = leaders
3792 .iter()
3793 .copied()
3794 .filter(|l| borda[l] == top_borda)
3795 .collect();
3796 tie_break = Some(if borda_leaders.len() == 1 {
3797 format!(
3798 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3799 leaders.len()
3800 )
3801 } else {
3802 format!(
3803 "{} way tie on both first-choice votes and Borda points, broken by label order",
3804 leaders.len()
3805 )
3806 });
3807 leaders = borda_leaders;
3808 leaders.sort_unstable();
3809 }
3810 let winner = *leaders
3811 .first()
3812 .or(viable.first())
3813 .context("no candidate to declare a winner from")?;
3814
3815 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3816 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3817 let deliberated = !self.state.deliberation.is_empty();
3818
3819 let quota_seats: std::collections::BTreeSet<&str> =
3823 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3824 let mut present = 0usize;
3825 for (i, j) in self.state.judgements.iter().enumerate() {
3826 if quota_seats.contains(j.seat.as_str()) {
3827 continue;
3828 }
3829 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3830 let voted = self
3831 .state
3832 .votes
3833 .iter()
3834 .any(|v| v.judge == i + 1 && v.vote.is_some());
3835 if ranked || voted {
3836 present += 1;
3837 }
3838 }
3839 let needs_quorum = viable.len() > 1;
3845 let judges_total = if needs_quorum {
3846 self.roles.judges.len()
3847 } else {
3848 0
3849 };
3850 let quorum = if needs_quorum {
3851 judges_total / 2 + 1
3852 } else {
3853 0
3854 };
3855 let met_quorum = !needs_quorum || present >= quorum;
3856 let uncontested = (!needs_quorum).then(|| {
3857 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3858 });
3859
3860 self.state.event(
3861 "tally",
3862 match &uncontested {
3863 Some(reason) => format!("winner {winner} — {reason}"),
3864 None => format!(
3865 "winner {winner} — votes {} | initial {} | {} changed | \
3866 {present}/{judges_total} judges{}",
3867 first_choice
3868 .iter()
3869 .map(|(k, v)| format!("{k}:{v}"))
3870 .collect::<Vec<_>>()
3871 .join(" "),
3872 if unanimous_initial {
3873 "unanimous"
3874 } else {
3875 "split"
3876 },
3877 changed_votes,
3878 if met_quorum {
3879 String::new()
3880 } else {
3881 format!(" — below quorum ({quorum} required)")
3882 },
3883 ),
3884 },
3885 );
3886 if !met_quorum {
3887 self.state.event(
3888 "stall",
3889 format!(
3890 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3891 the run stops here, resumable"
3892 ),
3893 );
3894 }
3895 self.state.tally = Some(Tally {
3896 first_choice,
3897 borda,
3898 winner,
3899 rankings: tops.len(),
3900 unanimous_initial,
3901 deliberated,
3902 changed_votes,
3903 unanimous_final,
3904 tie_break,
3905 judges: judges_total,
3906 present,
3907 quorum,
3908 met_quorum,
3909 uncontested,
3910 });
3911 self.state.status = if met_quorum {
3912 RunStatus::Reviewing
3913 } else {
3914 RunStatus::Stalled
3915 };
3916 self.state.save()?;
3917 Ok(())
3918 }
3919
3920 #[allow(clippy::too_many_lines)]
3941 async fn recover_stall(&mut self) -> Result<bool> {
3942 let run_id = self.state.id.clone();
3947 let prompts = self.state.config.prompts.clone();
3948 let quota_seats: BTreeSet<&str> =
3953 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3954 let absent: Vec<String> = self
3955 .state
3956 .judgements
3957 .iter()
3958 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3959 .map(|j| j.seat.clone())
3960 .collect();
3961 if absent.is_empty() {
3962 return Ok(false);
3963 }
3964 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3965 if viable.len() <= 1 {
3966 return Ok(false);
3967 }
3968 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3969 let language = self.state.config.graph.language.clone();
3970 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3971 let sessions = self.state.config.graph.sessions;
3972 let artifacts = agent::artifacts_dir(&self.state.dir());
3973 let root = self.state.worktree_root();
3974 let base_short = short(&self.state.base_commit);
3975 let candidates: Vec<Candidate> = viable.clone();
3976
3977 let mut positions: Vec<usize> = absent
3979 .iter()
3980 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3981 .collect();
3982 if positions.is_empty() {
3983 return Ok(false);
3984 }
3985 positions.sort_unstable();
3986 positions.dedup();
3987
3988 let mut judge_jobs = Vec::new();
3990 for &j in &positions {
3991 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3992 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3993 let seat_key = format!("judge-{}", j + 1);
3994 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3995 let seat = self.seat(&seat_key, &spec.id);
3996 judge_jobs.push(SeatJob {
3997 spec,
3998 seat,
3999 prompt: prompt::judge(
4000 &self.state.instruction,
4001 &views,
4002 self.roles.judges.len(),
4003 &base_short,
4004 &language,
4005 ),
4006 cwd: root.join(seat_key),
4007 timeout,
4008 allow_write: false,
4009 sessions,
4010 artifacts: artifacts.clone(),
4011 stem: format!("judge-{}-recover", j + 1),
4012 handover: None,
4013 });
4014 }
4015
4016 let labels_for_check = labels.clone();
4017 let mut judge_losses = Vec::new();
4018 let retries = self.state.config.graph.retries;
4019 let cache = self.state.config.cache_dir();
4020 let ctx = WaveCtx {
4021 carry_seats: false,
4022 pauses: Some((&self.pause, &self.interrupt)),
4023 run: &run_id,
4024 node: "judge",
4025 prompts: &prompts,
4026 cache: cache.as_deref(),
4027 round: None,
4028 };
4029 let results = ask_json_wave::<Ranking>(
4030 judge_jobs,
4031 Arc::clone(&self.sem),
4032 retries,
4033 &self.roles.judge_roster,
4034 &ctx,
4035 &mut judge_losses,
4036 &mut self.state,
4037 &move |r: &Ranking| r.validate(&labels_for_check),
4038 )
4039 .await;
4040 if self.state.resource_block.is_some() {
4041 self.stop_for_resource()?;
4042 return Ok(false);
4043 }
4044
4045 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4047 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4048 let agent_id = seat.agent.clone();
4049 self.state.seats.insert(seat.key.clone(), seat);
4050 let record = &mut self.state.judgements[j];
4051 match res {
4052 Ok((ranking, out)) => {
4053 record.agent = agent_id;
4054 record.ranking = ranking.normalized();
4055 record.reasons = ranking.reasons;
4056 record.strengths = ranking.strengths;
4057 record.confidence = ranking.confidence;
4058 record.failed = None;
4059 record.duration_ms = out.duration_ms;
4060 recovered.insert(j);
4061 self.state.event(
4062 "recover",
4063 format!("judge {} ranked again after the limit", j + 1),
4064 );
4065 }
4066 Err(e) => {
4067 self.state
4068 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4069 }
4070 }
4071 }
4072
4073 let mut vote_jobs = Vec::new();
4075 let mut vote_pos: Vec<usize> = Vec::new();
4076 for &j in &recovered {
4077 let seat_key = format!("judge-{}", j + 1);
4078 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4079 let seat = self.seat(&seat_key, &spec.id);
4080 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4081 let text = if has_context(&spec, &seat, sessions) {
4082 prompt::final_vote(&labels, &language)
4083 } else {
4084 full.clone()
4085 };
4086 vote_jobs.push(SeatJob {
4087 spec,
4088 seat,
4089 prompt: text,
4090 cwd: root.join(seat_key),
4091 timeout,
4092 allow_write: false,
4093 sessions,
4094 artifacts: artifacts.clone(),
4095 stem: format!("vote-judge-{}-recover", j + 1),
4096 handover: Some(full),
4097 });
4098 vote_pos.push(j);
4099 }
4100 let allowed = labels.clone();
4101 let mut vote_losses = Vec::new();
4102 let vote_retries = self.state.config.graph.retries;
4103 let vote_cache = self.state.config.cache_dir();
4104 let ctx = WaveCtx {
4105 carry_seats: false,
4106 pauses: Some((&self.pause, &self.interrupt)),
4107 run: &run_id,
4108 node: "vote",
4109 prompts: &prompts,
4110 cache: vote_cache.as_deref(),
4111 round: None,
4112 };
4113 let votes = ask_json_wave::<FinalVote>(
4114 vote_jobs,
4115 Arc::clone(&self.sem),
4116 vote_retries,
4117 &self.roles.judge_roster,
4118 &ctx,
4119 &mut vote_losses,
4120 &mut self.state,
4121 &move |v: &FinalVote| match v.label() {
4122 Some(c) if allowed.contains(&c) => Ok(()),
4123 other => bail!("vote {other:?} is not one of {allowed:?}"),
4124 },
4125 )
4126 .await;
4127 if self.state.resource_block.is_some() {
4128 self.stop_for_resource()?;
4129 return Ok(false);
4130 }
4131 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4132 let agent_id = seat.agent.clone();
4133 self.state.seats.insert(seat.key.clone(), seat);
4134 match res {
4135 Ok((v, _)) => {
4136 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4137 rec.vote = v.label();
4138 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4139 } else {
4140 self.state.votes.push(VoteRecord {
4141 judge: j + 1,
4142 agent: agent_id,
4143 vote: v.label(),
4144 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4145 changed: false,
4146 });
4147 }
4148 self.state.event(
4149 "recover",
4150 format!("judge {} voted again after the limit", j + 1),
4151 );
4152 }
4153 Err(e) => {
4154 self.state
4155 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4156 }
4157 }
4158 }
4159
4160 let recovered_keys: BTreeSet<String> = recovered
4164 .iter()
4165 .map(|&j| format!("judge-{}", j + 1))
4166 .collect();
4167 self.state
4168 .quota
4169 .retain(|q| !recovered_keys.contains(&q.seat));
4170 for loss in judge_losses.into_iter().chain(vote_losses) {
4174 if recovered_keys.contains(&loss.seat) {
4175 continue;
4176 }
4177 self.state.quota.retain(|q| q.seat != loss.seat);
4178 self.state.quota.push(loss);
4179 }
4180
4181 self.state.tally = None;
4183 self.tally()?;
4184 Ok(self
4185 .state
4186 .tally
4187 .as_ref()
4188 .map(|t| t.met_quorum)
4189 .unwrap_or(false))
4190 }
4191
4192 async fn graft(&mut self) -> Result<()> {
4208 let Err(e) = self.graft_inner().await else {
4209 return Ok(());
4210 };
4211 if e.is::<crate::repo_lock::ResourceBlocked>() {
4212 return Err(e);
4213 }
4214 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4215 self.state.status = RunStatus::Blocked;
4216 self.state.event("graft", format!("blocked: {e:#}"));
4217 self.state.save()?;
4218 return Err(e);
4219 }
4220 if let Some(rec) = self.state.graft.clone()
4225 && !rec.outcome.settled()
4226 {
4227 if let Some(winner) = self.state.winner().cloned() {
4228 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4229 if re.is::<crate::repo_lock::ResourceBlocked>() {
4230 return Err(re);
4231 }
4232 self.state.status = RunStatus::Blocked;
4233 self.state.event("graft", format!("blocked: {re:#}"));
4234 self.state.save()?;
4235 return Err(re);
4236 }
4237 }
4238 if let Some(g) = self.state.graft.as_mut() {
4239 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4240 }
4241 }
4242 self.state.event("graft", format!("skipped: {e:#}"));
4243 if let Err(e) = self.state.save() {
4244 tracing::warn!("save after graft: {e:#}");
4245 }
4246 Ok(())
4247 }
4248
4249 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4252 let _coordination = self.coordinate(wt).await?;
4253 crate::recovery::restore_repair(wt, pre)
4254 .await
4255 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4256 }
4257
4258 fn graft_skipped(&mut self, why: &str) {
4260 let message = format!("skipped: {why}");
4261 if !self
4262 .state
4263 .events
4264 .iter()
4265 .any(|e| e.node == "graft" && e.message == message)
4266 {
4267 self.state.event("graft", message);
4268 }
4269 }
4270
4271 async fn graft_inner(&mut self) -> Result<()> {
4272 if let Some(rec) = self.state.graft.clone()
4275 && !rec.outcome.settled()
4276 {
4277 if self.state.reviews.is_empty()
4281 && let Some(winner) = self.state.winner().cloned()
4282 {
4283 self.graft_restore(&winner.worktree, &rec.pre).await?;
4284 }
4285 if rec.outcome == GraftOutcome::WaitingForRestore {
4286 self.state.graft = None;
4287 self.state.save()?;
4288 } else {
4289 if let Some(g) = self.state.graft.as_mut() {
4290 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4291 }
4292 self.state.event(
4293 "graft",
4294 "rolled back: the previous round was interrupted; the winner is untouched",
4295 );
4296 self.state.save()?;
4297 return Ok(());
4298 }
4299 }
4300 let plan = match graft::plan(&self.state) {
4301 Ok(plan) => plan,
4302 Err(skip) => {
4303 if skip != graft::Skip::AlreadyDone {
4304 self.graft_skipped(skip.reason());
4305 self.state.save()?;
4306 }
4307 return Ok(());
4308 }
4309 };
4310 let Some(winner) = self.state.winner().cloned() else {
4311 return Ok(());
4312 };
4313 let wt = winner.worktree.clone();
4314 if !git::is_clean(&wt).await? {
4315 self.graft_skipped("the winner's worktree has uncommitted changes");
4316 self.state.save()?;
4317 return Ok(());
4318 }
4319 let pre = git::rev_parse(&wt, "HEAD").await?;
4320 self.state.graft = Some(GraftRecord {
4323 pre: pre.clone(),
4324 post: None,
4325 lines: None,
4326 limit: None,
4327 agent: None,
4328 outcome: GraftOutcome::Running,
4329 requested: graft::requested(&plan),
4330 adoption: None,
4331 adoption_error: None,
4332 });
4333 self.state.event(
4334 "graft",
4335 format!(
4336 "porting strengths of candidate(s) {} onto {}",
4337 plan.strengths
4338 .iter()
4339 .map(|(l, _, _)| l.to_string())
4340 .collect::<Vec<_>>()
4341 .join(", "),
4342 plan.winner
4343 ),
4344 );
4345 self.state.save()?;
4346
4347 let fix_prompt = prompt::graft(
4348 &self.state.instruction,
4349 plan.winner,
4350 &plan.strengths,
4351 &self.state.config.graph.language,
4352 );
4353 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4354 let sessions = self.state.config.graph.sessions;
4355 let artifacts = agent::artifacts_dir(&self.state.dir());
4356 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4357 let (_, seat, out) = self
4358 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4359 SeatJob {
4360 prompt: fix_prompt.clone(),
4361 spec,
4362 seat,
4363 cwd: wt.clone(),
4364 timeout,
4365 allow_write: true,
4366 sessions,
4367 artifacts: artifacts.clone(),
4368 stem: "graft".to_owned(),
4369 handover: None,
4370 }
4371 })
4372 .await;
4373 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4374 self.state.graft.as_mut().expect("graft record").outcome =
4377 GraftOutcome::WaitingForRestore;
4378 return self.stop_for_resource();
4379 }
4380 let mut why: Option<String> = None;
4381 match &out {
4382 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4383 why = Some("the fixer produced nothing usable".to_owned());
4384 }
4385 AgentOutcome::Ok(_) => {}
4386 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4387 AgentOutcome::Quota(o) => {
4388 self.state.quota.push(QuotaLoss {
4389 seat: seat.key.clone(),
4390 node: "graft".to_owned(),
4391 at: Timestamp::now(),
4392 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4393 });
4394 why = Some("rate limited (quota)".to_owned());
4395 }
4396 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4397 why = Some(format!(
4398 "the fixer failed: {}",
4399 e.lines()
4400 .next()
4401 .unwrap_or("")
4402 .chars()
4403 .take(160)
4404 .collect::<String>()
4405 ));
4406 }
4407 }
4408 let agent_id = seat.agent.clone();
4409 self.state.seats.insert(seat.key.clone(), seat);
4410
4411 let mut post = None;
4412 let mut lines = None;
4413 let _coordination = self.coordinate(&wt).await?;
4414 if why.is_none() {
4415 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4416 if let Ok(r) = &rescued {
4417 self.state.note_withheld("graft", &r.withheld);
4418 }
4419 let head = git::rev_parse(&wt, "HEAD").await?;
4420 let dirty = match &rescued {
4424 Ok(r) => git::status_porcelain(&wt)
4425 .await
4426 .map(|st| {
4427 st.lines().any(|l| {
4428 !r.withheld
4429 .iter()
4430 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4431 })
4432 })
4433 .unwrap_or(true),
4434 Err(_) => true,
4435 };
4436 if dirty {
4437 why = Some("the fixer left changes that could not be committed".to_owned());
4438 } else if head == pre {
4439 why = Some("the fixer changed nothing".to_owned());
4440 } else {
4441 lines = git::changed_lines(&wt, &pre, &head).await;
4442 post = Some(head);
4443 }
4444 }
4445 let (adoption, adoption_error) = match &out {
4446 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4447 _ => (None, None),
4448 };
4449 let requested = graft::requested(&plan);
4450 let outcome = match why {
4451 None => {
4452 self.state.event(
4453 "graft",
4454 format!(
4455 "applied: {}",
4456 lines.map_or_else(
4457 || "changed lines unavailable".to_owned(),
4458 |n| format!("{n} changed lines")
4459 )
4460 ),
4461 );
4462 GraftOutcome::Applied
4463 }
4464 Some(why) => {
4465 crate::recovery::restore_repair(&wt, &pre)
4468 .await
4469 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4470 self.state.event(
4471 "graft",
4472 format!("rolled back: {why}; the winner is untouched"),
4473 );
4474 GraftOutcome::RolledBack(why)
4475 }
4476 };
4477 self.state.graft = Some(GraftRecord {
4478 pre,
4479 post,
4480 lines,
4481 limit: None,
4482 agent: Some(agent_id),
4483 outcome,
4484 requested,
4485 adoption,
4486 adoption_error,
4487 });
4488 self.state.save()?;
4489 Ok(())
4490 }
4491
4492 async fn fold_losers(&mut self) -> Result<()> {
4495 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4496 return Ok(());
4497 };
4498 if !self.state.candidate_preservation.is_empty() {
4499 bail!("candidate preservation incomplete; refusing cleanup");
4500 }
4501 let repo = self.state.repo.clone();
4502 let _coordination = self.coordinate(&repo).await?;
4503 let mut folded = Vec::new();
4504 for i in 0..self.state.candidates.len() {
4505 let c = &self.state.candidates[i];
4506 if c.label == winner || c.folded {
4507 continue;
4508 }
4509 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4510 remove_preserved_worktree(&repo, &wt).await?;
4511 if git::branch_exists(&repo, &branch).await?
4512 && !git::branch_delete(&repo, &branch).await?
4513 {
4514 bail!("candidate branch removal refused");
4515 }
4516 self.state.candidates[i].folded = true;
4517 self.state.save()?;
4518 folded.push(label.to_string());
4519 }
4520 let root = self.state.worktree_root();
4522 for j in 1..=self.roles.judges.len() {
4523 let wt = root.join(format!("judge-{j}"));
4524 if wt.exists() {
4525 remove_preserved_worktree(&repo, &wt).await?;
4526 }
4527 }
4528 if self.state.config.graph.advise {
4531 for k in 1..=self.state.config.graph.advisors {
4532 let wt = root.join(format!("advisor-{k}"));
4533 if wt.exists() {
4534 remove_preserved_worktree(&repo, &wt).await?;
4535 }
4536 }
4537 }
4538 if !folded.is_empty() {
4539 self.state
4540 .event("fold", format!("folded candidates {}", folded.join(", ")));
4541 self.state.save()?;
4542 }
4543 Ok(())
4544 }
4545
4546 async fn sync_to_base(&mut self) -> Result<()> {
4586 if self.state.status == RunStatus::AlreadyInBase {
4587 return Ok(());
4588 }
4589 let conflicted = self
4590 .state
4591 .base_sync
4592 .as_ref()
4593 .is_some_and(|s| s.conflict.is_some());
4594 let Some(winner) = self.state.winner().cloned() else {
4595 return Ok(());
4596 };
4597
4598 let repo = self.state.repo.clone();
4599 let remote = self.state.config.merge.remote.clone();
4600 let base_branch = self.state.base_branch.clone();
4601 let tracking = format!("{remote}/{base_branch}");
4602
4603 git::fetch(&repo, &remote, &base_branch).await.ok();
4604 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4609 return Ok(());
4610 };
4611
4612 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4613 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4614 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4615
4616 if (behind > 0 || conflicted || head == tip)
4625 && self
4626 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4627 .await?
4628 {
4629 return Ok(());
4630 }
4631 if conflicted {
4632 return Ok(());
4633 }
4634
4635 if behind == 0 {
4636 if let Some(from) = self
4645 .state
4646 .rebase_fixes
4647 .iter()
4648 .rev()
4649 .find_map(|r| r.from.clone())
4650 && from != head
4651 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4652 .await
4653 .is_ok_and(|o| o.ok())
4654 {
4655 git::sync_to_head(&winner.worktree).await?;
4656 }
4657 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4663 if let Some(why) = &conflict {
4664 self.state.status = RunStatus::Blocked;
4665 self.state.event("land", why.clone());
4666 }
4667 self.state.base_sync = Some(BaseSync {
4668 tip,
4669 behind: 0,
4670 attempts,
4671 conflict,
4672 already_in: None,
4673 });
4674 self.state.save()?;
4675 return Ok(());
4676 }
4677
4678 if attempts >= BASE_SYNC_ROUNDS {
4679 let why = format!(
4680 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4681 rebase(s); rebasing again would only race it",
4682 winner.branch
4683 );
4684 self.state.status = RunStatus::Blocked;
4685 self.state.base_sync = Some(BaseSync {
4686 tip,
4687 behind,
4688 attempts,
4689 conflict: Some(why.clone()),
4690 already_in: None,
4691 });
4692 self.state.event("land", why);
4693 self.state.save()?;
4694 return Ok(());
4695 }
4696
4697 self.state.event(
4698 "land",
4699 format!(
4700 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4701 winner.branch
4702 ),
4703 );
4704 self.state.save()?;
4705
4706 let branch_tracking = format!("{remote}/{}", winner.branch);
4712 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4713 let remote_tip = if self.state.task_pr_branch().is_none()
4714 && matches!(&fetched_branch, Ok(o) if o.ok())
4715 {
4716 git::rev_parse(&repo, &branch_tracking).await.ok()
4717 } else {
4718 None
4719 };
4720 if let Some(theirs) = &remote_tip
4724 && !git::is_ancestor(&repo, theirs, &head).await
4725 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4726 .await
4727 .is_ok_and(|missing| missing.is_empty())
4728 {
4729 let why = format!(
4730 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4731 them",
4732 short(theirs),
4733 winner.branch
4734 );
4735 self.state.status = RunStatus::Blocked;
4736 self.state.base_sync = Some(BaseSync {
4737 tip,
4738 behind,
4739 attempts,
4740 conflict: Some(why.clone()),
4741 already_in: None,
4742 });
4743 self.state.event("land", why);
4744 self.state.save()?;
4745 return Ok(());
4746 }
4747
4748 let scratch = self.state.dir().join("base-sync");
4749 let rebased = match crate::rebase::rebase_with_fixer(
4750 &mut self.state,
4751 &scratch,
4752 &winner.branch,
4753 &tracking,
4754 )
4755 .await
4756 {
4757 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4758 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4759 Err(e) => Err(e),
4760 };
4761 let attempts = attempts + 1;
4762 match rebased {
4763 Ok(None) => {
4764 git::sync_to_head(&winner.worktree).await?;
4768 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4769 let mut conflict = None;
4770 if let Some(pinned) = &remote_tip {
4771 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4772 match pushed {
4773 Ok(o) if o.ok() => self.state.event(
4774 "land",
4775 format!("pushed rebased {} to {remote}", winner.branch),
4776 ),
4777 Ok(o) => {
4778 conflict = Some(format!(
4779 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4780 winner.branch,
4781 short(pinned),
4782 o.stderr.chars().take(600).collect::<String>()
4783 ));
4784 }
4785 Err(e) => {
4786 conflict = Some(format!(
4787 "rebased {} locally but could not push it: {e:#}",
4788 winner.branch
4789 ));
4790 }
4791 }
4792 }
4793 if let Some(why) = &conflict {
4794 self.state.status = RunStatus::Blocked;
4795 self.state.event("land", why.clone());
4796 }
4797 self.state.base_sync = Some(BaseSync {
4798 tip: tip.clone(),
4799 behind: 0,
4800 attempts,
4801 conflict,
4802 already_in: None,
4803 });
4804 self.state
4805 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4806 }
4807 Ok(Some(conflict)) => {
4808 let why = format!(
4809 "{} conflicts with {tracking} and did not rebase: {}",
4810 winner.branch,
4811 conflict.chars().take(600).collect::<String>()
4812 );
4813 self.state.status = RunStatus::Blocked;
4814 self.state.base_sync = Some(BaseSync {
4815 tip,
4816 behind,
4817 attempts,
4818 conflict: Some(why.clone()),
4819 already_in: None,
4820 });
4821 self.state.event("land", why);
4822 }
4823 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4827 self.state.save()?;
4828 return Ok(());
4829 }
4830 Err(e) => {
4831 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4832 self.state.status = RunStatus::Blocked;
4833 self.state.base_sync = Some(BaseSync {
4834 tip,
4835 behind,
4836 attempts,
4837 conflict: Some(why.clone()),
4838 already_in: None,
4839 });
4840 self.state.event("land", why);
4841 }
4842 }
4843 self.state.save()?;
4844 Ok(())
4845 }
4846
4847 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4853 if self.state.task_pr_branch().is_some() {
4856 return None;
4857 }
4858 let repo = self.state.repo.clone();
4859 let remote = self.state.config.merge.remote.clone();
4860 let fetched = git::fetch(&repo, &remote, branch).await;
4861 if !matches!(&fetched, Ok(o) if o.ok()) {
4862 return None;
4863 }
4864 let branch_tracking = format!("{remote}/{branch}");
4865 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4866 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4867 return None;
4868 }
4869 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4870 .await
4871 .is_ok_and(|missing| missing.is_empty())
4872 {
4873 return Some(format!(
4874 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4875 them",
4876 short(&theirs)
4877 ));
4878 }
4879 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4880 Ok(o) if o.ok() => {
4881 self.state
4882 .event("land", format!("pushed rebased {branch} to {remote}"));
4883 None
4884 }
4885 Ok(o) => Some(format!(
4886 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4887 someone may have pushed): {}",
4888 short(&theirs),
4889 o.stderr.chars().take(600).collect::<String>()
4890 )),
4891 Err(e) => Some(format!(
4892 "{branch} is rebased locally but could not be pushed: {e:#}"
4893 )),
4894 }
4895 }
4896
4897 async fn settle_already_in(
4914 &mut self,
4915 branch: &str,
4916 tip: &str,
4917 head: &str,
4918 attempts: usize,
4919 behind: usize,
4920 ) -> Result<bool> {
4921 let repo = self.state.repo.clone();
4922 let remote = self.state.config.merge.remote.clone();
4923 let start = self.state.base_commit.clone();
4924 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4925 Ok(Some(e)) => e,
4926 Ok(None) => return Ok(false),
4927 Err(e) => {
4928 tracing::warn!("already-in-base check for {branch}: {e:#}");
4929 return Ok(false);
4930 }
4931 };
4932 let mut verified = vec![head.to_owned()];
4933 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4934 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4935 if matches!(&fetched, Ok(o) if o.ok())
4936 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4937 && theirs != head
4938 {
4939 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4940 Ok(Some(_)) => verified.push(theirs),
4941 _ => return Ok(false),
4942 }
4943 }
4944 let base_branch = self.state.base_branch.clone();
4945 let message = format!(
4946 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4947 land",
4948 evidence.names(),
4949 evidence.proof.as_str()
4950 );
4951 let closed =
4952 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4953 .await;
4954 match closed {
4955 Ok(Ok(url)) => self
4956 .state
4957 .event("land", format!("closed {url}: superseded on {base_branch}")),
4958 Ok(Err(why)) => self
4959 .state
4960 .event("land", format!("did not close a pull request: {why}")),
4961 Err(e) => {
4962 let why = format!(
4963 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4964 not be closed ({e:#}); resume to retry"
4965 );
4966 self.state.status = RunStatus::Blocked;
4967 self.state.base_sync = Some(BaseSync {
4968 tip: tip.to_owned(),
4969 behind,
4970 attempts,
4971 conflict: Some(why.clone()),
4972 already_in: None,
4973 });
4974 self.state.event("land", why);
4975 self.state.save()?;
4976 return Ok(true);
4977 }
4978 }
4979 self.state.status = RunStatus::AlreadyInBase;
4980 self.state.base_sync = Some(BaseSync {
4981 tip: tip.to_owned(),
4982 behind,
4983 attempts,
4984 conflict: None,
4985 already_in: Some(evidence),
4986 });
4987 self.state.event("land", message);
4988 self.state.save()?;
4989 self.settle_questions();
4990 Ok(true)
4991 }
4992
4993 fn landing_base(&self) -> String {
5003 self.state
5004 .base_sync
5005 .as_ref()
5006 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5007 }
5008
5009 pub async fn fix_selected(
5042 &mut self,
5043 ids: &[String],
5044 reason: &str,
5045 allow_stale: bool,
5046 ) -> Result<()> {
5047 let reason = reason.trim();
5048 if reason.is_empty() {
5049 bail!("a fix request needs a reason — that is the operator's own record of why");
5050 }
5051 if ids.is_empty() {
5052 bail!("no finding id given");
5053 }
5054 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5055 bail!(
5056 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5057 has already concluded — can be given a targeted fix. A run still \
5058 in progress should simply be resumed; a `merged` run's branch has \
5059 already landed, so its answer is a fresh `magi review <branch>`, \
5060 not reopening this run's own record",
5061 self.state.id,
5062 self.state.status.as_str()
5063 );
5064 }
5065 let Some(winner) = self.state.winner().cloned() else {
5066 bail!("run {} has no winning candidate to fix", self.state.id);
5067 };
5068 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5069 bail!(
5070 "branch `{}` no longer exists; this run cannot be extended",
5071 winner.branch
5072 );
5073 }
5074 let home = crate::run::home();
5075 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5076 bail!(
5077 "run {} is currently being worked on by another magi process",
5078 self.state.id
5079 );
5080 }
5081 let _claim = FixClaim::acquire(&self.state.dir())?;
5087
5088 let mut seen = BTreeSet::new();
5092 let mut findings = Vec::new();
5093 let mut missing = Vec::new();
5094 for id in ids {
5095 if !seen.insert(id.clone()) {
5096 continue;
5097 }
5098 match self.state.finding(id) {
5099 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5100 id: f.id.clone(),
5101 severity: f.severity,
5102 reviewer_vote: rec.vote,
5103 round: round.round,
5104 round_head: round.head.clone(),
5105 reviewer: rec.reviewer,
5106 agent: rec.agent.clone(),
5107 file: f.file.clone(),
5108 line: f.line,
5109 title: f.title.clone(),
5110 detail: f.detail.clone(),
5111 outcome: OperatorFixOutcome::Pending,
5112 }),
5113 None => missing.push(id.clone()),
5114 }
5115 }
5116 if !missing.is_empty() {
5117 bail!(
5118 "unknown finding id(s): {}; nothing was changed",
5119 missing.join(", ")
5120 );
5121 }
5122
5123 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5124 let stale_details: Vec<(String, String)> = findings
5125 .iter()
5126 .filter(|f| f.round_head != head_at_request)
5127 .map(|f| (f.id.clone(), f.round_head.clone()))
5128 .collect();
5129 let stale = !stale_details.is_empty();
5130 if stale && !allow_stale {
5131 bail!(
5132 "the branch has moved since some finding(s) were raised — {} — now \
5133 at {}; pass --allow-stale to fix anyway, or re-run review first",
5134 stale_details
5135 .iter()
5136 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5137 .collect::<Vec<_>>()
5138 .join(", "),
5139 short(&head_at_request)
5140 );
5141 }
5142
5143 let request = OperatorFixRequest {
5144 requested_at: Timestamp::now(),
5145 reason: reason.to_owned(),
5146 findings,
5147 head_at_request: head_at_request.clone(),
5148 allow_stale,
5149 stale,
5150 fix: None,
5151 result_head: None,
5152 follow_up_review_run: None,
5153 };
5154 self.state.event(
5155 "fix",
5156 format!(
5157 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5158 request.findings.len(),
5159 request
5160 .findings
5161 .iter()
5162 .map(|f| f.id.as_str())
5163 .collect::<Vec<_>>()
5164 .join(", "),
5165 ),
5166 );
5167 self.state.operator_fixes.push(request);
5174 self.state.save()?;
5175 let request_index = self.state.operator_fixes.len() - 1;
5176
5177 let repo = self.state.repo.clone();
5186 let coordination = self.coordinate(&repo).await?;
5187 if winner.worktree.exists() {
5188 let dirty = git::git(
5191 &winner.worktree,
5192 &["status", "--porcelain", "--untracked-files=all"],
5193 )
5194 .await?;
5195 let only_withheld = dirty.lines().all(|l| {
5196 l.strip_prefix("?? ")
5197 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5198 });
5199 if !only_withheld {
5200 bail!(
5201 "`{}` has uncommitted changes; refusing to touch it — commit or \
5202 discard them first",
5203 winner.worktree.display()
5204 );
5205 }
5206 git::worktree_remove(&self.state.repo, &winner.worktree)
5207 .await
5208 .ok();
5209 }
5210 let fix_worktree = self.state.worktree_root().join("operator-fix");
5211 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5212 git::git(
5213 &self.state.repo,
5214 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5215 )
5216 .await
5217 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5218 if !git::is_clean(&fix_worktree).await? {
5219 git::worktree_remove(&self.state.repo, &fix_worktree)
5220 .await
5221 .ok();
5222 bail!(
5223 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5224 winner.branch
5225 );
5226 }
5227 let repo_for_setup = self.state.repo.clone();
5228 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5229 Ok(()) => {
5230 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5231 }
5232 Err(e) => Err(e),
5233 };
5234 if let Err(e) = setup {
5235 git::worktree_remove(&self.state.repo, &fix_worktree)
5236 .await
5237 .ok();
5238 return Err(e);
5239 }
5240
5241 let run_id = self.state.id.clone();
5242 let prompts = self.state.config.prompts.clone();
5243 let language = self.state.config.graph.language.clone();
5244 let sessions = self.state.config.graph.sessions;
5245 let artifacts = agent::artifacts_dir(&self.state.dir());
5246 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5247 .findings
5248 .iter()
5249 .map(|f| Finding {
5250 id: f.id.clone(),
5251 severity: f.severity,
5252 file: f.file.clone(),
5253 line: f.line,
5254 title: f.title.clone(),
5255 detail: f.detail.clone(),
5256 })
5257 .collect();
5258 let fix_prompt = prompt::operator_fix_in(
5259 &self.state.instruction,
5260 &finding_list,
5261 reason,
5262 &stale_details,
5263 &head_at_request,
5264 &language,
5265 &self.state.config.graph.github_language,
5266 );
5267 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5268 drop(coordination);
5269 let (job, seat, out) = self
5270 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5271 prompt: fix_prompt.clone(),
5272 spec,
5273 seat,
5274 cwd: fix_worktree.clone(),
5275 timeout,
5276 allow_write: true,
5277 sessions,
5278 artifacts: artifacts.clone(),
5279 stem: "operator-fix".to_owned(),
5280 handover: None,
5281 })
5282 .await;
5283 let agent_id = seat.agent.clone();
5284
5285 let mut fix = FixRecord {
5286 agent: agent_id,
5287 addressed: Vec::new(),
5288 rejected: Vec::new(),
5289 notes: String::new(),
5290 committed: false,
5291 failed: None,
5292 duration_ms: 0,
5293 continuation: None,
5294 };
5295 let mut final_seat = seat.clone();
5296 match out {
5297 AgentOutcome::Ok(o) => {
5298 fix.duration_ms = o.duration_ms;
5299 let parsed = verdict::extract_json::<FixReport>(&o.text);
5300 let incomplete_reason = match &parsed {
5301 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5302 "the reply parsed, but it reported a command whose own CLI \
5303 never confirmed an exit status"
5304 .to_owned(),
5305 ),
5306 Ok(_) => None,
5307 Err(e) => Some(e.to_string()),
5308 };
5309 match incomplete_reason {
5310 None => {
5311 let report = parsed.expect("checked Ok above");
5312 fix.addressed = report.addressed;
5313 fix.rejected = report.rejected;
5314 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5315 }
5316 Some(reason) => {
5317 let (resumed_seat, resolved, failure, cont) = self
5318 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5319 .await;
5320 fix.duration_ms += cont.cumulative_wait_ms;
5321 fix.continuation = Some(cont);
5322 final_seat = resumed_seat;
5323 match resolved {
5324 Some(report) => {
5325 fix.addressed = report.addressed;
5326 fix.rejected = report.rejected;
5327 fix.notes =
5328 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5329 }
5330 None => fix.failed = failure,
5331 }
5332 }
5333 }
5334 }
5335 AgentOutcome::Dropped(o) => {
5336 fix.duration_ms = o.duration_ms;
5337 let why = o
5338 .dropped
5339 .as_ref()
5340 .map(|d| d.why.as_str())
5341 .unwrap_or("the CLI ended the stream without delivering its answer");
5342 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5343 }
5344 AgentOutcome::Quota(o) => {
5345 self.state.quota.push(QuotaLoss {
5346 seat: final_seat.key.clone(),
5347 node: "fix".to_owned(),
5348 at: Timestamp::now(),
5349 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5350 });
5351 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5352 }
5353 AgentOutcome::Failed(e) => fix.failed = Some(e),
5354 AgentOutcome::ResourceBlocked(e) => {
5355 fix.failed = Some(format!("the fixer was not started: {e}"));
5356 }
5357 }
5358 if fix.continuation.is_none() {
5359 fix.continuation = Some(ContinuationRecord::not_needed());
5360 }
5361 self.state.seats.insert(final_seat.key.clone(), final_seat);
5362
5363 let rescue_message = format!(
5364 "magi: operator-selected fix ({}) (uncommitted work)",
5365 self.state.operator_fixes[request_index]
5366 .findings
5367 .iter()
5368 .map(|f| f.id.as_str())
5369 .collect::<Vec<_>>()
5370 .join(", ")
5371 );
5372 let repo = self.state.repo.clone();
5373 let coordination = self.coordinate(&repo).await?;
5374 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5375 self.state.note_withheld("fix", &r.withheld);
5376 }
5377 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5378 fix.committed = after != head_at_request;
5379 git::worktree_remove(&self.state.repo, &fix_worktree)
5380 .await
5381 .ok();
5382 drop(coordination);
5383
5384 self.state.event(
5385 "fix",
5386 match &fix.failed {
5387 Some(reason) => format!(
5388 "operator fix: adoption report was lost ({reason}); {}",
5389 if fix.committed {
5390 "committed"
5391 } else {
5392 "NO new commit"
5393 }
5394 ),
5395 None => format!(
5396 "operator fix: {} addressed, {} rejected, {}",
5397 fix.addressed.len(),
5398 fix.rejected.len(),
5399 if fix.committed {
5400 "committed"
5401 } else {
5402 "NO new commit"
5403 }
5404 ),
5405 },
5406 );
5407
5408 for f in &mut self.state.operator_fixes[request_index].findings {
5415 f.outcome = if fix.failed.is_some() {
5416 OperatorFixOutcome::Unreported
5417 } else if fix.addressed.contains(&f.id) {
5418 OperatorFixOutcome::Addressed
5419 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5420 OperatorFixOutcome::Rejected { why: r.why.clone() }
5421 } else {
5422 OperatorFixOutcome::Unreported
5423 };
5424 }
5425
5426 let committed = fix.committed;
5427 if committed {
5428 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5429 }
5430 self.state.operator_fixes[request_index].fix = Some(fix);
5431 self.state.save()?;
5434
5435 if committed {
5436 self.state.event(
5437 "fix",
5438 format!(
5439 "operator fix committed {}; opening a follow-up review-only run",
5440 short(&after)
5441 ),
5442 );
5443 let origin =
5446 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5447 match Self::review(
5448 &self.state.repo,
5449 &winner.branch,
5450 self.state.config.clone(),
5451 origin,
5452 )
5453 .await
5454 {
5455 Ok(mut follow_up) => {
5456 follow_up.state.event(
5457 "start",
5458 format!(
5459 "requested by an operator fix on run {} for finding(s) {}",
5460 self.state.id,
5461 self.state.operator_fixes[request_index]
5462 .findings
5463 .iter()
5464 .map(|f| f.id.as_str())
5465 .collect::<Vec<_>>()
5466 .join(", "),
5467 ),
5468 );
5469 follow_up.state.save()?;
5470 let follow_up_id = follow_up.state.id.clone();
5471 let adopted = match crate::direct::adopt(
5474 &follow_up.state,
5475 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5476 ) {
5477 Ok(a) => a,
5478 Err(e) => {
5479 self.state.event(
5482 "fix",
5483 format!(
5484 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5485 ),
5486 );
5487 self.state.save()?;
5488 return Ok(());
5489 }
5490 };
5491 let executed = follow_up.execute().await;
5492 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5493 if let Some(a) = adopted {
5494 a.finish(&follow_up.state, executed);
5495 }
5496 if let Some(e) = failure {
5497 self.state.event(
5498 "fix",
5499 format!(
5500 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5501 ),
5502 );
5503 }
5504 self.state.operator_fixes[request_index].follow_up_review_run =
5505 Some(follow_up_id);
5506 }
5507 Err(e) => {
5508 self.state.event(
5509 "fix",
5510 format!("committed the fix but could not open a follow-up review: {e:#}"),
5511 );
5512 }
5513 }
5514 self.state.save()?;
5515 }
5516
5517 Ok(())
5518 }
5519
5520 async fn ask_fixer(
5540 &mut self,
5541 winner: &Candidate,
5542 node: &'static str,
5543 round: Option<usize>,
5544 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5545 ) -> (SeatJob, SeatState, AgentOutcome) {
5546 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5547 self.ask_fixer_among(attempts, node, round, None, build)
5548 .await
5549 }
5550
5551 async fn ask_fixer_among(
5556 &mut self,
5557 attempts: Vec<(AgentSpec, String)>,
5558 node: &'static str,
5559 round: Option<usize>,
5560 restore: Option<(&Path, &str)>,
5561 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5562 ) -> (SeatJob, SeatState, AgentOutcome) {
5563 let run_id = self.state.id.clone();
5564 let prompts = self.state.config.prompts.clone();
5565 let cache = self.state.config.cache_dir();
5566 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5567 let mut last = None;
5568 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5569 let seat = self.seat(&key, &spec.id);
5570 let mut job = build(spec.clone(), seat);
5571 if i > 0 {
5572 job.stem = format!("{}-{}", job.stem, spec.id);
5573 }
5574 let ctx = WaveCtx {
5575 carry_seats: false,
5576 pauses: Some((&self.pause, &self.interrupt)),
5577 run: &run_id,
5578 node,
5579 prompts: &prompts,
5580 cache: cache.as_deref(),
5581 round,
5582 };
5583 let (seat, out) =
5584 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5585 let advances = match &out {
5588 AgentOutcome::Ok(o) => agent::output_advances(o),
5589 AgentOutcome::ResourceBlocked(_) => false,
5590 _ => true,
5591 };
5592 if let Some(next) = ids.get(i + 1)
5593 && advances
5594 {
5595 self.state.seats.insert(seat.key.clone(), seat.clone());
5596 let class =
5597 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5598 let reason = match &out {
5599 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5600 other => fail_reason(other),
5601 };
5602 if let Some((wt, rev)) = restore
5606 && let Err(e) = self.graft_restore(wt, rev).await
5607 {
5608 let why =
5609 format!("could not restore the worktree before the next fixer: {e:#}");
5610 tracing::warn!("{why}");
5611 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5612 AgentOutcome::ResourceBlocked(why)
5613 } else {
5614 AgentOutcome::Failed(why)
5615 };
5616 last = Some((job, seat, out));
5617 break;
5618 }
5619 record_handover(
5620 &mut self.state,
5621 node,
5622 &seat.key,
5623 &spec.id,
5624 next,
5625 &class,
5626 &reason,
5627 );
5628 continue;
5629 }
5630 last = Some((job, seat, out));
5631 break;
5632 }
5633 last.expect("the fixer chain always has an entry")
5634 }
5635
5636 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5647 use crate::recovery::{Mode, Repair};
5648 if !self.state.reviews.is_empty() {
5649 return Ok(());
5650 }
5651 let Some(mut rec) = self.state.recovery.clone() else {
5652 return Ok(());
5653 };
5654 if rec.mode != Mode::VerifyRepair {
5655 return Ok(());
5656 }
5657 let wt = winner.worktree.clone();
5658 let mut head = git::rev_parse(&wt, "HEAD").await?;
5659 match rec.repair.clone() {
5660 Repair::Done { .. } => return Ok(()),
5661 Repair::WaitingForRestore { before } => {
5662 self.graft_restore(&wt, &before).await?;
5663 head = before;
5664 rec.repair = Repair::NotStarted;
5665 self.state.recovery = Some(rec.clone());
5666 self.state.save()?;
5667 }
5668 Repair::Started { before } => {
5669 if let Err(e) = self.graft_restore(&wt, &before).await {
5673 if e.is::<crate::repo_lock::ResourceBlocked>() {
5674 return Err(e);
5675 }
5676 return self
5677 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5678 .await;
5679 }
5680 rec.repair = Repair::Done {
5681 before: before.clone(),
5682 after: before,
5683 committed: false,
5684 failed: Some("interrupted; rolled back".to_owned()),
5685 };
5686 self.state
5687 .event("recovery", "the repair was interrupted and rolled back");
5688 self.state.recovery = Some(rec);
5689 self.state.save()?;
5690 return Ok(());
5691 }
5692 Repair::NotStarted => {}
5693 }
5694 if crate::recovery::recheck(&mut rec, &head) {
5695 self.state.event("recovery", rec.summary());
5696 self.state.recovery = Some(rec);
5697 self.state.save()?;
5698 return Ok(());
5699 }
5700 rec.repair = Repair::Started {
5701 before: head.clone(),
5702 };
5703 self.state.recovery = Some(rec.clone());
5704 self.state.save()?;
5705
5706 let language = self.state.config.graph.language.clone();
5707 let sessions = self.state.config.graph.sessions;
5708 let artifacts = agent::artifacts_dir(&self.state.dir());
5709 let fix_prompt = prompt::recovery_fix(
5710 &self.state.instruction,
5711 &rec,
5712 self.state.config.graph.review_rounds,
5713 &language,
5714 );
5715 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5716 self.state.event(
5717 "recovery",
5718 format!(
5719 "repair: the fixer starts from the failed verification on {}",
5720 short(&head)
5721 ),
5722 );
5723 let (_job, seat, out) = self
5724 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5725 prompt: fix_prompt.clone(),
5726 spec,
5727 seat,
5728 cwd: wt.clone(),
5729 timeout,
5730 allow_write: true,
5731 sessions,
5732 artifacts: artifacts.clone(),
5733 stem: "fix-recovery".to_owned(),
5734 handover: None,
5735 })
5736 .await;
5737 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5738 rec.repair = Repair::WaitingForRestore { before: head };
5739 self.state.recovery = Some(rec);
5740 return self.stop_for_resource();
5741 }
5742 let failed = match &out {
5743 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5744 AgentOutcome::Quota(o) => {
5745 self.state.quota.push(QuotaLoss {
5746 seat: seat.key.clone(),
5747 node: "fix".to_owned(),
5748 at: Timestamp::now(),
5749 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5750 });
5751 Some("rate limited (quota); the fixer could not run".to_owned())
5752 }
5753 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5754 AgentOutcome::Failed(e) => Some(e.clone()),
5755 };
5756 self.state.seats.insert(seat.key.clone(), seat);
5757 let coordination = self.coordinate(&wt).await?;
5758 let failed = match failed {
5759 Some(why) => {
5761 if let Err(e) = git::restore_to(&wt, &head).await {
5762 return self
5763 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5764 .await;
5765 }
5766 Some(why)
5767 }
5768 None => {
5769 if let Ok(r) = git::rescue_commit(
5770 &wt,
5771 "magi: recovery repair of the failed verification (uncommitted work)",
5772 )
5773 .await
5774 {
5775 self.state.note_withheld("fix", &r.withheld);
5776 }
5777 None
5778 }
5779 };
5780 let after = git::rev_parse(&wt, "HEAD").await?;
5781 drop(coordination);
5782 self.state.event(
5783 "recovery",
5784 match &failed {
5785 Some(why) => format!("repair: nothing kept ({why})"),
5786 None if after != head => {
5787 format!("repair: committed {}", short(&after))
5788 }
5789 None => "repair: the fixer made no commit".to_owned(),
5790 },
5791 );
5792 rec.repair = Repair::Done {
5793 before: head.clone(),
5794 committed: after != head,
5795 after,
5796 failed,
5797 };
5798 self.state.recovery = Some(rec);
5799 self.state.save()?;
5800 Ok(())
5801 }
5802
5803 async fn stop_recovery(
5806 &mut self,
5807 mut rec: crate::recovery::RecoveryRecord,
5808 head: &str,
5809 why: String,
5810 ) -> Result<()> {
5811 rec.repair = crate::recovery::Repair::Started {
5813 before: head.to_owned(),
5814 };
5815 self.state.recovery = Some(rec);
5816 self.state
5817 .event("recovery", format!("repair could not be undone: {why}"));
5818 self.state.status = RunStatus::Blocked;
5819 self.state.save()?;
5820 bail!("recovery repair could not be rolled back: {why}")
5821 }
5822
5823 async fn review_loop(&mut self) -> Result<()> {
5824 if self
5829 .state
5830 .base_sync
5831 .as_ref()
5832 .is_some_and(|s| s.conflict.is_some())
5833 {
5834 return Ok(());
5835 }
5836 let run_id = self.state.id.clone();
5841 let prompts = self.state.config.prompts.clone();
5842 let Some(winner) = self.state.winner().cloned() else {
5843 return Ok(());
5844 };
5845 let max_rounds = self.state.config.graph.review_rounds;
5846 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5856 if status == RunStatus::Gating {
5859 self.record_contested_handoff();
5860 }
5861 self.state.status = status;
5862 self.state.save()?;
5863 return Ok(());
5864 }
5865 self.state.status = RunStatus::Reviewing;
5866 if self
5876 .state
5877 .reviews
5878 .last()
5879 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5880 {
5881 let shell = self.state.config.shell();
5882 return self
5883 .stop_reviewing(
5884 "the last round's own verification never resolved",
5885 &shell,
5886 &winner.worktree,
5887 )
5888 .await;
5889 }
5890
5891 self.recovery_repair(&winner).await?;
5892 if self.resource_halt()? {
5893 return Ok(());
5894 }
5895
5896 let repo = self.state.repo.clone();
5897 let root = self.state.worktree_root();
5898 let language = self.state.config.graph.language.clone();
5899 let sessions = self.state.config.graph.sessions;
5900 let artifacts = self.merge_repair_artifacts();
5901 let base = self.landing_base();
5902 let base_short = short(&base);
5903 let reviewers = self.roles.reviewers.clone();
5904 let shell = self.state.config.shell();
5905
5906 for round in (self.state.reviews.len() + 1)..=max_rounds {
5907 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5908 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5909 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5910 let prev_verification = self
5919 .state
5920 .reviews
5921 .last()
5922 .and_then(|r| r.verification_summary(&head));
5923 let prior_section = match (&self.state.recovery, round) {
5926 (Some(rec), 1) if self.state.reviews.is_empty() => {
5927 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5928 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5929 Some(crate::recovery::head_of(
5930 &git::diff(&winner.worktree, reviewed, "HEAD")
5931 .await
5932 .unwrap_or_default(),
5933 crate::recovery::SINCE_DIFF_MAX,
5934 ))
5935 }
5936 _ => None,
5937 };
5938 Some(prompt::recovery_review(rec, since.as_deref()))
5939 }
5940 _ => None,
5941 };
5942
5943 let mut jobs = Vec::new();
5947 let coordination = self.coordinate(&repo).await?;
5948 for (r, spec) in reviewers.iter().cloned().enumerate() {
5949 let wt = root.join(format!("review-{}", r + 1));
5950 if wt.exists() {
5953 git::reset_detached(&wt, &head).await?;
5954 } else {
5955 git::worktree_add_detached(&repo, &wt, &head).await?;
5956 }
5957 self.setup_seat_worktree(&repo, &wt, false).await?;
5958 let seat_key = format!("review-{}", r + 1);
5959 let spec = pick_start_spec(
5962 &self.roles.reviewer_roster,
5963 spec,
5964 self.state.seat_history.get(&seat_key),
5965 );
5966 let seat = self.seat(&seat_key, &spec.id);
5967 jobs.push(SeatJob {
5968 prompt: prompt::review(&prompt::ReviewCtx {
5969 instruction: &self.state.instruction,
5970 branch: &winner.branch,
5971 base_short: &base_short,
5972 stat: &stat,
5973 patch: &patch,
5974 verification: prev_verification.as_ref(),
5975 prior: prior_section.as_deref(),
5976 reviewers: reviewers.len(),
5977 round,
5978 rounds: max_rounds,
5979 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5982 lens: Lens::for_seat(r),
5983 language: &language,
5984 github_language: &self.state.config.graph.github_language,
5985 }),
5986 spec,
5987 seat,
5988 cwd: wt,
5989 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5990 allow_write: false,
5991 sessions,
5992 artifacts: artifacts.clone(),
5993 stem: format!("review-{round}-{}", r + 1),
5994 handover: None,
5995 });
5996 }
5997
5998 drop(coordination);
5999 self.state.event(
6000 "review",
6001 format!(
6002 "round {round}: {} reviewers on {}",
6003 jobs.len(),
6004 short(&head)
6005 ),
6006 );
6007 let mut quota_losses = Vec::new();
6008 let review_retries = self.state.config.graph.retries;
6009 let review_cache = self.state.config.cache_dir();
6010 let ctx = WaveCtx {
6011 carry_seats: true,
6012 pauses: Some((&self.pause, &self.interrupt)),
6013 run: &run_id,
6014 node: "review",
6015 prompts: &prompts,
6016 cache: review_cache.as_deref(),
6017 round: Some(round),
6018 };
6019 let results = ask_json_wave::<Review>(
6020 jobs,
6021 Arc::clone(&self.sem),
6022 review_retries,
6023 &self.roles.reviewer_roster,
6024 &ctx,
6025 &mut quota_losses,
6026 &mut self.state,
6027 &|_: &Review| Ok(()),
6028 )
6029 .await;
6030 if self.state.resource_block.is_some() {
6031 return self.stop_for_resource();
6032 }
6033 let round_quota_missing = quota_losses.len();
6037 self.state.quota.extend(quota_losses);
6038
6039 let mut records = Vec::new();
6040 let mut all_findings = Vec::new();
6041 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6042 let agent_id = seat.agent.clone();
6043 self.state.seats.insert(seat.key.clone(), seat);
6044 let mut record = ReviewRecord {
6045 reviewer: r + 1,
6046 agent: agent_id,
6047 summary: String::new(),
6048 findings: Vec::new(),
6049 vote: None,
6050 failed: None,
6051 duration_ms: 0,
6052 attempts,
6058 };
6059 match res {
6060 Ok((review, out)) => {
6061 record.summary =
6069 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6070 record.vote = Some(review.vote);
6071 record.duration_ms = out.duration_ms;
6072 for (n, mut f) in review.findings.into_iter().enumerate() {
6073 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6076 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6077 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6078 f.file = f
6084 .file
6085 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6086 all_findings.push(f.clone());
6087 record.findings.push(f);
6088 }
6089 self.state.event(
6090 "review",
6091 format!(
6092 "round {round}: reviewer {} voted {} with {} finding(s)",
6093 r + 1,
6094 review.vote.label(),
6095 record.findings.len()
6096 ),
6097 );
6098 }
6099 Err(e) => {
6100 record.failed = Some(e.to_string());
6101 self.state.event(
6102 "review",
6103 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6104 );
6105 }
6106 }
6107 records.push(record);
6108 }
6109
6110 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6117 let vote_split =
6118 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6119 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6120 if vote_split {
6121 self.state.event(
6122 "review",
6123 format!(
6124 "round {round}: votes split ({}) — one round of reconsideration",
6125 initial_votes
6126 .iter()
6127 .map(|v| v.label())
6128 .collect::<Vec<_>>()
6129 .join(", ")
6130 ),
6131 );
6132 let panel: Vec<ReviewSeatReport<'_>> = records
6135 .iter()
6136 .filter_map(|r| {
6137 r.vote.map(|vote| ReviewSeatReport {
6138 reviewer: r.reviewer,
6139 vote,
6140 summary: &r.summary,
6141 findings: &r.findings,
6142 })
6143 })
6144 .collect();
6145
6146 let mut jobs = Vec::new();
6147 let mut seats_at = Vec::new();
6148 for (r, spec) in reviewers.iter().cloned().enumerate() {
6149 if records[r].vote.is_none() {
6153 continue;
6154 }
6155 let wt = root.join(format!("review-{}", r + 1));
6156 let seat_key = format!("review-{}", r + 1);
6157 let spec = self.occupant(&seat_key, spec);
6158 let seat = self.seat(&seat_key, &spec.id);
6159 let build = |with_patch: bool| {
6167 prompt::review_reconsider(&ReviewReconsiderCtx {
6168 instruction: &self.state.instruction,
6169 reviewer: r + 1,
6170 lens: Lens::for_seat(r),
6171 panel: &panel,
6172 patch: with_patch.then_some(ReviewPatch {
6173 branch: &winner.branch,
6174 base_short: &base_short,
6175 stat: &stat,
6176 patch: &patch,
6177 }),
6178 round,
6179 rounds: max_rounds,
6180 language: &language,
6181 })
6182 };
6183 let full = build(true);
6184 let prompt = if has_context(&spec, &seat, sessions) {
6185 build(false)
6186 } else {
6187 full.clone()
6188 };
6189 jobs.push(SeatJob {
6190 prompt,
6191 spec,
6192 seat,
6193 cwd: wt,
6194 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6195 allow_write: false,
6196 sessions,
6197 artifacts: artifacts.clone(),
6198 stem: format!("review-{round}-reconsider-{}", r + 1),
6199 handover: Some(full),
6200 });
6201 seats_at.push(r);
6202 }
6203
6204 let mut recon_quota_losses = Vec::new();
6205 let recon_cache = self.state.config.cache_dir();
6206 let recon_ctx = WaveCtx {
6207 carry_seats: true,
6208 pauses: Some((&self.pause, &self.interrupt)),
6209 run: &run_id,
6210 node: "review",
6211 prompts: &prompts,
6212 cache: recon_cache.as_deref(),
6213 round: Some(round),
6214 };
6215 let recon_results = ask_json_wave::<ReviewRevote>(
6216 jobs,
6217 Arc::clone(&self.sem),
6218 review_retries,
6219 &self.roles.reviewer_roster,
6220 &recon_ctx,
6221 &mut recon_quota_losses,
6222 &mut self.state,
6223 &|_: &ReviewRevote| Ok(()),
6224 )
6225 .await;
6226 if self.state.resource_block.is_some() {
6227 return self.stop_for_resource();
6228 }
6229 self.state.quota.extend(recon_quota_losses);
6230
6231 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6232 let agent_id = seat.agent.clone();
6233 self.state.seats.insert(seat.key.clone(), seat);
6234 let mut rec = ReviewRevoteRecord {
6235 reviewer: r + 1,
6236 agent: agent_id,
6237 vote: None,
6238 reason: String::new(),
6239 failed: None,
6240 };
6241 match res {
6242 Ok((rv, _)) => {
6243 rec.vote = Some(rv.vote);
6244 rec.reason =
6245 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6246 self.state.event(
6247 "review",
6248 format!(
6249 "round {round}: reviewer {} revoted {}",
6250 r + 1,
6251 rv.vote.label()
6252 ),
6253 );
6254 }
6255 Err(e) => {
6256 rec.failed = Some(e.to_string());
6257 self.state.event(
6258 "review",
6259 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6260 );
6261 }
6262 }
6263 reconsideration.push(rec);
6264 }
6265 } else if initial_votes.len() > 1 {
6266 self.state.event(
6267 "review",
6268 format!(
6269 "round {round}: votes agreed ({}) — no reconsideration",
6270 initial_votes[0].label()
6271 ),
6272 );
6273 }
6274
6275 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6276 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6277 let defer_e2e =
6288 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6289 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6290 let reason =
6291 format!("{blocking} blocking finding(s) already required a fix this round");
6292 self.state.event(
6293 "verify",
6294 format!(
6295 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6296 {}); it will run once a round has none left",
6297 short(&head)
6298 ),
6299 );
6300 (Vec::new(), false, true, Some(reason))
6301 } else {
6302 let e2e_commands = self.state.config.verify.e2e.clone();
6303 let cache_dir = self.state.config.cache_dir();
6304 let context = format!("round {round}");
6305 let wait = self.repository_wait();
6306 let (e2e, verify_retried) = with_cache_lease(
6307 &mut self.state,
6308 cache_dir.as_deref(),
6309 "e2e",
6310 "e2e",
6311 &winner.worktree,
6312 &head,
6313 verify_timeout,
6314 &context,
6315 &wait,
6316 |state, budget| {
6317 let shell = shell.clone();
6318 let e2e_commands = e2e_commands.clone();
6319 let worktree = winner.worktree.clone();
6320 let context = context.clone();
6321 async move {
6322 run_e2e_with_retry(
6323 state,
6324 &shell,
6325 &e2e_commands,
6326 &worktree,
6327 budget,
6328 &context,
6329 )
6330 .await
6331 }
6332 },
6333 )
6334 .await?;
6335 (e2e, verify_retried, false, None)
6336 };
6337
6338 let expected = records.len();
6339 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6340 let incomplete = answered < expected;
6341 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6342 let policy = self.state.config.graph.incomplete_review;
6343 let clean = round_is_clean(
6344 blocking,
6345 e2e_ok,
6346 answered,
6347 expected,
6348 round_quota_missing,
6349 policy,
6350 );
6351
6352 let mut round_record = ReviewRound {
6353 round,
6354 head: head.clone(),
6355 verified_head: None,
6356 verified_at: None,
6357 reviews: records,
6358 e2e,
6359 verify_retried,
6360 e2e_deferred,
6361 e2e_defer_reason,
6362 fix: None,
6363 blocking,
6364 answered,
6365 expected,
6366 clean,
6367 progressed: false,
6368 vote_split,
6369 reconsideration,
6370 verdict: None,
6371 };
6372 round_record.verdict = ReviewVote::worst(
6376 round_record
6377 .final_votes()
6378 .into_iter()
6379 .map(|(_, _, vote)| vote),
6380 );
6381 if !matches!(
6392 round_record.e2e_status(),
6393 E2eStatus::Deferred | E2eStatus::NotConfigured
6394 ) {
6395 round_record.verified_head = Some(head.clone());
6396 round_record.verified_at = Some(Timestamp::now());
6397 }
6398 let this_round_verification = round_record.verification_summary(&head);
6399
6400 if incomplete {
6401 let missing: Vec<String> = round_record
6402 .reviews
6403 .iter()
6404 .filter(|r| r.failed.is_some())
6405 .map(|r| format!("review-{}", r.reviewer))
6406 .collect();
6407 self.state.event(
6408 "review",
6409 format!(
6410 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6411 missing.join(", ")
6412 ),
6413 );
6414 }
6415
6416 if clean {
6417 self.state.event(
6418 "review",
6419 if incomplete && policy == IncompleteReviewPolicy::Warn {
6420 format!(
6421 "round {round}: clean (warn policy, incomplete panel) — no \
6422 blocking findings from the seats that answered, verification green"
6423 )
6424 } else if incomplete {
6425 format!(
6426 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6427 quorum) — no blocking findings from the seats that answered, \
6428 verification green",
6429 expected - answered
6430 )
6431 } else {
6432 format!("round {round}: clean — no blocking findings, verification green")
6433 },
6434 );
6435 self.state.reviews.push(round_record);
6436 self.state.status = RunStatus::Gating;
6437 self.state.save()?;
6438 return Ok(());
6439 }
6440
6441 if incomplete && blocking == 0 && e2e_ok {
6449 self.state.reviews.push(round_record);
6450 self.state.save()?;
6451 if round == max_rounds {
6452 self.state.status = RunStatus::Blocked;
6453 self.state.event(
6454 "review",
6455 format!(
6456 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6457 refusing to call it clean",
6458 expected - answered
6459 ),
6460 );
6461 return Ok(());
6462 }
6463 continue;
6464 }
6465
6466 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6478 self.state.reviews.push(round_record);
6479 return self
6480 .stop_reviewing(
6481 "the round's own verification could not run",
6482 &shell,
6483 &winner.worktree,
6484 )
6485 .await;
6486 }
6487
6488 if round == max_rounds {
6489 self.state.reviews.push(round_record);
6490 return self
6491 .stop_reviewing(
6492 &format!(
6493 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6494 ),
6495 &shell,
6496 &winner.worktree,
6497 )
6498 .await;
6499 }
6500
6501 let blocking_findings: Vec<_> = all_findings
6504 .iter()
6505 .filter(|f| f.severity.blocks())
6506 .cloned()
6507 .collect();
6508 let fix_prompt = prompt::fix_in(
6509 &self.state.instruction,
6510 &blocking_findings,
6511 this_round_verification.as_ref(),
6512 round,
6513 max_rounds,
6514 &language,
6515 &self.state.config.graph.github_language,
6516 );
6517 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6518 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6519 let (job, seat, out) = self
6520 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6521 prompt: fix_prompt.clone(),
6522 spec,
6523 seat,
6524 cwd: winner.worktree.clone(),
6525 timeout,
6526 allow_write: true,
6527 sessions,
6528 artifacts: artifacts.clone(),
6529 stem: format!("fix-{round}"),
6530 handover: None,
6531 })
6532 .await;
6533 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6536 return self.stop_for_resource();
6537 }
6538 let agent_id = seat.agent.clone();
6539
6540 let mut fix = FixRecord {
6541 agent: agent_id,
6542 addressed: Vec::new(),
6543 rejected: Vec::new(),
6544 notes: String::new(),
6545 committed: false,
6546 failed: None,
6547 duration_ms: 0,
6548 continuation: None,
6549 };
6550 let mut continuation = ContinuationRecord::not_needed();
6551 let mut final_seat = seat.clone();
6552 match out {
6553 AgentOutcome::Ok(o) => {
6554 fix.duration_ms = o.duration_ms;
6555 let parsed = verdict::extract_json::<FixReport>(&o.text);
6556 let incomplete_reason = match &parsed {
6563 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6564 "the reply parsed, but it reported a command whose own CLI never \
6565 confirmed an exit status"
6566 .to_owned(),
6567 ),
6568 Ok(_) => None,
6569 Err(e) => Some(e.to_string()),
6570 };
6571 match incomplete_reason {
6572 None => {
6573 let report = parsed.expect("checked Ok above");
6574 fix.addressed = report.addressed;
6575 fix.rejected = report.rejected;
6576 fix.notes =
6577 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6578 }
6579 Some(reason) => {
6580 let (resumed_seat, resolved, failure, cont) = self
6581 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6582 .await;
6583 fix.duration_ms += cont.cumulative_wait_ms;
6584 continuation = cont;
6585 final_seat = resumed_seat;
6586 match resolved {
6587 Some(report) => {
6588 fix.addressed = report.addressed;
6589 fix.rejected = report.rejected;
6590 fix.notes = blind::sanitize_prose(
6591 &report.notes,
6592 &self.state.config.blind,
6593 );
6594 }
6595 None => fix.failed = failure,
6596 }
6597 }
6598 }
6599 }
6600 AgentOutcome::Dropped(o) => {
6602 fix.duration_ms = o.duration_ms;
6603 let why = o
6604 .dropped
6605 .as_ref()
6606 .map(|d| d.why.as_str())
6607 .unwrap_or("the CLI ended the stream without delivering its answer");
6608 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6609 }
6610 AgentOutcome::Quota(o) => {
6611 self.state.quota.push(QuotaLoss {
6612 seat: final_seat.key.clone(),
6613 node: "fix".to_owned(),
6614 at: Timestamp::now(),
6615 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6616 });
6617 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6618 }
6619 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6620 }
6621 fix.continuation = Some(continuation);
6622 self.state.seats.insert(final_seat.key.clone(), final_seat);
6623 let coordination = self.coordinate(&winner.worktree).await?;
6624 if let Ok(r) = git::rescue_commit(
6625 &winner.worktree,
6626 &format!("magi: review round {round} fixes (uncommitted work)"),
6627 )
6628 .await
6629 {
6630 self.state.note_withheld("fix", &r.withheld);
6631 }
6632 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6633 fix.committed = after != before;
6634 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6642 drop(coordination);
6643 let progressed = diff_after != patch;
6644 let commit_note = if fix.committed {
6645 "committed"
6646 } else {
6647 "NO new commit"
6648 };
6649 let tree_note = if progressed {
6650 "changed vs base"
6651 } else {
6652 "unchanged vs base"
6653 };
6654 self.state.event(
6655 "fix",
6656 match &fix.failed {
6657 Some(reason) => {
6663 format!(
6664 "round {round}: fixer's adoption report was lost ({reason}); \
6665 {commit_note}, tree {tree_note}"
6666 )
6667 }
6668 None => format!(
6669 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6670 {tree_note}{}",
6671 fix.addressed.len(),
6672 fix.rejected.len(),
6673 if continuation.outcome == ContinuationOutcome::Resumed {
6674 format!(
6675 " (adoption report recovered after {} continuation(s))",
6676 continuation.attempts
6677 )
6678 } else {
6679 String::new()
6680 },
6681 ),
6682 },
6683 );
6684 round_record.fix = Some(fix);
6685 round_record.progressed = progressed;
6686 self.state.reviews.push(round_record);
6687 self.state.save()?;
6688
6689 if matches!(
6702 continuation.outcome,
6703 ContinuationOutcome::Exhausted
6704 | ContinuationOutcome::QuotaLost
6705 | ContinuationOutcome::NoSession
6706 ) {
6707 return self
6708 .stop_reviewing(
6709 "the fixer's adoption report never came back, even after resuming its \
6710 own seat; refusing to start another round against the same worktree \
6711 while that is unresolved",
6712 &shell,
6713 &winner.worktree,
6714 )
6715 .await;
6716 }
6717
6718 let streak = self
6719 .state
6720 .reviews
6721 .iter()
6722 .rev()
6723 .take_while(|r| !r.progressed)
6724 .count();
6725 if streak >= STAGNANT_LIMIT {
6726 return self
6727 .stop_reviewing(
6728 &format!(
6729 "the tree has not moved against base for {streak} round(s) in a row"
6730 ),
6731 &shell,
6732 &winner.worktree,
6733 )
6734 .await;
6735 }
6736 }
6737 Ok(())
6738 }
6739
6740 fn record_contested_handoff(&mut self) {
6774 if self.state.contested_handoff.is_some() {
6775 return;
6776 }
6777 let Some(contested) = self
6778 .state
6779 .reviews
6780 .last()
6781 .and_then(ReviewRound::contested_handoff)
6782 else {
6783 return;
6784 };
6785 self.state.event(
6786 "review",
6787 format!(
6788 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6789 wait for the owner's approval",
6790 contested.findings.len(),
6791 contested.rejecters.len()
6792 ),
6793 );
6794 self.state.contested_handoff = Some(contested);
6795 }
6796
6797 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6798 let round_idx = self.state.reviews.len() - 1;
6799 let needs_catchup_run = matches!(
6807 self.state.reviews[round_idx].e2e_status(),
6808 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6809 );
6810 if needs_catchup_run {
6811 let round = self.state.reviews[round_idx].round;
6812 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6813 let commands = self.state.config.verify.e2e.clone();
6814 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6815 let cache_dir = self.state.config.cache_dir();
6816 let context = format!(
6817 "round {round}: verification unresolved, catching up before the final decision"
6818 );
6819 let wait = self.repository_wait();
6820 let (outcomes, verify_retried) = with_cache_lease(
6821 &mut self.state,
6822 cache_dir.as_deref(),
6823 "e2e",
6824 "e2e",
6825 worktree,
6826 &attempted_head,
6827 timeout,
6828 &context,
6829 &wait,
6830 |state, budget| {
6831 let shell = shell.to_vec();
6832 let commands = commands.clone();
6833 let context = context.clone();
6834 async move {
6835 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6836 .await
6837 }
6838 },
6839 )
6840 .await?;
6841 let last = &mut self.state.reviews[round_idx];
6842 last.e2e = outcomes;
6843 last.verify_retried = verify_retried;
6844 last.verified_head = Some(attempted_head);
6851 last.verified_at = Some(Timestamp::now());
6852 if verify_inconclusive(&last.e2e) {
6853 self.state.save()?;
6860 return Ok(());
6861 }
6862 last.e2e_deferred = false;
6863 }
6864 let last = &self.state.reviews[round_idx];
6865 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6866
6867 match last.e2e_status() {
6868 E2eStatus::Failed => {
6869 let red: Vec<String> = last
6870 .e2e
6871 .iter()
6872 .filter(|o| !o.ok())
6873 .map(|o| {
6874 format!(
6875 "`{}` -> {:?}\n{}",
6876 o.command,
6877 o.code,
6878 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6879 )
6880 })
6881 .collect();
6882 self.state
6883 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6884 self.state.status = RunStatus::Blocked;
6885 }
6886 E2eStatus::ResourceBlocked => {
6891 self.state.event(
6892 "review",
6893 format!(
6894 "{why}; e2e could not run (shared build cache unavailable); not \
6895 deciding yet"
6896 ),
6897 );
6898 }
6899 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6900 self.state.event(
6901 "review",
6902 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6903 );
6904 self.record_contested_handoff();
6905 self.state.status = RunStatus::Gating;
6906 }
6907 }
6908 self.state.save()?;
6909 Ok(())
6910 }
6911
6912 async fn gate(&mut self) -> Result<()> {
6915 if self.state.status == RunStatus::Failed
6927 || self
6928 .state
6929 .base_sync
6930 .as_ref()
6931 .is_some_and(|s| s.conflict.is_some())
6932 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6933 != Some(RunStatus::Gating)
6934 {
6935 return Ok(());
6936 }
6937 if self.state.gate_ran {
6938 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6949 self.state.status = RunStatus::Blocked;
6950 self.state.save()?;
6951 }
6952 return Ok(());
6953 }
6954 let Some(winner) = self.state.winner().cloned() else {
6955 return Ok(());
6956 };
6957 self.state.status = RunStatus::Gating;
6958 let mut outcomes = self.run_gate(&winner).await?;
6959 loop {
6960 if verify_inconclusive(&outcomes) {
6971 self.state.save()?;
6972 return Ok(());
6973 }
6974 if outcomes.iter().all(CommandOutcome::ok) {
6975 break;
6976 }
6977 match self.gate_fix_round(&winner, &outcomes).await? {
6978 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6979 GateFix::Stop => break,
6980 GateFix::Defer => {
6981 self.state.save()?;
6982 return Ok(());
6983 }
6984 }
6985 }
6986 let passed = outcomes.iter().all(CommandOutcome::ok);
6987 self.state.gate = outcomes;
6988 self.state.gate_ran = true;
6989 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6990 if !passed {
6991 self.state.status = RunStatus::Blocked;
6992 let spent = self.state.gate_fixes.len();
6993 self.state.event(
6994 "gate",
6995 if spent == 0 {
6996 "gate failed; not merging".to_owned()
6997 } else {
6998 format!("gate failed after {spent} gate-fix round(s); not merging")
6999 },
7000 );
7001 }
7002 self.state.save()?;
7003 Ok(())
7004 }
7005
7006 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7016 let commands = state.config.verify.pre_gate.clone();
7017 if commands.is_empty() {
7018 return;
7019 }
7020 let shell = state.config.shell();
7021 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7022 let (outcomes, _) = run_commands(
7023 state,
7024 "pre_gate",
7025 "pre_gate",
7026 0,
7027 &shell,
7028 &commands,
7029 &winner.worktree,
7030 timeout,
7031 )
7032 .await;
7033 for o in &outcomes {
7034 if !o.ok() {
7035 tracing::warn!(
7036 "pre_gate `{}` failed ({:?}); the gate decides",
7037 o.command,
7038 o.code
7039 );
7040 }
7041 state.event(
7042 "pre_gate",
7043 format!(
7044 "`{}` -> {}",
7045 o.command,
7046 if o.ok() {
7047 "pass".to_owned()
7048 } else {
7049 format!(
7050 "FAIL ({:?})\n{}",
7051 o.code,
7052 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7053 )
7054 }
7055 ),
7056 );
7057 }
7058 state.pre_gate = outcomes;
7059 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7060 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7061 Ok(head) => {
7062 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7063 state.pre_gate_commit = Some(head);
7064 }
7065 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7066 },
7067 Ok(false) => {}
7068 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7069 }
7070 if let Err(e) = state.save() {
7071 tracing::warn!("could not persist the pre_gate record: {e:#}");
7072 }
7073 }
7074
7075 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7078 let wait = self.repository_wait();
7079 run_landing_gate(&mut self.state, winner, &wait).await
7080 }
7081
7082 async fn gate_fix_round(
7095 &mut self,
7096 winner: &Candidate,
7097 outcomes: &[CommandOutcome],
7098 ) -> Result<GateFix> {
7099 let cap = self.state.config.graph.gate_fix_rounds;
7100 let spent = self.state.gate_fixes.len();
7101 if spent >= cap {
7102 if cap > 0 {
7103 self.state.event(
7104 "gate",
7105 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7106 );
7107 }
7108 return Ok(GateFix::Stop);
7109 }
7110 if !gate_fixable(outcomes) {
7111 self.state.event(
7112 "gate",
7113 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7114 command or similar); not spending a fix round on it",
7115 );
7116 return Ok(GateFix::Stop);
7117 }
7118 let min_free = self.state.config.disk.min_free_bytes;
7119 if min_free > 0 {
7120 match crate::disk::free_bytes(&winner.worktree) {
7121 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7122 Ok(free) => {
7123 self.state.event(
7124 "gate",
7125 format!(
7126 "only {free} bytes free ({min_free} required by `[disk] \
7127 min_free_bytes`); not spending a fix round on a failure the disk \
7128 may explain"
7129 ),
7130 );
7131 return Ok(GateFix::Stop);
7132 }
7133 Err(e) => {
7134 self.state.event(
7135 "gate",
7136 format!("free disk space could not be measured ({e:#}); no fix round"),
7137 );
7138 return Ok(GateFix::Stop);
7139 }
7140 }
7141 }
7142
7143 let attempt = spent + 1;
7144 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7145 let base = self.landing_base();
7146 let fix_prompt = prompt::gate_fix_in(
7147 &self.state.instruction,
7148 &failed,
7149 attempt,
7150 cap,
7151 &self.state.config.graph.language,
7152 &self.state.config.graph.github_language,
7153 );
7154 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7155 let sessions = self.state.config.graph.sessions;
7156 let artifacts = self.merge_repair_artifacts();
7157 self.state.event(
7158 "gate",
7159 format!("gate failed; gate-fix round {attempt} of {cap}"),
7160 );
7161 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7162 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7163 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7166 head: before.clone(),
7167 passed: false,
7168 outcomes: Vec::new(),
7169 });
7170 self.state.save()?;
7171 let (_, seat, out) = self
7172 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7173 prompt: fix_prompt.clone(),
7174 spec,
7175 seat,
7176 cwd: winner.worktree.clone(),
7177 timeout,
7178 allow_write: true,
7179 sessions,
7180 artifacts: artifacts.clone(),
7181 stem: format!("gate-fix-{attempt}"),
7182 handover: None,
7183 })
7184 .await;
7185 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7188 return Ok(GateFix::Defer);
7189 }
7190 let mut record = GateFixRecord {
7191 agent: seat.agent.clone(),
7192 failed,
7193 notes: String::new(),
7194 committed: false,
7195 error: None,
7196 };
7197 match out {
7198 AgentOutcome::Ok(o) => {
7199 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7202 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7203 }
7204 }
7205 AgentOutcome::Dropped(_) => {
7206 record.error = Some("the CLI dropped the stream".to_owned());
7207 }
7208 AgentOutcome::Quota(o) => {
7209 self.state.quota.push(QuotaLoss {
7210 seat: seat.key.clone(),
7211 node: "gate-fix".to_owned(),
7212 at: Timestamp::now(),
7213 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7214 });
7215 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7216 }
7217 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7218 }
7219 self.state.seats.insert(seat.key.clone(), seat);
7220 let coordination = self.coordinate(&winner.worktree).await?;
7221 if let Ok(r) = git::rescue_commit(
7222 &winner.worktree,
7223 &format!("magi: gate fix {attempt} (uncommitted work)"),
7224 )
7225 .await
7226 {
7227 self.state.note_withheld("gate-fix", &r.withheld);
7228 }
7229 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7230 record.committed = after != before;
7231 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7232 let note = record.error.clone();
7233 self.state.gate_fixes.push(record);
7234 self.state.save()?;
7235 if !changed {
7236 self.state.event(
7237 "gate",
7238 match note {
7239 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7240 None => format!("gate-fix round {attempt}: the tree did not change"),
7241 },
7242 );
7243 return Ok(GateFix::Stop);
7244 }
7245 self.state.event(
7246 "gate",
7247 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7248 );
7249
7250 drop(coordination);
7251 let wait = self.repository_wait();
7252 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7253 }
7254
7255 async fn guarded_pr_message(
7262 &mut self,
7263 winner: &Candidate,
7264 facts: Option<&BranchFacts>,
7265 posting: bool,
7266 ) -> Result<Option<PrMessage>> {
7267 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7268 let head = match final_head.as_deref() {
7269 Some(h) => crate::pr_outcome::FinalHead::At(h),
7270 None => crate::pr_outcome::FinalHead::Unknown,
7271 };
7272 let summarized = if posting {
7274 Some(self.summarize_change(winner, final_head.as_deref()).await)
7275 } else {
7276 None
7277 };
7278 if self.state.resource_block.is_some() {
7283 self.state.event(
7284 "pr-summary",
7285 "a shared resource was busy; nothing was published, resume to summarize again",
7286 );
7287 return Ok(None);
7288 }
7289 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7290 anyhow::bail!(
7291 "the final commit moved during public description generation; retry before publishing"
7292 );
7293 }
7294 let note = match (&summarized, &self.state.change_summary) {
7295 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7296 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7297 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7298 (None, _) => ChangeNote::NotAsked,
7299 };
7300 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7301 if !posting {
7302 let identity = crate::scrub::Identity::current();
7303 return Ok(Some(PrMessage {
7304 title: crate::scrub::scrub(&pr.title, &identity),
7305 body: crate::scrub::scrub(&pr.body, &identity),
7306 }));
7307 }
7308 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7311 if let Some(known) = self
7312 .state
7313 .github_text
7314 .clone()
7315 .filter(|g| g.fingerprint == raw_fp)
7316 {
7317 return self.settle_withheld(winner, &pr, known).await;
7318 }
7319 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7320 let mut violations =
7321 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7322 let identity = crate::scrub::Identity::current();
7323 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7324 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7325 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7326 {
7327 violations.push(crate::github_text::Violation::SensitiveData);
7328 }
7329 let has_language = violations
7333 .iter()
7334 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7335 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7336 self.state.event(
7337 "github-text",
7338 if has_language {
7339 format!("description rejected: {violations:?}; requesting one rewrite")
7340 } else {
7341 format!("description rejected: {violations:?}; asking the owner")
7342 },
7343 );
7344 let mut rewritten = false;
7345 let mut rejected_rewrite = None;
7346 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7347 {
7348 let output = text;
7349 if let Some(title) = summary_title(&output) {
7350 let raw_body = summary_without_title(&output);
7353 let clean_title = crate::scrub::scrub(&title, &identity);
7354 let mut body = crate::scrub::scrub(&raw_body, &identity);
7355 let redacted = clean_title != title || body != raw_body;
7356 let title = clean_title;
7357 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7358 if !complete {
7359 self.state.event(
7360 "github-text",
7361 "description rewrite omitted required findings",
7362 );
7363 }
7364 if !body.trim().is_empty() {
7365 body.push_str("\n\n");
7366 body.push_str(&public_pr_evidence(&self.state, head));
7367 }
7368 let marker = format!("magi:run/{}", self.state.id);
7369 if !body.trim().is_empty() && !body.contains(&marker) {
7370 body.push_str(&format!("\n\n{marker}"));
7371 }
7372 let rewrite = PrMessage {
7373 title: title.clone(),
7374 body: body.clone(),
7375 };
7376 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7377 let mut failed = crate::github_text::check_for(
7378 &self.state.config.graph,
7379 &title,
7380 &body,
7381 redecision,
7382 );
7383 if (!crate::github_text::shareable(&title)
7384 || !crate::github_text::shareable(&body))
7385 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7386 {
7387 failed.push(crate::github_text::Violation::SensitiveData);
7388 }
7389 if complete
7390 && !title.trim().is_empty()
7391 && !body.trim().is_empty()
7392 && !failed.is_empty()
7393 {
7394 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7395 title: title.clone(),
7396 body: body.clone(),
7397 failed: failed
7398 .iter()
7399 .map(|v| crate::github_text::category(*v).to_owned())
7400 .collect(),
7401 });
7402 }
7403 if complete
7404 && !title.trim().is_empty()
7405 && !raw_body.trim().is_empty()
7406 && failed.is_empty()
7407 {
7408 decision = redecision;
7409 pr = PrMessage { title, body };
7410 rewritten = true;
7411 self.state
7412 .event("github-text", "description rewrite passed");
7413 if redacted {
7414 self.state.event(
7415 "github-text",
7416 "sensitive data removed from description rewrite",
7417 );
7418 }
7419 }
7420 }
7421 }
7422 if !rewritten {
7423 self.state.event(
7424 "github-text",
7425 "description rewrite unavailable or rejected; asking the owner",
7426 );
7427 if let Some(w) =
7428 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7429 {
7430 self.retire_github_text_question("the withheld text changed");
7433 let record = crate::run::GithubTextAsk {
7434 rewrite: rejected_rewrite,
7435 fingerprint: raw_fp,
7436 title: w.title,
7437 body: w.body,
7438 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7439 question: None,
7440 asks: 0,
7441 resolved: false,
7442 chosen_title: None,
7443 choice: None,
7444 };
7445 return self.settle_withheld(winner, &pr, record).await;
7446 }
7447 }
7448 }
7449 let (title, body) =
7450 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7451 Ok(Some(PrMessage { title, body }))
7452 }
7453
7454 async fn summarize_change(
7465 &mut self,
7466 winner: &Candidate,
7467 head: Option<&str>,
7468 ) -> std::result::Result<(), String> {
7469 let Some(head) = head else {
7470 return Err("the final commit could not be read".to_owned());
7471 };
7472 let remote = self.state.config.merge.remote.clone();
7473 let tracking = format!("{remote}/{}", self.state.base_branch);
7474 if !git::rev_exists(&winner.worktree, &tracking).await {
7475 return Err(format!("the remote base `{tracking}` could not be read"));
7476 }
7477 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7478 Ok(b) if !b.is_empty() => b,
7479 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7480 };
7481 let evidence = crate::pr_outcome::summary_evidence(
7482 &self.state,
7483 crate::pr_outcome::FinalHead::At(head),
7484 );
7485 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7486 if self
7487 .state
7488 .change_summary
7489 .as_ref()
7490 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7491 {
7492 return Ok(());
7493 }
7494 if self.state.change_summary.take().is_some() {
7495 self.state.event(
7496 "pr-summary",
7497 "the final diff or evidence changed; the earlier change summary was discarded",
7498 );
7499 }
7500 if let Err(e) = write_artifact(
7502 &self.state,
7503 &format!("public-description-{fingerprint}.md"),
7504 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7505 ) {
7506 tracing::warn!("could not save public description evidence: {e:#}");
7507 }
7508 let chain = match agent::pick_chain(
7509 &self.state.config.agents,
7510 self.state.config.roles.synthesizer.as_ref(),
7511 &agent::installed,
7512 "synthesizer",
7513 ) {
7514 Ok(chain) => chain,
7515 Err(e) => {
7516 tracing::warn!("no agent for the change summary: {e:#}");
7517 self.state.event("pr-summary", "no summary agent available");
7518 return Err("no agent was available to summarize it".to_owned());
7519 }
7520 };
7521 let subjects = git::subjects(&winner.worktree, &base, head)
7522 .await
7523 .unwrap_or_default();
7524 let stat = git::diff_stat(&winner.worktree, &base, head)
7525 .await
7526 .unwrap_or_default();
7527 let stat: String = stat.chars().take(6_000).collect();
7528 let mut prompt = crate::prompt::change_summary_in(
7529 &base,
7530 head,
7531 &subjects,
7532 stat.trim(),
7533 &self.state.config.graph.github_language,
7534 );
7535 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7536 let sessions = self.state.config.graph.sessions;
7537 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7538 let prompts = self.state.config.prompts.clone();
7539 let cache = self.state.config.cache_dir();
7540 let run = self.state.id.clone();
7541 for (i, spec) in chain.into_iter().enumerate() {
7542 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7543 let job = SeatJob {
7544 spec: spec.clone(),
7545 seat,
7546 cwd: winner.worktree.clone(),
7547 prompt: prompt.clone(),
7548 timeout,
7549 allow_write: false,
7550 sessions,
7551 artifacts: agent::artifacts_dir(&self.state.dir()),
7552 stem: if i == 0 {
7553 "change-summary".to_owned()
7554 } else {
7555 format!("change-summary-{}", spec.id)
7556 },
7557 handover: None,
7558 };
7559 let ctx = WaveCtx {
7560 run: &run,
7561 node: "pr-summary",
7562 prompts: &prompts,
7563 cache: cache.as_deref(),
7564 round: None,
7565 carry_seats: false,
7566 pauses: Some((&self.pause, &self.interrupt)),
7567 };
7568 let (seat, outcome) =
7569 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7570 self.state.seats.insert(seat.key.clone(), seat);
7571 let text = match outcome {
7572 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7573 AgentOutcome::ResourceBlocked(why) => {
7574 if self.state.resource_block.is_none() {
7577 self.state.resource_block = Some(crate::run::ResourceBlock {
7578 node: "pr-summary".to_owned(),
7579 kind: "shared resource".to_owned(),
7580 detail: why,
7581 at: Timestamp::now(),
7582 });
7583 }
7584 return Err("a shared resource was busy".to_owned());
7585 }
7586 _ => continue,
7587 };
7588 let blind = &self.state.config.blind;
7589 let Some(title) = summary_title(&text) else {
7590 continue;
7591 };
7592 let body: String = summary_without_title(&text)
7593 .lines()
7594 .filter(|l| !l.contains("magi:run/"))
7595 .collect::<Vec<_>>()
7596 .join("\n");
7597 let body = blind::sanitize_prose(body.trim(), blind);
7598 let title = blind::sanitize_prose(&title, blind);
7599 if body.trim().is_empty() || title.trim().is_empty() {
7600 continue;
7601 }
7602 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7603 continue;
7604 }
7605 if git::rev_parse(&winner.worktree, "HEAD")
7607 .await
7608 .ok()
7609 .as_deref()
7610 != Some(head)
7611 {
7612 return Err("the branch moved while the summary was written".to_owned());
7613 }
7614 if git::merge_base(&winner.worktree, &tracking, head)
7615 .await
7616 .ok()
7617 .as_deref()
7618 != Some(base.as_str())
7619 || crate::pr_outcome::summary_evidence(
7620 &self.state,
7621 crate::pr_outcome::FinalHead::At(head),
7622 ) != evidence
7623 {
7624 return Err("the summary evidence changed while it was written".to_owned());
7625 }
7626 self.state.change_summary = Some(crate::run::ChangeSummary {
7627 head: head.to_owned(),
7628 base,
7629 evidence: fingerprint,
7630 title,
7631 body,
7632 });
7633 self.state.event(
7634 "pr-summary",
7635 format!("change summary written by {} for the final diff", spec.id),
7636 );
7637 if let Err(e) = self.state.save() {
7638 tracing::warn!("could not save the change summary: {e:#}");
7639 }
7640 return Ok(());
7641 }
7642 self.state
7643 .event("pr-summary", "no agent produced a change summary");
7644 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7645 }
7646
7647 async fn rewrite_pr_text(
7655 &mut self,
7656 winner: &Candidate,
7657 pr: &PrMessage,
7658 violations: &[crate::github_text::Violation],
7659 ) -> Option<String> {
7660 let prompt = format!(
7661 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7662 pr.title, pr.body
7663 );
7664 let sessions = self.state.config.graph.sessions;
7665 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7666 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7668 let key = format!("impl-{}", winner.label);
7669 if let Some(spec) = self
7670 .state
7671 .config
7672 .agents
7673 .iter()
7674 .find(|a| a.id == winner.agent)
7675 .cloned()
7676 && let Some(existing) = self.state.seats.get(&key)
7677 && existing.agent == spec.id
7678 && agent::has_session(spec.kind, existing, sessions)
7679 {
7680 attempts.push((spec, existing.clone(), true));
7681 } else {
7682 let chain = agent::pick_chain(
7683 &self.state.config.agents,
7684 self.state.config.roles.language_judge.as_ref(),
7685 &agent::installed,
7686 "language_judge",
7687 );
7688 match chain {
7689 Ok(chain) => {
7690 for spec in chain {
7691 let seat = crate::fixer::seat_for(
7692 &mut self.state,
7693 "github-text-rewrite",
7694 &spec.id,
7695 );
7696 attempts.push((spec, seat, false));
7697 }
7698 }
7699 Err(e) => {
7700 tracing::warn!("no agent for the description rewrite: {e:#}");
7701 self.state
7702 .event("github-text", "no rewrite agent available");
7703 return None;
7704 }
7705 }
7706 }
7707 let prompts = self.state.config.prompts.clone();
7708 let cache = self.state.config.cache_dir();
7709 let run = self.state.id.clone();
7710 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7711 let job = SeatJob {
7712 spec: spec.clone(),
7713 seat,
7714 cwd: winner.worktree.clone(),
7715 prompt: prompt.clone(),
7716 timeout: retry_budget(timeout_full, resumed),
7717 allow_write: false,
7718 sessions,
7719 artifacts: agent::artifacts_dir(&self.state.dir()),
7720 stem: if i == 0 {
7721 "github-text-rewrite".to_owned()
7722 } else {
7723 format!("github-text-rewrite-{}", spec.id)
7724 },
7725 handover: None,
7726 };
7727 let ctx = WaveCtx {
7728 run: &run,
7729 node: "github-text",
7730 prompts: &prompts,
7731 cache: cache.as_deref(),
7732 round: None,
7733 carry_seats: false,
7734 pauses: Some((&self.pause, &self.interrupt)),
7735 };
7736 let (seat, outcome) =
7737 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7738 self.state.seats.insert(seat.key.clone(), seat);
7739 match outcome {
7740 AgentOutcome::Ok(o)
7741 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7742 {
7743 return Some(o.text);
7744 }
7745 AgentOutcome::ResourceBlocked(_) => return None,
7746 _ => {}
7747 }
7748 }
7749 None
7750 }
7751
7752 fn retire_github_text_question(&mut self, why: &str) {
7754 let Some(id) = self
7755 .state
7756 .github_text
7757 .as_ref()
7758 .and_then(|g| g.question.clone())
7759 else {
7760 return;
7761 };
7762 let _ = ask::Questions::open().update(&id, |q| {
7763 q.abandon(why);
7764 Ok(())
7765 });
7766 }
7767
7768 async fn settle_withheld(
7772 &mut self,
7773 winner: &Candidate,
7774 pr: &PrMessage,
7775 mut g: crate::run::GithubTextAsk,
7776 ) -> Result<Option<PrMessage>> {
7777 use crate::github_text as gt;
7778 let store = ask::Questions::open();
7779 let timeout = self.state.config.graph.answer_timeout;
7780 if !g.resolved {
7781 let mut standing = g
7782 .question
7783 .as_ref()
7784 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7785 if let Some(q) = standing.as_ref()
7786 && q.status.open()
7787 && gt::expired(q, timeout)
7788 {
7789 let why = format!("no answer within {}s of asking", timeout.max(1));
7790 standing = store
7791 .update(&q.id, |q| {
7792 q.abandon(&why);
7793 Ok(())
7794 })
7795 .ok()
7796 .map(|(q, ())| q);
7797 }
7798 let mut ask_again: Option<Option<Vec<String>>> = None;
7799 match standing {
7800 None if g.asks <= 1 => {
7803 g.asks = 0;
7804 ask_again = Some(None);
7805 }
7806 None => {
7807 g.resolved = true;
7808 g.chosen_title = None;
7809 g.choice = Some(GithubTextChoice::Fallback);
7810 }
7811 Some(q) if q.status.open() => {
7812 self.park_on_github_text(&g)?;
7813 return Ok(None);
7814 }
7815 Some(q) => match gt::read_reply(&q) {
7816 gt::Reply::Fallback => {
7817 g.resolved = true;
7818 g.chosen_title = None;
7819 g.choice = Some(GithubTextChoice::Fallback);
7820 }
7821 gt::Reply::Original => {
7822 g.resolved = true;
7823 g.chosen_title = None;
7824 g.choice = Some(GithubTextChoice::Original);
7825 }
7826 gt::Reply::Rewrite => {
7827 g.resolved = true;
7828 g.chosen_title = None;
7829 g.choice = Some(if g.rewrite.is_some() {
7830 GithubTextChoice::Rewrite
7831 } else {
7832 GithubTextChoice::Fallback
7833 });
7834 }
7835 gt::Reply::Title(text) => {
7836 let first = text
7837 .lines()
7838 .map(str::trim)
7839 .find(|l| !l.is_empty())
7840 .unwrap_or("");
7841 let decision = if first.is_empty() {
7842 None
7843 } else {
7844 let candidate = PrMessage {
7845 title: first.to_owned(),
7846 body: String::new(),
7847 };
7848 self.judge_pr_language(&winner.worktree, &candidate).await
7849 };
7850 match gt::vet_title(
7851 &text,
7852 decision,
7853 self.state.config.graph.github_english(),
7854 ) {
7855 Ok(title) => {
7856 g.resolved = true;
7857 g.chosen_title = Some(title);
7858 g.choice = Some(GithubTextChoice::MyTitle);
7859 }
7860 Err(cats) if g.asks < 2 => {
7861 ask_again = Some(Some(
7862 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7863 ))
7864 }
7865 Err(_) => {
7866 g.resolved = true;
7867 g.chosen_title = None;
7868 g.choice = Some(GithubTextChoice::Fallback);
7869 }
7870 }
7871 }
7872 },
7873 }
7874 if let Some(rejected) = ask_again {
7875 let w = gt::Withheld {
7878 title: g.title,
7879 body: g.body,
7880 categories: g.categories.clone(),
7881 };
7882 let artifact = match crate::run::write_artifact(
7885 &self.state,
7886 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7887 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7888 ) {
7889 Ok(p) => Some(p),
7890 Err(e) => {
7891 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7892 None
7893 }
7894 };
7895 let (detail, comparison) = gt::question_content(
7896 &w,
7897 &pr.title,
7898 &pr.body,
7899 &self.state.id,
7900 rejected.as_deref(),
7901 artifact.as_deref(),
7902 g.rewrite.as_ref(),
7903 );
7904 let mut q = ask::Question::new(
7905 self.state.id.clone(),
7906 gt::ASK_NODE.to_owned(),
7907 gt::ASK_SEAT.to_owned(),
7908 gt::question_summary(&self.state.config.graph.language, &w),
7909 detail,
7910 gt::question_choices(&w, g.rewrite.as_ref()),
7911 );
7912 q.posting_comparison = Some(comparison);
7913 q.answer_timeout = timeout;
7914 g.question = Some(q.id.clone());
7917 g.asks += 1;
7918 self.state.github_text = Some(g.clone());
7919 self.state.event(
7920 "github-text",
7921 format!("asking the owner about the withheld text ({})", q.short()),
7922 );
7923 self.state.save()?;
7924 store
7925 .put_executable(&mut q)
7926 .context("file the posting-gate question")?;
7927 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7928 tracing::warn!(
7929 "could not notify about posting-gate question {}: {e:#}",
7930 q.short()
7931 );
7932 }
7933 self.park_on_github_text(&g)?;
7934 return Ok(None);
7935 }
7936 }
7937 let resolved = g.chosen_title.clone();
7938 let original = g.choice == Some(GithubTextChoice::Original);
7939 self.state.event(
7941 "github-text",
7942 if original {
7943 format!(
7944 "posting the original text for: {}",
7945 match (g.title, g.body) {
7946 (true, true) => "title, description",
7947 (true, false) => "title",
7948 _ => "description",
7949 }
7950 )
7951 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7952 "posting the saved rewrite title and description".to_owned()
7953 } else if resolved.is_some() {
7954 "posting the owner's replacement title".to_owned()
7955 } else {
7956 "posting the neutral text for the withheld fields".to_owned()
7957 },
7958 );
7959 self.state.github_text = Some(g.clone());
7960 self.state.save()?;
7961 if g.choice == Some(GithubTextChoice::Rewrite)
7962 && let Some(r) = g.rewrite.as_ref()
7963 {
7964 return Ok(Some(PrMessage {
7965 title: r.title.clone(),
7966 body: r.body.clone(),
7967 }));
7968 }
7969 let w = gt::Withheld {
7970 title: g.title,
7971 body: g.body,
7972 categories: g.categories.clone(),
7973 };
7974 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7975 if let Some(t) = resolved {
7976 title = t;
7977 }
7978 let english = gt::LanguageDecision {
7980 title_english: true,
7981 body_english: true,
7982 body_complete: true,
7983 };
7984 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7985 if original {
7986 if g.title {
7989 title = pr.title.clone();
7990 }
7991 if g.body {
7992 body = pr.body.clone();
7993 }
7994 }
7995 Ok(Some(PrMessage { title, body }))
7996 }
7997
7998 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7999 self.state.github_text = Some(g.clone());
8000 self.state.end_pass_parked(ParkCause::Question);
8001 self.state.parked = true;
8002 self.state.event(
8003 "github-text",
8004 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8005 );
8006 self.state.save()
8007 }
8008
8009 async fn judge_pr_language(
8012 &mut self,
8013 cwd: &Path,
8014 pr: &PrMessage,
8015 ) -> Option<crate::github_text::LanguageDecision> {
8016 self.state.config.roles.language_judge.as_ref()?;
8017 if !self.state.config.graph.language_gate() {
8018 return None;
8019 }
8020 let decision =
8021 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8022 self.state.event(
8023 "github-text",
8024 if decision.is_some() {
8025 "language decided by the language judge"
8026 } else {
8027 "language judge unavailable; using built-in heuristics"
8028 },
8029 );
8030 decision
8031 }
8032
8033 async fn merge(&mut self) -> Result<()> {
8034 if self
8049 .state
8050 .base_sync
8051 .as_ref()
8052 .is_some_and(|s| s.conflict.is_some())
8053 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8054 != Some(RunStatus::Gating)
8055 || !self.state.gate_allows_progress()
8063 {
8064 return Ok(());
8065 }
8066 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8075 return Ok(());
8076 }
8077 let Some(winner) = self.state.winner().cloned() else {
8078 return Ok(());
8079 };
8080 let repo = self.state.repo.clone();
8081 let base = self.state.base_branch.clone();
8082 let mode = self.state.config.merge.mode;
8083 let style = self.state.config.merge.style;
8084 let facts = if is_review_run(&self.state) {
8085 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8086 let start = review_base(
8087 &repo,
8088 &self.state.config.merge.remote,
8089 &base,
8090 &self.state.base_commit,
8091 &winner.branch,
8092 )
8093 .await;
8094 branch_facts(&repo, &start, &winner.branch).await
8095 } else {
8096 None
8097 };
8098 let leaked = if is_review_run(&self.state) {
8101 leaked_subjects(&self.state, &winner.branch).await
8102 } else {
8103 Some(Vec::new())
8104 };
8105 let Some(pr) = self
8109 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8110 .await?
8111 else {
8112 return Ok(());
8113 };
8114 let message = pr.commit_message();
8115
8116 let outcome = match mode {
8117 MergeMode::None => MergeOutcome {
8118 mode,
8119 ok: true,
8120 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8121 empty: false,
8122 },
8123 MergeMode::Pr | MergeMode::Local
8124 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8125 {
8126 MergeOutcome {
8127 mode,
8128 ok: false,
8129 detail: empty_candidate_detail(&self.state, &base),
8130 empty: true,
8131 }
8132 }
8133 MergeMode::Local => {
8134 let on = git::current_branch(&repo).await?;
8135 if on.as_deref() != Some(base.as_str()) {
8136 MergeOutcome {
8137 mode,
8138 ok: false,
8139 detail: format!(
8140 "{} has {} checked out, not the base branch {base}",
8141 repo.display(),
8142 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8143 ),
8144 empty: false,
8145 }
8146 } else if !git::is_clean(&repo).await? {
8147 MergeOutcome {
8148 mode,
8149 ok: false,
8150 detail: format!("{} is dirty; refusing to merge", repo.display()),
8151 empty: false,
8152 }
8153 } else {
8154 let out = match style {
8155 MergeStyle::Merge => {
8156 git::merge_no_ff(&repo, &winner.branch, &message).await?
8157 }
8158 MergeStyle::Squash => {
8159 git::merge_squash(&repo, &winner.branch, &message).await?
8160 }
8161 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8162 };
8163 MergeOutcome {
8164 mode,
8165 ok: out.ok(),
8166 detail: if out.ok() { out.stdout } else { out.stderr },
8167 empty: false,
8168 }
8169 }
8170 }
8171 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8172 self.state.pr_publication_pending = true;
8173 self.state.save()?;
8174 let out =
8175 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8176 self.state.pr_publication_pending = out.is_err();
8177 MergeOutcome {
8178 mode,
8179 ok: out.is_ok(),
8180 detail: match out {
8181 Ok(url) => url,
8182 Err(e) => format!("{e:#}"),
8183 },
8184 empty: false,
8185 }
8186 }
8187 MergeMode::Pr => {
8188 let remote = self.state.config.merge.remote.clone();
8189 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8190 if !pushed.ok() {
8191 MergeOutcome {
8192 mode,
8193 ok: false,
8194 detail: pushed.stderr,
8195 empty: false,
8196 }
8197 } else {
8198 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8204 let out = match pr_merge_plan(found) {
8205 PrPlan::Create => {
8206 gh_pr_create(
8207 &winner.worktree,
8208 &base,
8209 &winner.branch,
8210 &pr.title,
8211 &pr.body,
8212 )
8213 .await
8214 }
8215 PrPlan::Adopt { url, title } => {
8216 self.state
8217 .event("merge", format!("Pr: adopted open pull request {url}"));
8218 if self.state.github_text.as_ref().is_some_and(|g| {
8219 g.resolved
8220 && g.choice == Some(GithubTextChoice::Rewrite)
8221 && g.rewrite.is_some()
8222 }) {
8223 land::set_pr_text_verbatim(
8226 &winner.worktree,
8227 &url,
8228 &pr.title,
8229 &pr.body,
8230 )
8231 .await
8232 .map(|()| url)
8233 } else {
8234 if title != pr.title
8235 && (!is_review_run(&self.state)
8236 || leaked
8237 .as_deref()
8238 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8239 && let Err(e) =
8240 if self.state.github_text.as_ref().is_some_and(|g| {
8241 g.resolved
8242 && g.title
8243 && g.choice == Some(GithubTextChoice::Original)
8244 }) {
8245 land::set_pr_title_verbatim(
8247 &winner.worktree,
8248 &url,
8249 &pr.title,
8250 )
8251 .await
8252 } else {
8253 land::set_pr_title(
8254 &mut self.state,
8255 &winner.worktree,
8256 &url,
8257 &pr.title,
8258 )
8259 .await
8260 }
8261 {
8262 tracing::warn!("could not refresh title of {url}: {e:#}");
8263 self.state
8264 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8265 }
8266 Ok(url)
8267 }
8268 }
8269 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8270 };
8271 match out {
8272 Ok(url) => MergeOutcome {
8273 mode,
8274 ok: true,
8275 detail: url,
8276 empty: false,
8277 },
8278 Err(e) => MergeOutcome {
8279 mode,
8280 ok: false,
8281 detail: e.to_string(),
8282 empty: false,
8283 },
8284 }
8285 }
8286 }
8287 };
8288
8289 self.state.status = match (mode, outcome.ok) {
8290 (MergeMode::None, _) => RunStatus::Ready,
8291 (_, true) if outcome.empty => RunStatus::Ready,
8292 (MergeMode::Pr, true) => RunStatus::Ready,
8293 (_, true) => RunStatus::Merged,
8294 (_, false) => RunStatus::Blocked,
8295 };
8296 self.state.event(
8297 "merge",
8298 format!(
8299 "{:?}: {}",
8300 mode,
8301 outcome.detail.lines().next().unwrap_or("")
8302 ),
8303 );
8304 self.state.merge = Some(outcome);
8305 self.state.save()?;
8306
8307 if self.state.config.graph.land
8313 && mode == MergeMode::Pr
8314 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8315 {
8316 self.run_land().await?;
8317 }
8318 self.settle_questions();
8323 Ok(())
8324 }
8325
8326 async fn run_land(&mut self) -> Result<()> {
8337 crate::pr_head::ensure_owner(&self.state)?;
8338 let url = self
8339 .state
8340 .pr
8341 .as_ref()
8342 .map(|pr| pr.url.clone())
8343 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8344 .unwrap_or_default();
8345 let url = url.lines().next().unwrap_or("").trim().to_owned();
8346 if !url.starts_with("http") {
8347 return Ok(());
8348 }
8349 loop {
8352 match land::land(&mut self.state, &url).await {
8353 Ok(_)
8354 if self
8355 .state
8356 .merge_repairs
8357 .last()
8358 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8359 {
8360 if self.repair_merge_review().await? {
8361 continue;
8362 }
8363 }
8364 Ok(pr) if self.state.parked => {
8365 let _ = pr;
8369 }
8370 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8371 self.state.status = RunStatus::Superseded;
8372 self.state.save()?;
8373 }
8374 Ok(pr) => {
8375 self.state.status = match pr.state {
8376 land::PrLifecycle::Merged => RunStatus::Merged,
8377 _ => RunStatus::Blocked,
8378 };
8379 if bump::should_release_bump(self.state.status)
8386 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8387 {
8388 self.state
8394 .event("bump", format!("release bump skipped: {e:#}"));
8395 }
8396 if self.state.status == RunStatus::Merged {
8399 crate::followup::after_merge(&mut self.state, &pr.url).await;
8400 }
8401 self.state.save()?;
8402 }
8403 Err(e) => {
8404 if e.is::<crate::repo_lock::ResourceBlocked>() {
8405 return Err(e);
8406 }
8407 self.state.status = RunStatus::Blocked;
8408 self.state.event("land", format!("gave up: {e}"));
8409 self.state.save()?;
8410 }
8411 }
8412 break;
8413 }
8414 Ok(())
8415 }
8416
8417 async fn repair_merge_review(&mut self) -> Result<bool> {
8420 use crate::run::MergeRepairPhase as Phase;
8421 crate::pr_head::ensure_owner(&self.state)?;
8422 let i = self.state.merge_repairs.len() - 1;
8423 let request = self.state.merge_repairs[i].clone();
8424 if request.phase == Phase::Fixing {
8425 let restored = match self.state.winner().cloned() {
8426 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8427 None => Err(anyhow::anyhow!("repair has no winning branch")),
8428 };
8429 let why = match restored {
8430 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8431 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8432 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8433 };
8434 self.hold_merge_repair(i, why)?;
8435 return Ok(false);
8436 }
8437 let failure = if i >= self.state.config.graph.land_rounds
8438 || self.state.config.graph.review_rounds == 0
8439 {
8440 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8441 } else {
8442 None
8443 };
8444 if let Some(why) = failure {
8445 self.hold_merge_repair(i, why)?;
8446 return Ok(false);
8447 }
8448 if request.phase == Phase::Requested {
8449 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8450 Ok(pr) => pr,
8451 Err(e) => {
8452 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8453 return Ok(false);
8454 }
8455 };
8456 self.state.merge_repairs[i].phase = Phase::Fixing;
8457 self.state.land_approval = None;
8458 self.state.land_armed_head = None;
8459 self.state.save()?;
8460 let evidence = serde_json::to_string_pretty(&request)?;
8463 let reason = format!(
8464 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8465 );
8466 let budget = self.state.config.graph.land_rounds;
8467 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8468 match result {
8469 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8470 Ok(land::Fixed::Failed(why)) => {
8471 self.hold_merge_repair(
8472 i,
8473 format!(
8474 "repair failed: {why}; inspect the branch and run an explicit review"
8475 ),
8476 )?;
8477 return Ok(false);
8478 }
8479 Err(e) => {
8480 self.hold_merge_repair(
8481 i,
8482 format!(
8483 "repair failed: {e:#}; inspect the branch and run an explicit review"
8484 ),
8485 )?;
8486 return Ok(false);
8487 }
8488 }
8489 self.begin_merge_repair_review(i)?;
8493 }
8494 self.review_loop().await?;
8495 self.gate().await?;
8496 if self.resource_halt()? || self.state.parked {
8497 return Ok(false);
8498 }
8499 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8500 != Some(RunStatus::Gating)
8501 || !self.state.gate_allows_progress()
8502 {
8503 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8504 return Ok(false);
8505 }
8506 let winner = self
8509 .state
8510 .winner()
8511 .cloned()
8512 .context("repair has no winning branch")?;
8513 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8514 if self.state.gate_head.as_deref() != Some(after.as_str())
8515 || !git::is_clean(&winner.worktree).await?
8516 {
8517 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8518 return Ok(false);
8519 }
8520 let expected = self
8521 .state
8522 .pr_pushed_head
8523 .as_deref()
8524 .unwrap_or(&request.head)
8525 .to_owned();
8526 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8527 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8530 let refspec = format!("{after}:refs/heads/{public}");
8531 let push = git::git_raw(
8532 &winner.worktree,
8533 &["push", &lease, &self.state.config.merge.remote, &refspec],
8534 )
8535 .await;
8536 match push {
8537 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8538 Ok(out) => {
8539 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8540 return Ok(false);
8541 }
8542 Err(e) => {
8543 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8544 return Ok(false);
8545 }
8546 }
8547 self.state.merge_repairs[i].phase = Phase::Complete;
8548 self.state.status = RunStatus::Landing;
8549 self.state.event(
8550 "repair",
8551 "new review and gate completed; fresh owner merge decision required",
8552 );
8553 self.state.save()?;
8554 Ok(true)
8555 }
8556
8557 fn merge_repair_artifacts(&self) -> PathBuf {
8558 let artifacts = agent::artifacts_dir(&self.state.dir());
8559 match self
8560 .state
8561 .merge_repairs
8562 .last()
8563 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8564 {
8565 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8566 None => artifacts,
8567 }
8568 }
8569
8570 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8571 self.state.reviews.clear();
8572 self.state.gate.clear();
8573 self.state.gate_ran = false;
8574 self.state.gate_head = None;
8575 self.state.gate_fixes.clear();
8576 self.state.gate_repair_e2e = None;
8577 self.state.pre_gate.clear();
8578 self.state.pre_gate_commit = None;
8579 self.state.contested_handoff = None;
8580 self.state.land_reruns = None;
8581 self.state.land_approval = None;
8582 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8583 self.state.status = RunStatus::Reviewing;
8584 self.state.save()
8585 }
8586
8587 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8588 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8589 self.state.merge_repairs[index].failure = Some(why.clone());
8590 self.state.status = RunStatus::Blocked;
8591 self.state.event("repair", why);
8592 self.state.save()
8593 }
8594
8595 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8599 if let Some(existing) = self.state.seats.get(key)
8600 && existing.agent == agent
8601 {
8602 return existing.clone();
8603 }
8604 let fresh = if self.state.seats.contains_key(key) {
8607 handover_seat(key, agent, self.state.next_seat_seed())
8608 } else {
8609 SeatState::new(key, agent, self.state.seed)
8610 };
8611 self.state.seats.insert(key.to_owned(), fresh.clone());
8612 fresh
8613 }
8614
8615 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8621 match self.state.seats.get(key) {
8622 Some(s) if s.agent != spec.id => {
8623 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8624 }
8625 _ => spec,
8626 }
8627 }
8628
8629 fn view(&self, c: &Candidate) -> CandidateView {
8631 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8632 .unwrap_or_default();
8633 let (patch, _) = blind::sanitize_patch(
8634 &format!("candidate {} patch", c.label),
8635 &raw,
8636 &self.state.config.blind,
8637 );
8638 CandidateView {
8639 label: c.label,
8640 branch: c.branch.clone(),
8641 summary: c.summary.clone(),
8642 stat: c.stat.clone(),
8643 patch,
8644 }
8645 }
8646
8647 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8649 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8650 prompt::judge(
8651 "(see above)",
8652 &views,
8653 self.roles.judges.len(),
8654 base_short,
8655 "en",
8656 )
8657 }
8658
8659 fn vote_prompt_full(
8665 &self,
8666 j: usize,
8667 labels: &[char],
8668 language: &str,
8669 candidates: &[Candidate],
8670 base_short: &str,
8671 ) -> String {
8672 let mut text = format!(
8673 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8674 prompt::final_vote(labels, language),
8675 self.state.instruction,
8676 self.candidate_block(candidates, base_short)
8677 );
8678 if let Some(own) = self
8679 .state
8680 .judgements
8681 .get(j)
8682 .filter(|r| !r.ranking.is_empty())
8683 {
8684 let reasons = own
8685 .reasons
8686 .iter()
8687 .map(|(k, v)| format!("- {k}: {v}"))
8688 .collect::<Vec<_>>()
8689 .join("\n");
8690 text.push_str(&format!(
8691 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8692 own.ranking.iter().collect::<String>(),
8693 if reasons.is_empty() {
8694 ""
8695 } else {
8696 ", because:\n"
8697 }
8698 ));
8699 }
8700 if !self.state.deliberation.is_empty() {
8701 text.push_str("\n# What was argued before this vote\n");
8702 for t in self.transcript(&[], j) {
8703 text.push_str(&format!(
8704 "\n## {}{}\n\n{}\n",
8705 t.who,
8706 if t.is_self { " (you)" } else { "" },
8707 t.body.trim()
8708 ));
8709 }
8710 }
8711 text
8712 }
8713
8714 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8721 let mut turns = Vec::new();
8722 for j in &self.state.judgements {
8723 if j.ranking.is_empty() {
8724 continue;
8725 }
8726 let reasons = j
8727 .reasons
8728 .iter()
8729 .map(|(k, v)| format!("- {k}: {v}"))
8730 .collect::<Vec<_>>()
8731 .join("\n");
8732 turns.push(Turn {
8733 who: format!("Judge {} (opening ranking)", j.judge),
8734 is_self: j.judge == self_idx + 1,
8735 body: format!(
8736 "Ranked {}{}{reasons}",
8737 j.ranking.iter().collect::<String>(),
8738 if reasons.is_empty() {
8739 ""
8740 } else {
8741 ", because:\n"
8742 }
8743 ),
8744 });
8745 }
8746 for t in self
8747 .state
8748 .deliberation
8749 .iter()
8750 .flat_map(|r| r.turns.iter())
8751 .chain(current)
8752 {
8753 turns.push(Turn {
8754 who: format!("Judge {}", t.judge),
8755 is_self: t.judge == self_idx + 1,
8756 body: t.body.clone(),
8757 });
8758 }
8759 turns
8760 }
8761}
8762
8763fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8765 agent::has_session(spec.kind, seat, sessions)
8766}
8767
8768fn next_untried_in_roster<'a>(
8789 roster: &'a [AgentSpec],
8790 start: usize,
8791 tried: &BTreeSet<String>,
8792) -> Option<&'a AgentSpec> {
8793 roster
8794 .get(start + 1..)?
8795 .iter()
8796 .find(|s| !tried.contains(&s.id))
8797}
8798
8799fn pick_successor<'a>(
8820 roster: &'a [AgentSpec],
8821 start: usize,
8822 tried: &BTreeSet<String>,
8823 carried: Option<&BTreeSet<String>>,
8824 others: &BTreeSet<String>,
8825) -> Option<&'a AgentSpec> {
8826 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8827 roster
8828 .get(start + 1..)
8829 .and_then(|tail| {
8830 tail.iter()
8831 .filter(free)
8832 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8833 })
8834 .or_else(|| {
8835 carried
8836 .is_some()
8837 .then(|| roster.iter().find(free))
8838 .flatten()
8839 })
8840 .or_else(|| match carried {
8841 Some(c) => next_for_seat(roster, start, tried, c),
8842 None => next_untried_in_roster(roster, start, tried),
8843 })
8844}
8845
8846fn next_for_seat<'a>(
8856 roster: &'a [AgentSpec],
8857 start: usize,
8858 round_tried: &BTreeSet<String>,
8859 carried_failed: &BTreeSet<String>,
8860) -> Option<&'a AgentSpec> {
8861 roster
8862 .get(start + 1..)?
8863 .iter()
8864 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8865 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8866}
8867
8868fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8874 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8875 return spec;
8876 };
8877 let ok = |id: &str| !h.failed.contains(id);
8878 if let Some(last) = h.last_ok.as_deref()
8879 && ok(last)
8880 && let Some(s) = roster.iter().find(|s| s.id == last)
8881 {
8882 return s.clone();
8883 }
8884 if ok(&spec.id) {
8885 return spec;
8886 }
8887 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8888 roster
8889 .get(from + 1..)
8890 .into_iter()
8891 .flatten()
8892 .chain(roster.iter())
8893 .find(|s| ok(&s.id))
8894 .cloned()
8895 .unwrap_or(spec)
8896}
8897
8898pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8902 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8903}
8904
8905const TIMED_OUT: &str = "timed out";
8908
8909impl FailClass {
8910 fn of(out: &AgentOutcome) -> Option<Self> {
8912 match out {
8913 AgentOutcome::Ok(_) => None,
8914 AgentOutcome::Quota(_) => Some(Self::Quota),
8915 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8916 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8917 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8918 AgentOutcome::ResourceBlocked(_) => None,
8921 }
8922 }
8923
8924 fn stem_word(&self) -> &'static str {
8926 match self {
8927 Self::Quota => "quota",
8928 _ => "handover",
8929 }
8930 }
8931}
8932
8933fn failure_signature(msg: &str) -> String {
8937 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8938 if let Some(code) = line
8941 .strip_prefix("process exited with no usable output (exit code ")
8942 .and_then(|s| s.strip_suffix(')'))
8943 .and_then(|s| s.parse::<i32>().ok())
8944 {
8945 line = format!("exited with some({code}) and no usable output");
8946 } else if matches!(
8947 line.as_str(),
8948 "process exited with no usable output (no exit code reported)"
8949 | "process exited with no usable output (exit code unavailable)"
8950 ) {
8951 line = "exited with none and no usable output".to_owned();
8952 }
8953 let mut out = Vec::new();
8954 for word in line.split_whitespace() {
8955 if word.contains('/') || word.contains('\\') {
8956 out.push("<path>".to_owned());
8957 continue;
8958 }
8959 let mut w = String::new();
8960 let mut in_digits = false;
8961 for c in word.chars() {
8962 if c.is_ascii_digit() {
8963 if !in_digits {
8964 w.push('#');
8965 }
8966 in_digits = true;
8967 } else {
8968 in_digits = false;
8969 w.push(c);
8970 }
8971 }
8972 out.push(w);
8973 }
8974 out.join(" ").chars().take(120).collect()
8975}
8976
8977fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8984 match cur {
8985 FailClass::Quota | FailClass::Timeout => true,
8986 FailClass::Other(_) => prev != Some(cur),
8987 }
8988}
8989
8990fn fail_reason(out: &AgentOutcome) -> String {
8992 match out {
8993 AgentOutcome::Ok(_) => String::new(),
8994 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8995 AgentOutcome::Dropped(o) => format!(
8996 "the CLI dropped the stream ({})",
8997 o.dropped
8998 .as_ref()
8999 .map(|d| d.why.as_str())
9000 .unwrap_or("it ended without delivering its answer")
9001 ),
9002 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9003 e.lines().next().unwrap_or("").chars().take(160).collect()
9004 }
9005 }
9006}
9007
9008pub(crate) fn record_handover(
9011 state: &mut RunState,
9012 node: &str,
9013 seat: &str,
9014 from: &str,
9015 to: &str,
9016 class: &FailClass,
9017 reason: &str,
9018) {
9019 let message = if *class == FailClass::Quota {
9020 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9021 } else {
9022 format!("{seat}: handed over {from} -> {to} ({reason})")
9023 };
9024 state.event(node, message);
9025 state.handovers.push(Handover {
9026 at: Timestamp::now(),
9027 node: node.to_owned(),
9028 seat: seat.to_owned(),
9029 from: from.to_owned(),
9030 to: to.to_owned(),
9031 reason: reason.to_owned(),
9032 });
9033}
9034
9035fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9049 commands.iter().any(|c| c.exit_code.is_none())
9050}
9051
9052fn verified_noop_claim(
9065 usable: bool,
9066 commands: &[agent::CommandEvidence],
9067 text: &str,
9068) -> Option<String> {
9069 (usable && !has_unconfirmed_command(commands))
9070 .then(|| verdict::verified_noop(text))
9071 .flatten()
9072}
9073
9074fn short(commit: &str) -> String {
9075 commit.chars().take(7).collect()
9076}
9077
9078fn make_executable(path: &Path) -> Result<()> {
9079 #[cfg(unix)]
9080 {
9081 use std::os::unix::fs::PermissionsExt as _;
9082 let mut perms = std::fs::metadata(path)?.permissions();
9083 perms.set_mode(0o755);
9084 std::fs::set_permissions(path, perms)?;
9085 }
9086 #[cfg(not(unix))]
9087 {
9088 let _ = path;
9089 }
9090 Ok(())
9091}
9092
9093struct WaveCtx<'a> {
9100 run: &'a str,
9103 node: &'a str,
9105 prompts: &'a Prompts,
9106 cache: Option<&'a Path>,
9108 round: Option<usize>,
9111 carry_seats: bool,
9115 pauses: Option<(&'a Pause, &'a Pause)>,
9118}
9119
9120fn repository_wait(
9122 state: &RunState,
9123 pauses: Option<(&Pause, &Pause)>,
9124) -> crate::repo_lock::WaitPolicy {
9125 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9126 let inherited = crate::repo_lock::WaitPolicy::current();
9127 crate::repo_lock::WaitPolicy::new(
9128 Duration::from_secs(state.config.graph.timeout_resource_wait),
9129 move || {
9130 inherited.cancelled()
9131 || pauses
9132 .as_ref()
9133 .is_some_and(|(a, b)| a.parked() || b.parked())
9134 },
9135 )
9136}
9137
9138fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9139 if error.is::<crate::repo_lock::ResourceBlocked>() {
9140 state.resource_block = Some(crate::run::ResourceBlock {
9141 node: "repository".into(),
9142 kind: if error.to_string().contains("disk pressure") {
9143 "disk pressure"
9144 } else {
9145 "repository coordination"
9146 }
9147 .into(),
9148 detail: error.to_string(),
9149 at: Timestamp::now(),
9150 });
9151 state.event("resource", error.to_string());
9152 }
9153}
9154
9155async fn coordinate_operation(
9156 state: &mut RunState,
9157 cwd: &Path,
9158 wait: &crate::repo_lock::WaitPolicy,
9159) -> Result<Option<crate::repo_lock::SeatLease>> {
9160 let result =
9161 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9162 .await;
9163 if let Err(e) = &result {
9164 record_repository_block(state, e);
9165 }
9166 result
9167}
9168
9169async fn run_one(
9171 job: SeatJob,
9172 sem: Arc<Semaphore>,
9173 ctx: &WaveCtx<'_>,
9174 state: &mut RunState,
9175 attempt: usize,
9176) -> (SeatState, AgentOutcome) {
9177 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9178 .await
9179 .pop()
9180 .expect("one job in, one result out");
9181 (seat, out)
9182}
9183
9184async fn wave(
9190 jobs: Vec<SeatJob>,
9191 sem: Arc<Semaphore>,
9192 ctx: &WaveCtx<'_>,
9193 state: &mut RunState,
9194 attempt: usize,
9195) -> Vec<(usize, SeatState, AgentOutcome)> {
9196 let WaveCtx {
9197 run,
9198 node,
9199 prompts,
9200 cache,
9201 round,
9202 carry_seats: _,
9203 pauses,
9204 } = *ctx;
9205 for job in &jobs {
9206 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9207 }
9208 if let Err(e) = state.save() {
9209 tracing::warn!("could not persist in-progress seats: {e:#}");
9214 }
9215 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9231 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9235 let cache_guard = if let Some(cache_dir) = cache {
9236 if jobs_had_a_writer {
9237 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9238 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9239 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9240 {
9241 Ok(Some(g)) => Some(g),
9242 Ok(None) => {
9245 state.resource_block = Some(crate::run::ResourceBlock {
9246 node: node.to_owned(),
9247 kind: "shared build cache".to_owned(),
9248 detail: "the run is parking".to_owned(),
9249 at: Timestamp::now(),
9250 });
9251 return block_wave(
9252 jobs,
9253 state,
9254 node,
9255 attempt,
9256 "parked while waiting for the shared build cache",
9257 );
9258 }
9259 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9262 Err(e) => {
9263 let why = format!("{e:#}");
9267 state.resource_block = Some(crate::run::ResourceBlock {
9268 node: node.to_owned(),
9269 kind: "shared build cache".to_owned(),
9270 detail: why.clone(),
9271 at: Timestamp::now(),
9272 });
9273 state.event(
9274 "resource",
9275 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9276 );
9277 return block_wave(jobs, state, node, attempt, &why);
9278 }
9279 }
9280 } else {
9281 None
9282 }
9283 } else {
9284 None
9285 };
9286 state.seats_spawning(node, attempt);
9289 if let Err(e) = state.save() {
9290 tracing::warn!("could not persist the start of a wave: {e:#}");
9291 }
9292 let slot_limit = resource_limit
9296 + jobs
9297 .iter()
9298 .map(|j| j.timeout)
9299 .max()
9300 .unwrap_or_default()
9301 .saturating_mul(jobs.len() as u32);
9302 let mut set = tokio::task::JoinSet::new();
9303 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9306 let overlay = prompts.overlay(node);
9307 let repository_wait = repository_wait(state, pauses);
9308 for (i, mut job) in jobs.into_iter().enumerate() {
9309 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9310 if cache.is_some() {
9311 job.prompt.push('\n');
9312 job.prompt
9313 .push_str(&prompt::build_cache_note(node, job.allow_write));
9314 }
9315 let sem = Arc::clone(&sem);
9316 let run = run.to_owned();
9317 let node = node.to_owned();
9318 let attachments = if node == "implement" {
9321 state.attachments.clone()
9322 } else {
9323 Vec::new()
9324 };
9325 let cache = cache
9336 .filter(|_| job.allow_write && cache_guard.is_some())
9337 .map(Path::to_path_buf);
9338 let spawned_tx = spawned_tx.clone();
9339 let repository_wait = repository_wait.clone();
9340 let coordination = state.config.coordination.clone();
9341 let fetched_base = state.fetched_base.clone();
9342 set.spawn(async move {
9343 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9347 Ok(p) => p,
9348 Err(_) => {
9349 return (
9350 i,
9351 job.seat,
9352 AgentOutcome::ResourceBlocked(format!(
9353 "no parallel slot became free within {}s",
9354 slot_limit.as_secs()
9355 )),
9356 false,
9357 );
9358 }
9359 };
9360 spawned_tx.send(job.seat.key.clone()).ok();
9361 let mut seat = job.seat;
9362 let out = repository_wait
9363 .scope(agent::invoke(
9364 &job.spec,
9365 &mut seat,
9366 &Invocation {
9367 scope: crate::agent::InvocationScope::Repository,
9368 coordination: coordination.as_ref(),
9369 fetched_base: fetched_base.as_ref(),
9370 cwd: &job.cwd,
9371 prompt: &job.prompt,
9372 timeout: job.timeout,
9373 allow_write: job.allow_write,
9374 unsandboxed: false,
9375 sessions: job.sessions,
9376 artifacts: &job.artifacts,
9377 stem: &job.stem,
9378 run: &run,
9379 node: &node,
9380 cache_dir: cache.as_deref(),
9381 attachments: &attachments,
9382 writable: &[],
9383 },
9384 ))
9385 .await;
9386 let interrupted = out
9387 .as_ref()
9388 .err()
9389 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9390 let out = match out {
9391 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9392 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9393 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9401 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9402 Ok(o) => AgentOutcome::Failed(format!(
9403 "process exited with no usable output ({})",
9404 crate::notices::exit_status(o.exit_code)
9405 )),
9406 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9407 AgentOutcome::ResourceBlocked(e.to_string())
9408 }
9409 Err(e) => AgentOutcome::Failed(e.to_string()),
9410 };
9411 (i, seat, out, interrupted)
9412 });
9413 }
9414 drop(spawned_tx);
9415 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9416 loop {
9417 let joined = tokio::select! {
9418 Some(key) = spawned_rx.recv() => {
9419 state.seat_spawned(&key);
9421 if let Err(e) = state.save() {
9422 tracing::warn!("could not persist a seat's start: {e:#}");
9423 }
9424 continue;
9425 }
9426 joined = set.join_next() => joined,
9427 };
9428 let Some(joined) = joined else {
9429 break;
9430 };
9431 let (i, seat, out, interrupted) = match joined {
9432 Ok(v) => v,
9433 Err(e) => {
9437 tracing::error!("agent task panicked: {e}");
9438 continue;
9439 }
9440 };
9441 state.seat_finished(&seat.key);
9442 if interrupted
9443 && node == "implement"
9444 && let Some(label) = state
9445 .candidates
9446 .iter()
9447 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9448 .map(|candidate| candidate.label)
9449 {
9450 state.candidate_preservation.insert(
9451 label,
9452 "interrupted implementation awaiting persistence".into(),
9453 );
9454 state.interrupted_implementations.insert(label);
9455 }
9456 if let AgentOutcome::ResourceBlocked(why) = &out
9457 && state.resource_block.is_none()
9458 {
9459 state.resource_block = Some(crate::run::ResourceBlock {
9460 node: node.to_owned(),
9461 kind: if why.contains("disk pressure") {
9462 "disk pressure"
9463 } else if why.contains("repository coordination") {
9464 "repository coordination"
9465 } else {
9466 "parallel slot"
9467 }
9468 .to_owned(),
9469 detail: why.clone(),
9470 at: Timestamp::now(),
9471 });
9472 let activity = if interrupted {
9473 "agent execution interrupted"
9474 } else {
9475 "no agent was started"
9476 };
9477 state.event("resource", format!("{node}: {activity} - {why}"));
9478 }
9479 record_jobs(state, node, round, &seat.key, &out);
9480 if let Err(e) = state.save() {
9481 tracing::warn!("could not persist a seat's completion: {e:#}");
9482 }
9483 if collected.len() <= i {
9484 collected.resize_with(i + 1, || None);
9485 }
9486 collected[i] = Some((i, seat, out));
9487 }
9488 if state
9494 .active
9495 .values()
9496 .any(|a| a.node == node && a.attempt == attempt)
9497 {
9498 state
9499 .active
9500 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9501 if let Err(e) = state.save() {
9502 tracing::warn!("could not persist the end of a wave: {e:#}");
9503 }
9504 }
9505 if let Some(cache_dir) = cache
9512 && jobs_had_a_writer
9513 {
9514 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9515 }
9516 if let Some(guard) = cache_guard {
9517 guard.release();
9518 }
9519 collected.into_iter().flatten().collect()
9520}
9521
9522fn record_jobs(
9533 state: &mut RunState,
9534 node: &str,
9535 round: Option<usize>,
9536 seat: &str,
9537 out: &AgentOutcome,
9538) {
9539 let commands: &[agent::CommandEvidence] = match out {
9540 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9541 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9542 };
9543 let checked_at = Timestamp::now();
9544 for c in commands {
9545 state.jobs.push(JobRecord {
9546 node: node.to_owned(),
9547 round,
9548 seat: seat.to_owned(),
9549 id: c.id.clone(),
9550 description: c.description.clone(),
9551 checked_at,
9552 status: match c.exit_code {
9553 Some(0) => JobStatus::Completed,
9554 Some(_) => JobStatus::Failed,
9555 None => JobStatus::Unknown,
9556 },
9557 exit_code: c.exit_code,
9558 result_summary: c.result_summary.clone(),
9559 source: c.source.clone(),
9560 });
9561 }
9562}
9563
9564fn round_is_clean(
9585 blocking: usize,
9586 e2e_ok: bool,
9587 answered: usize,
9588 expected: usize,
9589 quota_missing: usize,
9590 policy: IncompleteReviewPolicy,
9591) -> bool {
9592 if blocking != 0 || !e2e_ok {
9593 return false;
9594 }
9595 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9596 return true;
9597 }
9598 answered > 0 && expected - answered <= quota_missing
9599}
9600
9601pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9625 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9626 return Some(RunStatus::Gating);
9627 }
9628 let last = reviews.last()?;
9629 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9630 if reviews.len() < max_rounds && !stagnant {
9631 return None;
9632 }
9633 if last.incomplete() && last.blocking == 0 {
9634 return Some(RunStatus::Blocked);
9635 }
9636 if last.e2e_status() == E2eStatus::ResourceBlocked {
9637 return None;
9638 }
9639 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9640 RunStatus::Gating
9641 } else {
9642 RunStatus::Blocked
9643 })
9644}
9645
9646pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9661 if nudged {
9662 (full / 4).max(Duration::from_secs(120)).min(full)
9663 } else {
9664 full
9665 }
9666}
9667
9668#[allow(clippy::too_many_arguments)]
9697async fn ask_json_wave<T>(
9698 jobs: Vec<SeatJob>,
9699 sem: Arc<Semaphore>,
9700 retries: usize,
9701 roster: &[AgentSpec],
9702 ctx: &WaveCtx<'_>,
9703 losses: &mut Vec<QuotaLoss>,
9704 state: &mut RunState,
9705 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9706) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9707where
9708 T: serde::de::DeserializeOwned + Send + 'static,
9709{
9710 ask_wave_with(
9711 jobs,
9712 sem,
9713 retries,
9714 roster,
9715 ctx,
9716 losses,
9717 state,
9718 &|text: &str| {
9719 let v = verdict::extract_json::<T>(text)?;
9720 validate(&v)?;
9721 Ok(v)
9722 },
9723 )
9724 .await
9725}
9726
9727#[allow(clippy::too_many_arguments)]
9736async fn ask_wave_with<T>(
9737 jobs: Vec<SeatJob>,
9738 sem: Arc<Semaphore>,
9739 retries: usize,
9740 roster: &[AgentSpec],
9741 ctx: &WaveCtx<'_>,
9742 losses: &mut Vec<QuotaLoss>,
9743 state: &mut RunState,
9744 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9745) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9746where
9747 T: Send + 'static,
9748{
9749 let n = jobs.len();
9750 let originals: Vec<SeatJob> = jobs;
9751 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9752 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9753 let mut nudges: Vec<usize> = vec![0; n];
9760 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9764 let mut tried: Vec<BTreeSet<String>> = specs
9765 .iter()
9766 .map(|s| BTreeSet::from([s.id.clone()]))
9767 .collect();
9768 let starts: Vec<usize> = specs
9769 .iter()
9770 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9771 .collect();
9772 let carry = ctx.carry_seats && !roster.is_empty();
9773 let carried: Vec<BTreeSet<String>> = originals
9776 .iter()
9777 .map(|j| {
9778 state
9779 .seat_history
9780 .get(&j.seat.key)
9781 .filter(|_| carry)
9782 .map(|h| h.failed.clone())
9783 .unwrap_or_default()
9784 })
9785 .collect();
9786 let mut prev: Vec<Option<FailClass>> = originals
9787 .iter()
9788 .map(|j| {
9789 state
9790 .seat_history
9791 .get(&j.seat.key)
9792 .filter(|_| carry)
9793 .and_then(|h| h.last_fail.clone())
9794 })
9795 .collect();
9796 let next_agent =
9797 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9798 let others: BTreeSet<String> = specs
9799 .iter()
9800 .enumerate()
9801 .filter(|(j, _)| *j != i)
9802 .map(|(_, s)| s.id.clone())
9803 .collect();
9804 pick_successor(
9805 roster,
9806 starts[i],
9807 tried,
9808 carry.then(|| &carried[i]),
9809 &others,
9810 )
9811 .cloned()
9812 };
9813 let mut fresh: Vec<Option<String>> = vec![None; n];
9814 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9815 let mut pending: Vec<usize> = (0..n).collect();
9816
9817 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9822 for round in 0..max_rounds {
9823 if pending.is_empty() {
9824 break;
9825 }
9826 let mut batch = Vec::with_capacity(pending.len());
9827 let mut renudged: Vec<&str> = Vec::new();
9828 for &i in &pending {
9829 let src = &originals[i];
9830 let full: &str = match &src.handover {
9835 Some(h) if specs[i].id != src.spec.id => h,
9836 _ => &src.prompt,
9837 };
9838 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9841 (
9842 full.to_owned(),
9843 src.timeout,
9844 format!("{}-{word}-{}", src.stem, specs[i].id),
9845 )
9846 } else if nudges[i] == 0 {
9847 (full.to_owned(), src.timeout, src.stem.clone())
9848 } else {
9849 renudged.push(src.seat.key.as_str());
9850 let why = done[i]
9851 .as_ref()
9852 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9853 .unwrap_or_else(|| "no parsable answer".to_owned());
9854 let nudge = prompt::nudge(&why);
9855 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9856 let prompt = if nudged {
9857 nudge
9858 } else {
9859 format!("{full}\n\n---\n\n{nudge}")
9860 };
9861 (
9862 prompt,
9863 retry_budget(src.timeout, nudged),
9864 format!("{}-retry{}", src.stem, nudges[i]),
9865 )
9866 };
9867 batch.push(SeatJob {
9868 spec: specs[i].clone(),
9869 seat: seats[i].clone(),
9870 cwd: src.cwd.clone(),
9871 prompt,
9872 timeout,
9873 allow_write: src.allow_write,
9874 sessions: src.sessions,
9875 artifacts: src.artifacts.clone(),
9876 stem,
9877 handover: None,
9878 });
9879 }
9880
9881 if !renudged.is_empty() {
9882 state.event(
9883 ctx.node,
9884 format!("retry {round}: re-asking {}", renudged.join(", ")),
9885 );
9886 }
9887 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9888 let mut still = Vec::new();
9889 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9890 seats[i] = seat;
9891 if let AgentOutcome::ResourceBlocked(why) = &out {
9894 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9895 continue;
9896 }
9897 let class = FailClass::of(&out);
9898 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9901 && nudges[i] < retries
9902 && !roster.is_empty();
9903 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9904 let next = should_hand_over(prev[i].as_ref(), &cur)
9905 .then(|| next_agent(i, &tried[i], &specs))
9906 .flatten();
9907 if carry {
9908 let h = state
9909 .seat_history
9910 .entry(originals[i].seat.key.clone())
9911 .or_default();
9912 h.failed.insert(specs[i].id.clone());
9913 h.last_fail = Some(cur.clone());
9914 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9915 h.last_ok = None;
9916 }
9917 if let Err(e) = state.save() {
9920 tracing::warn!("could not persist a seat's failure history: {e:#}");
9921 }
9922 }
9923 if let Some(next) = next {
9924 record_handover(
9925 state,
9926 ctx.node,
9927 &originals[i].seat.key,
9928 &specs[i].id,
9929 &next.id,
9930 &cur,
9931 &fail_reason(&out),
9932 );
9933 tried[i].insert(next.id.clone());
9934 prev[i] = Some(cur.clone());
9935 seats[i] =
9936 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9937 specs[i] = next;
9938 fresh[i] = Some(cur.stem_word().to_owned());
9939 nudges[i] = 0;
9940 done[i] = Some(Err(anyhow::anyhow!(
9941 "handed over after: {}",
9942 fail_reason(&out)
9943 )));
9944 still.push(i);
9945 continue;
9946 }
9947 }
9948 if carry
9949 && !nudge_first
9950 && let Some(cur) = class.clone()
9951 {
9952 let h = state
9954 .seat_history
9955 .entry(originals[i].seat.key.clone())
9956 .or_default();
9957 h.failed.insert(specs[i].id.clone());
9958 h.last_fail = Some(cur);
9959 }
9960 let parsed = match out {
9961 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9962 AgentOutcome::Quota(o) => {
9963 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9964 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9965 }
9966 AgentOutcome::Dropped(o) => {
9970 let why = o
9971 .dropped
9972 .as_ref()
9973 .map(|d| d.why.as_str())
9974 .unwrap_or("the CLI ended the stream without delivering its answer");
9975 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9976 }
9977 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9978 Err(anyhow::anyhow!(e))
9979 }
9980 };
9981 let quota = class == Some(FailClass::Quota);
9982 let failed = parsed.is_err();
9983 done[i] = Some(parsed);
9984 if carry && !failed {
9985 let h = state
9986 .seat_history
9987 .entry(originals[i].seat.key.clone())
9988 .or_default();
9989 h.failed.remove(&specs[i].id);
9990 h.last_ok = Some(specs[i].id.clone());
9991 h.last_fail = None;
9992 }
9993 let agent_failure = class.is_some()
10001 && !nudge_first
10002 && !roster.is_empty()
10003 && next_agent(i, &tried[i], &specs).is_some();
10004 if failed && !quota && !agent_failure && nudges[i] < retries {
10005 nudges[i] += 1;
10006 still.push(i);
10007 }
10008 }
10009 pending = still;
10010 }
10011
10012 for (i, q) in last_quota.into_iter().enumerate() {
10015 if let Some(reset) = q {
10016 losses.push(QuotaLoss {
10017 seat: originals[i].seat.key.clone(),
10018 node: ctx.node.to_owned(),
10019 at: Timestamp::now(),
10020 reset,
10021 });
10022 }
10023 }
10024
10025 seats
10026 .into_iter()
10027 .zip(done)
10028 .zip(nudges)
10029 .map(|((seat, res), attempts)| {
10030 (
10031 seat,
10032 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10033 attempts,
10034 )
10035 })
10036 .collect()
10037}
10038
10039fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10041 results
10042 .iter()
10043 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10044}
10045
10046fn block_wave(
10050 jobs: Vec<SeatJob>,
10051 state: &mut RunState,
10052 node: &str,
10053 attempt: usize,
10054 why: &str,
10055) -> Vec<(usize, SeatState, AgentOutcome)> {
10056 state.resource_wait = None;
10057 state
10058 .active
10059 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10060 if let Err(e) = state.save() {
10061 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10062 }
10063 jobs.into_iter()
10064 .enumerate()
10065 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10066 .collect()
10067}
10068
10069async fn acquire_cache_lease(
10082 state: &mut RunState,
10083 cache_dir: &Path,
10084 owner: &crate::cache::Owner,
10085 budget: Duration,
10086 context: &str,
10087 cancel: &(dyn Fn() -> bool + Sync),
10088) -> Result<Option<crate::cache::Guard>> {
10089 let home = crate::run::home();
10090 let started = Instant::now();
10091 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10092 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10093 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10094 Err(e) => {
10095 state.event(
10096 "verify",
10097 format!("{context}: could not check the shared build cache: {e:#}"),
10098 );
10099 if let Err(e2) = state.save() {
10100 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10101 }
10102 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10103 }
10104 };
10105 state.event(
10106 "verify",
10107 format!(
10108 "{context}: waiting for the shared build cache at {} ({})",
10109 cache_dir.display(),
10110 busy.describe()
10111 ),
10112 );
10113 state.resource_wait = Some(crate::run::ResourceWait {
10116 node: context.to_owned(),
10117 kind: "shared build cache".to_owned(),
10118 holder: busy.describe(),
10119 since: Timestamp::now(),
10120 limit_secs: budget.as_secs(),
10121 });
10122 if let Err(e) = state.save() {
10123 tracing::warn!("could not persist a cache wait: {e:#}");
10124 }
10125 let remaining = budget.saturating_sub(started.elapsed());
10126 let waited = crate::cache::wait_for_unless(
10127 &home,
10128 cache_dir,
10129 owner,
10130 remaining,
10131 Duration::from_secs(5),
10132 cancel,
10133 )
10134 .await;
10135 state.resource_wait = None;
10136 match waited {
10137 Ok(g) => Ok(g),
10138 Err(e) => {
10139 state.event("verify", format!("{context}: {e:#}"));
10140 if let Err(e2) = state.save() {
10141 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10142 }
10143 Err(e)
10144 }
10145 }
10146}
10147
10148#[derive(Debug)]
10153struct CacheCheckError(String);
10154
10155impl std::fmt::Display for CacheCheckError {
10156 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10157 f.write_str(&self.0)
10158 }
10159}
10160
10161impl std::error::Error for CacheCheckError {}
10162
10163#[allow(clippy::too_many_arguments)]
10184async fn with_cache_lease<'s, F, Fut>(
10185 state: &'s mut RunState,
10186 cache_dir: Option<&Path>,
10187 node: &str,
10188 seat: &str,
10189 worktree: &Path,
10190 head: &str,
10191 budget: Duration,
10192 context: &str,
10193 wait: &crate::repo_lock::WaitPolicy,
10194 body: F,
10195) -> Result<(Vec<CommandOutcome>, bool)>
10196where
10197 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10198 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10199{
10200 let started = Instant::now();
10203 let home = crate::run::home();
10204 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10205 let guard = if let Some(cache_dir) = cache_dir {
10206 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10207 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10208 wait.cancelled()
10209 })
10210 .await
10211 .and_then(|g| {
10212 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10213 }) {
10214 Ok(g) => Some(g),
10215 Err(e) => {
10216 return Ok((
10217 vec![CommandOutcome {
10218 command: "(waiting for the shared build cache)".into(),
10219 code: None,
10220 output_tail: e.to_string(),
10221 duration_ms: started.elapsed().as_millis() as u64,
10222 resource_blocked: true,
10223 }],
10224 false,
10225 ));
10226 }
10227 }
10228 } else {
10229 None
10230 };
10231 let _coordination = coordinate_operation(state, worktree, wait).await?;
10232 if let Some(cache_dir) = cache_dir {
10233 let identity = crate::cache::Identity::new(worktree, head);
10234 if let Err(e) = crate::cache::ensure_fresh(
10235 &home,
10236 cache_dir,
10237 &identity,
10238 &state.config.verify.cargo_manifests(),
10239 ) {
10240 state.event(
10241 "verify",
10242 format!(
10243 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10244 worktree.display(),
10245 short(head)
10246 ),
10247 );
10248 if let Some(guard) = guard {
10249 guard.release();
10250 }
10251 return Ok((
10252 vec![CommandOutcome {
10253 command: "(confirming the shared build cache is fresh)".into(),
10254 code: None,
10255 output_tail: e.to_string(),
10256 duration_ms: started.elapsed().as_millis() as u64,
10257 resource_blocked: true,
10258 }],
10259 false,
10260 ));
10261 }
10262 }
10263 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10264 if !timed_out_pids.is_empty() {
10265 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10266 }
10267 drop(_coordination);
10270 if let Some(guard) = guard {
10271 guard.release();
10272 }
10273 Ok((outcomes, retried))
10274}
10275
10276async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10288 wait_for_pids_with(
10289 pids,
10290 crate::proc::pid_alive,
10291 LEASE_RELEASE_POLL,
10292 LEASE_RELEASE_MAX_WAIT,
10293 )
10294 .await;
10295}
10296
10297async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10303 pids: &[u32],
10304 alive: F,
10305 poll: Duration,
10306 max_wait: Duration,
10307) {
10308 let deadline = Instant::now() + max_wait;
10309 loop {
10310 if pids.iter().all(|&pid| !alive(pid)) {
10311 return;
10312 }
10313 if Instant::now() >= deadline {
10314 return;
10315 }
10316 tokio::time::sleep(poll).await;
10317 }
10318}
10319
10320fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10327 outcomes.iter().any(|o| o.resource_blocked)
10328}
10329
10330pub(crate) enum GateFix {
10332 Retry,
10334 Stop,
10337 Defer,
10340}
10341
10342fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10350 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10351 red.peek().is_some()
10352 && red.all(|o| {
10353 !o.resource_blocked
10354 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10355 && !o.output_tail.trim().is_empty()
10356 })
10357}
10358
10359fn e2e_outcome_label(o: &CommandOutcome) -> String {
10363 if o.ok() {
10364 return "pass".to_owned();
10365 }
10366 let reason = if o.build_failed() {
10367 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10368 } else {
10369 format!("FAIL ({:?})", o.code)
10370 };
10371 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10372}
10373
10374async fn run_e2e_with_retry(
10382 state: &mut RunState,
10383 shell: &[String],
10384 commands: &[String],
10385 worktree: &Path,
10386 timeout: Duration,
10387 context: &str,
10388) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10389 let (mut e2e, mut timed_out_pids) = run_commands(
10390 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10391 )
10392 .await;
10393 for o in &e2e {
10394 state.event(
10395 "verify",
10396 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10397 );
10398 }
10399 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10403 if verify_retried {
10404 state.event(
10405 "verify",
10406 format!(
10407 "{context}: verify could not build/link, not a test result — retrying once \
10408 before concluding"
10409 ),
10410 );
10411 let retried = run_commands(
10412 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10413 )
10414 .await;
10415 e2e = retried.0;
10416 timed_out_pids.extend(retried.1);
10419 for o in &e2e {
10420 state.event(
10421 "verify",
10422 format!(
10423 "{context}: retry `{}` -> {}",
10424 o.command,
10425 e2e_outcome_label(o)
10426 ),
10427 );
10428 }
10429 }
10430 (e2e, verify_retried, timed_out_pids)
10431}
10432
10433#[allow(clippy::too_many_arguments)]
10449async fn run_commands(
10450 state: &mut RunState,
10451 node: &str,
10452 task: &str,
10453 attempt: usize,
10454 shell: &[String],
10455 commands: &[String],
10456 cwd: &Path,
10457 timeout: Duration,
10458) -> (Vec<CommandOutcome>, Vec<u32>) {
10459 if commands.is_empty() {
10460 return (Vec::new(), Vec::new());
10465 }
10466 let mut out = Vec::new();
10467 let mut timed_out_pids = Vec::new();
10468 let total = commands.len();
10469 for (idx, command) in commands.iter().enumerate() {
10470 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10471 if let Err(e) = state.save() {
10472 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10473 }
10474 let started = Instant::now();
10475 let mut cmd = tokio::process::Command::new(&shell[0]);
10476 cmd.quiet();
10477 cmd.args(&shell[1..])
10478 .arg(command)
10479 .current_dir(cwd)
10480 .stdin(std::process::Stdio::null())
10481 .stdout(std::process::Stdio::piped())
10482 .stderr(std::process::Stdio::piped())
10483 .kill_on_drop(true);
10484 let mut outputs = Vec::new();
10485 let protected = (|| -> Result<()> {
10486 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10487 outputs.push(guard);
10488 }
10489 if let Some(cache) = state.config.cache_dir() {
10490 outputs.push(crate::retention::OutputLease::cache(
10491 &crate::run::home(),
10492 &cache,
10493 false,
10494 )?);
10495 }
10496 Ok(())
10497 })();
10498 if let Err(e) = protected {
10499 out.push(CommandOutcome {
10500 command: command.clone(),
10501 code: None,
10502 output_tail: format!("output ownership unavailable: {e:#}"),
10503 duration_ms: 0,
10504 resource_blocked: true,
10505 });
10506 continue;
10507 }
10508 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10509 let (code, body) = match spawned {
10510 Ok(mut running) => {
10511 let pid = running.child().id();
10512 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10513 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10514 let mut paths = vec![cwd.to_owned()];
10515 if let Some(cache) = state.config.cache_dir() {
10516 paths.push(cache);
10517 }
10518 let mut pressure = None;
10519 let status = tokio::select! {
10520 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10521 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10522 pressure = Some(reason); None
10523 }
10524 };
10525 let (code, failure) = match status {
10526 Some(Ok(Ok(status))) => (
10527 status.code(),
10528 status
10529 .code()
10530 .is_none()
10531 .then(|| format!("command terminated: {status}")),
10532 ),
10533 Some(Ok(Err(error))) => {
10534 (None, Some(format!("failed waiting for command: {error}")))
10535 }
10536 Some(Err(_)) => {
10537 if !running.coordinated()
10541 && let Some(pid) = pid
10542 {
10543 timed_out_pids.push(pid);
10544 }
10545 (
10546 None,
10547 Some(format!("timed out after {}s", timeout.as_secs())),
10548 )
10549 }
10550 None => (None, None),
10551 };
10552 running.finish().await;
10553 if let Some(reason) = pressure {
10554 state.resource_block = Some(crate::run::ResourceBlock {
10555 node: node.to_owned(),
10556 kind: "disk pressure".to_owned(),
10557 detail: reason.clone(),
10558 at: Timestamp::now(),
10559 });
10560 out.push(CommandOutcome {
10561 command: command.clone(),
10562 code: None,
10563 output_tail: reason,
10564 duration_ms: started.elapsed().as_millis() as u64,
10565 resource_blocked: true,
10566 });
10567 break;
10568 }
10569 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10570 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10571 if let Some(failure) = failure {
10572 body.push('\n');
10573 body.push_str(&failure);
10574 }
10575 (code, body)
10576 }
10577 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10578 };
10579 out.push(CommandOutcome {
10580 command: command.clone(),
10581 code,
10582 output_tail: tail(&body, OUTPUT_TAIL),
10583 duration_ms: started.elapsed().as_millis() as u64,
10584 resource_blocked: false,
10585 });
10586 }
10587 state.task_finished(task);
10588 if let Err(e) = state.save() {
10589 tracing::warn!("could not persist the end of {task}: {e:#}");
10590 }
10591 (out, timed_out_pids)
10592}
10593
10594fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10606 let repo = repo.display();
10607 match style {
10608 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10609 MergeStyle::Squash => {
10610 let subject = message
10613 .lines()
10614 .next()
10615 .unwrap_or(branch)
10616 .replace(['\\', '"', '$', '`'], "");
10617 format!(
10618 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10619 )
10620 }
10621 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10622 }
10623}
10624
10625const PR_TITLE_MAX: usize = 240;
10636
10637fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10648 let line = queue::first_line(text)?;
10649 let chars: Vec<char> = line.chars().collect();
10650 let (kept, mark) = if chars.len() <= max {
10651 (line.to_owned(), "")
10652 } else if let Some(end) = sentence_end(&chars, max) {
10653 (chars[..end].iter().collect::<String>(), "")
10654 } else {
10655 let room = max.saturating_sub(3);
10656 let cut = if chars[room].is_whitespace() {
10659 room
10660 } else {
10661 chars[..room]
10662 .iter()
10663 .rposition(|c| c.is_whitespace())
10664 .unwrap_or(room)
10665 };
10666 let head: String = chars[..cut].iter().collect();
10667 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10668 (head.to_owned(), "...")
10669 };
10670 if kept.is_empty() {
10671 return None;
10672 }
10673 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10674 return None;
10675 }
10676 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10677 return None;
10678 }
10679 Some(format!("{kept}{mark}"))
10680}
10681
10682fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10686 const MIN: usize = 20;
10687 for i in MIN..max.min(chars.len()) {
10688 if !matches!(chars[i], '.' | '!' | '?') {
10689 continue;
10690 }
10691 let Some(&next) = chars.get(i + 1) else {
10692 continue;
10693 };
10694 if !next.is_whitespace() {
10695 continue;
10696 }
10697 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10698 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10699 continue;
10700 }
10701 let word: String = chars[..i]
10702 .iter()
10703 .rev()
10704 .take_while(|c| !c.is_whitespace())
10705 .collect::<Vec<_>>()
10706 .into_iter()
10707 .rev()
10708 .collect();
10709 let word = word.to_ascii_lowercase();
10710 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10711 continue;
10712 }
10713 let end = chars[..i]
10714 .iter()
10715 .rposition(|c| !c.is_whitespace())
10716 .map_or(i, |p| p + 1);
10717 return Some(end);
10718 }
10719 None
10720}
10721
10722struct PrMessage {
10727 title: String,
10728 body: String,
10729}
10730
10731impl PrMessage {
10732 fn commit_message(&self) -> String {
10736 format!("{}\n\n{}", self.title, self.body)
10737 }
10738}
10739
10740fn title_marker(line: &str) -> Option<&str> {
10742 let line = line.trim();
10743 let head = line.get(..6)?;
10744 head.eq_ignore_ascii_case("title:")
10745 .then(|| line[6..].trim())
10746}
10747
10748fn summary_title(summary: &str) -> Option<String> {
10753 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10754 let raw = title_marker(first)?;
10755 if raw.is_empty() {
10756 return None;
10757 }
10758 let title = queue::title_from(raw, PR_TITLE_MAX);
10759 let lower = title.to_ascii_lowercase();
10760 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10761 return None;
10762 }
10763 Some(title)
10764}
10765
10766const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10768
10769pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10771
10772fn is_review_run(state: &RunState) -> bool {
10776 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10777}
10778
10779fn review_title(state: &RunState) -> Option<String> {
10784 english_subject(
10785 state.reviewed_commits.as_ref()?.first()?,
10786 state.config.graph.github_english(),
10787 )
10788}
10789
10790fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10793 let raw = raw.trim();
10794 if raw.is_empty() {
10795 return None;
10796 }
10797 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10798 return None;
10799 }
10800 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10801 return None;
10802 }
10803 let title = queue::title_from(raw, PR_TITLE_MAX);
10804 let lower = title.to_ascii_lowercase();
10805 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10806 return None;
10807 }
10808 Some(title)
10809}
10810
10811#[derive(Debug, Clone, PartialEq, Eq)]
10814struct BranchFacts {
10815 commits: Vec<(String, String)>,
10817 stat: String,
10819}
10820
10821const STAT_FILE_LINES: usize = 25;
10823const COMMITS_MAX_CHARS: usize = 20_000;
10825
10826async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10829 let commits = git::commit_log(repo, base, branch).await.ok()?;
10830 if commits.is_empty() {
10831 return None;
10832 }
10833 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10834 let lines: Vec<&str> = stat.lines().collect();
10835 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10836 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10837 let more = format!(" ... {omitted} more file(s)");
10838 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10839 kept.push(&more);
10840 kept.push(lines[lines.len() - 1]);
10841 kept.join("\n")
10842 } else {
10843 lines.join("\n")
10844 };
10845 Some(BranchFacts { commits, stat })
10846}
10847
10848fn markdown_safe(text: &str) -> String {
10851 text.replace("</details>", "</details>")
10852 .replace("\x60\x60\x60", "~~~")
10853}
10854
10855fn neutral_title(state: &RunState, winner: char) -> String {
10856 format!(
10857 "chore: land candidate {} of run {}",
10858 winner.to_ascii_uppercase(),
10859 state.id
10860 )
10861}
10862
10863pub(crate) async fn repair_landing_gate(
10866 state: &mut RunState,
10867 winner: &Candidate,
10868 outcomes: &[CommandOutcome],
10869) -> Result<GateFix> {
10870 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10871 return Ok(GateFix::Stop);
10872 }
10873 let mut runner = Runner {
10874 roles: state.config.resolve_roles()?,
10875 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10876 state: state.clone(),
10877 pause: Pause::new(),
10878 interrupt: Pause::new(),
10879 };
10880 let result = runner.gate_fix_round(winner, outcomes).await;
10881 *state = runner.state;
10882 result
10883}
10884
10885pub(crate) async fn check_gate_repair_e2e(
10888 state: &mut RunState,
10889 winner: &Candidate,
10890 wait: &crate::repo_lock::WaitPolicy,
10891) -> Result<GateFix> {
10892 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10893 return Ok(GateFix::Retry);
10894 }
10895 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10896 if state
10897 .gate_repair_e2e
10898 .as_ref()
10899 .is_some_and(|r| r.head == head && r.passed)
10900 {
10901 return Ok(GateFix::Retry);
10902 }
10903 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10904 head: head.clone(),
10905 passed: false,
10906 outcomes: Vec::new(),
10907 });
10908 state.save()?;
10909 let commands = state.config.verify.e2e.clone();
10910 let outcomes = if commands.is_empty() {
10911 Vec::new()
10912 } else {
10913 let shell = state.config.shell();
10914 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10915 let cache_dir = state.config.cache_dir();
10916 let (outcomes, _) = with_cache_lease(
10917 state,
10918 cache_dir.as_deref(),
10919 "e2e",
10920 "e2e",
10921 &winner.worktree,
10922 &head,
10923 timeout,
10924 "gate repair e2e",
10925 wait,
10926 |state, budget| async move {
10927 run_e2e_with_retry(
10928 state,
10929 &shell,
10930 &commands,
10931 &winner.worktree,
10932 budget,
10933 "gate repair e2e",
10934 )
10935 .await
10936 },
10937 )
10938 .await?;
10939 outcomes
10940 };
10941 let deferred = verify_inconclusive(&outcomes);
10942 let passed = outcomes.iter().all(CommandOutcome::ok)
10943 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10944 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10945 head,
10946 passed,
10947 outcomes,
10948 });
10949 state.save()?;
10950 if deferred {
10951 Ok(GateFix::Defer)
10952 } else if !passed {
10953 state.event(
10954 "gate",
10955 "verify.e2e failed after gate repair; nothing published",
10956 );
10957 Ok(GateFix::Stop)
10958 } else {
10959 Ok(GateFix::Retry)
10960 }
10961}
10962
10963pub(crate) async fn run_landing_gate(
10965 state: &mut RunState,
10966 winner: &Candidate,
10967 wait: &crate::repo_lock::WaitPolicy,
10968) -> Result<Vec<CommandOutcome>> {
10969 let shell = state.config.shell();
10970 let gate_commands = state.config.verify.gate.clone();
10971 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10972 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10973 Vec::new()
10974 } else {
10975 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10976 let cache_dir = state.config.cache_dir();
10977 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10978 let (outcomes, _) = with_cache_lease(
10979 state,
10980 cache_dir.as_deref(),
10981 "gate",
10982 "gate",
10983 &winner.worktree,
10984 &head,
10985 timeout,
10986 "final gate",
10987 wait,
10988 |state, budget| {
10989 let shell = shell.clone();
10990 let gate_commands = gate_commands.clone();
10991 let winner = winner.clone();
10992 async move {
10993 Runner::run_pre_gate(state, &winner).await;
10994 let (outcomes, timed_out_pids) = run_commands(
10995 state,
10996 "gate",
10997 "gate",
10998 0,
10999 &shell,
11000 &gate_commands,
11001 &winner.worktree,
11002 budget,
11003 )
11004 .await;
11005 (outcomes, false, timed_out_pids)
11006 }
11007 },
11008 )
11009 .await?;
11010 outcomes
11011 };
11012 if outcomes.is_empty() {
11013 state.event("gate", "no gate commands configured; not tested");
11018 }
11019 for o in &outcomes {
11020 state.event(
11021 "gate",
11022 format!(
11023 "`{}` -> {}",
11024 o.command,
11025 if o.ok() {
11026 "pass".to_owned()
11027 } else {
11028 format!(
11029 "FAIL ({:?})\n{}",
11030 o.code,
11031 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11032 )
11033 }
11034 ),
11035 );
11036 }
11037 Ok(outcomes)
11038}
11039
11040pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11046 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11047 ""
11048 } else {
11049 pr_title
11050 }
11051}
11052
11053pub fn landing_subject_source(state: &RunState) -> String {
11057 if is_review_run(state) {
11058 let winner = state.candidates.first().map_or('A', |c| c.label);
11059 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11060 }
11061 state.instruction.clone()
11062}
11063
11064fn summary_without_title(summary: &str) -> String {
11067 let mut lines = summary.trim().lines().peekable();
11068 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11069 lines.next();
11070 }
11071 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11072}
11073
11074#[cfg(test)]
11086fn pr_message(state: &RunState, winner: char) -> PrMessage {
11087 pr_message_with(state, winner, None)
11088}
11089
11090#[cfg(test)]
11093fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11094 let raw = pr_message_raw(
11095 state,
11096 winner,
11097 facts,
11098 crate::pr_outcome::FinalHead::Unreported,
11099 ChangeNote::NotAsked,
11100 );
11101 let id = crate::scrub::Identity::current();
11102 PrMessage {
11103 title: crate::scrub::scrub(&raw.title, &id),
11104 body: crate::scrub::scrub(&raw.body, &id),
11105 }
11106}
11107
11108#[derive(Debug, Clone, Copy)]
11110enum ChangeNote<'a> {
11111 NotAsked,
11113 Summary(&'a crate::run::ChangeSummary),
11115 Unavailable(&'a str),
11117}
11118
11119fn pr_message_raw(
11120 state: &RunState,
11121 winner: char,
11122 facts: Option<&BranchFacts>,
11123 head: crate::pr_outcome::FinalHead<'_>,
11124 change: ChangeNote<'_>,
11125) -> PrMessage {
11126 if !matches!(change, ChangeNote::NotAsked) {
11127 let (title, mut body) = match change {
11128 ChangeNote::Summary(c) => (
11129 c.title.clone(),
11130 format!("## Summary\n\n{}\n\n", c.body.trim()),
11131 ),
11132 ChangeNote::Unavailable(why) => (
11133 neutral_title(state, winner),
11134 format!(
11135 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11136 ),
11137 ),
11138 ChangeNote::NotAsked => unreachable!(),
11139 };
11140 body.push_str(&public_pr_evidence(state, head));
11141 body.push_str(&format!(
11142 "\n---\nmagi:run/{} magi:candidate-{}\n",
11143 state.id,
11144 winner.to_ascii_lowercase()
11145 ));
11146 return PrMessage { title, body };
11147 }
11148 pr_preview(state, winner, facts, head)
11149}
11150
11151fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11154 let mut body = crate::pr_outcome::review_rounds(state);
11155 body.push_str(&crate::pr_outcome::final_verification(state, head));
11156 let open = state.last_round_findings();
11157 if !open.is_empty() {
11158 body.push_str("## Open review findings\n\n");
11159 for f in open {
11160 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11161 }
11162 body.push('\n');
11163 }
11164 if let Some(c) = state.change_summary.as_ref()
11165 && head == crate::pr_outcome::FinalHead::At(&c.head)
11166 {
11167 body.push_str(&format!(
11168 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11169 c.base, c.head
11170 ));
11171 }
11172 body
11173}
11174
11175fn pr_preview(
11177 state: &RunState,
11178 winner: char,
11179 facts: Option<&BranchFacts>,
11180 head: crate::pr_outcome::FinalHead<'_>,
11181) -> PrMessage {
11182 let summary = state
11183 .candidates
11184 .iter()
11185 .find(|c| c.label == winner)
11186 .map(|c| c.summary.as_str())
11187 .unwrap_or_default();
11188 let review = is_review_run(state);
11191 let title = if review {
11192 facts
11193 .and_then(|f| {
11194 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11195 })
11196 .or_else(|| review_title(state))
11197 .or_else(|| {
11198 state
11199 .candidates
11200 .iter()
11201 .find(|c| c.label == winner)
11202 .filter(|c| !c.branch.starts_with("magi/"))
11203 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11204 })
11205 .unwrap_or_else(|| neutral_title(state, winner))
11206 } else {
11207 summary_title(summary).unwrap_or_else(|| {
11208 english_title(
11209 &state.instruction,
11210 PR_TITLE_MAX,
11211 state.config.graph.github_english(),
11212 )
11213 .unwrap_or_else(|| neutral_title(state, winner))
11214 })
11215 };
11216
11217 let mut body = String::new();
11218 let what = summary_without_title(summary);
11219 if !what.is_empty() {
11220 body.push_str("## Summary\n\n");
11221 body.push_str(&what);
11222 body.push_str("\n\n");
11223 }
11224
11225 body.push_str(&crate::pr_outcome::review_rounds(state));
11227 body.push_str(&crate::pr_outcome::final_verification(state, head));
11228 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11229
11230 let open = state.open_findings();
11231 if !open.is_empty() {
11232 body.push_str("## Open review findings\n\n");
11233 for f in &open {
11234 body.push_str(&format!(
11235 "- `{}` [{:?}] {}\n",
11236 f.id, f.severity, "Context retained in local review records."
11237 ));
11238 }
11239 body.push('\n');
11240 }
11241
11242 if let Some(fix) = fix
11243 && !fix.rejected.is_empty()
11244 {
11245 body.push_str("## Declined by the fixer\n\n");
11246 for r in &fix.rejected {
11247 body.push_str(&format!(
11248 "- `{}`: {}\n",
11249 r.id, "Reason retained in local review records."
11250 ));
11251 }
11252 body.push('\n');
11253 }
11254
11255 if review {
11256 body.push_str("## Commits under review\n\n");
11258 if let Some(facts) = facts {
11259 let mut left = COMMITS_MAX_CHARS;
11260 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11261 let mut entry = format!("- {}\n", markdown_safe(subject));
11262 for l in markdown_safe(text).lines() {
11263 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11264 }
11265 if left == 0 {
11266 body.push_str(&format!(
11267 "- ... {} more commit(s)\n",
11268 facts.commits.len() - i
11269 ));
11270 break;
11271 }
11272 if entry.len() > left {
11273 let mut end = left;
11276 while !entry.is_char_boundary(end) {
11277 end -= 1;
11278 }
11279 entry.truncate(end);
11280 entry.push_str("\n ... (truncated)\n");
11281 left = 0;
11282 } else {
11283 left -= entry.len();
11284 }
11285 body.push_str(&entry);
11286 }
11287 if !facts.stat.trim().is_empty() {
11288 body.push_str(&format!(
11289 "\n## Diff stat\n\n```\n{}\n```\n",
11290 markdown_safe(facts.stat.trim())
11291 ));
11292 }
11293 } else {
11294 match &state.reviewed_commits {
11295 Some(subjects) => {
11296 for s in subjects {
11297 body.push_str(&format!("- {}\n", s.trim()));
11298 }
11299 }
11300 None => {
11301 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11304 body.push_str(rest.trim());
11305 body.push('\n');
11306 }
11307 }
11308 }
11309 } else {
11310 let task = state.instruction.trim();
11311 let task = if task.is_empty() {
11312 "(empty task)"
11313 } else {
11314 task
11315 };
11316 body.push_str(&format!(
11317 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11318 task.replace("</details>", "</details>")
11319 ));
11320 }
11321
11322 body.push_str(&format!(
11323 "\n---\nmagi:run/{} magi:candidate-{}\n",
11324 state.id,
11325 winner.to_ascii_lowercase()
11326 ));
11327
11328 PrMessage { title, body }
11329}
11330
11331fn seeded_instruction(state: &RunState) -> String {
11335 match refs::describe(&state.seeds) {
11336 Some(facts) => format!(
11337 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11338 Candidates start from the unmerged branch named above, when there \
11339 is one, and carry any unmerged commit named by sha as a \
11340 cherry-pick. Check that this is what the task meant before \
11341 building on it.",
11342 state.instruction
11343 ),
11344 None => state.instruction.clone(),
11345 }
11346}
11347
11348async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11352 let base = &state.base_branch;
11353 let mut against = base.clone();
11354 if mode == MergeMode::Pr {
11355 let remote = &state.config.merge.remote;
11359 let tracking = format!("{remote}/{base}");
11360 let fetched = git::fetch(repo, remote, base).await;
11361 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11362 against = tracking;
11363 } else {
11364 return false;
11365 }
11366 }
11367 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11368}
11369
11370fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11373 let mut detail = format!(
11374 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11375 nothing to open a pull request for"
11376 );
11377 match refs::describe(&state.seeds) {
11378 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11379 None => detail.push_str(
11380 "\nThe task names no existing branch or commit; if it means to land work \
11381 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11382 ),
11383 }
11384 detail
11385}
11386
11387#[derive(Debug, PartialEq, Eq)]
11390enum PrPlan {
11391 Create,
11392 Adopt { url: String, title: String },
11393 Stop(String),
11394}
11395
11396fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11399 match found {
11400 Ok(land::OpenPr::None) => PrPlan::Create,
11401 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11402 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11403 "several open pull requests exist for this branch, not picking one: {}",
11404 urls.join(" ")
11405 )),
11406 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11407 }
11408}
11409
11410pub(crate) async fn gh_pr_create(
11412 cwd: &Path,
11413 base: &str,
11414 head: &str,
11415 title: &str,
11416 body: &str,
11417) -> Result<String> {
11418 let out = tokio::process::Command::new("gh")
11419 .args([
11420 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11421 ])
11422 .current_dir(cwd)
11423 .quiet()
11424 .stdin(std::process::Stdio::null())
11425 .output()
11426 .await
11427 .context("spawn gh")?;
11428 if out.status.success() {
11429 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11430 } else {
11431 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11432 }
11433}
11434
11435async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11437 match std::fs::symlink_metadata(path) {
11438 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11439 git::worktree_remove_clean(repo, path).await?;
11442 return Ok(());
11443 }
11444 Err(e) => return Err(e.into()),
11445 Ok(_) => {}
11446 }
11447 if !git::is_clean(path).await? {
11448 bail!(
11449 "worktree has uncommitted work; refusing removal: {}",
11450 path.display()
11451 );
11452 }
11453 if !git::worktree_remove_clean(repo, path).await? {
11454 bail!("worktree removal refused: {}", path.display());
11455 }
11456 Ok(())
11457}
11458
11459pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11468 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11469 bail!("refusing fold: {reason}");
11470 }
11471 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11472 bail!("candidate preservation incomplete; refusing cleanup");
11473 }
11474 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11478 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11479 let _driver = crate::driver::maintenance(home, &state.id)?;
11480 let mut output_guards = Vec::new();
11481 for entry in std::fs::read_dir(state.worktree_root())
11482 .into_iter()
11483 .flatten()
11484 .flatten()
11485 {
11486 if entry.path().join(".git").is_file() {
11487 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11488 }
11489 }
11490 let repo = state.repo.clone();
11491 let wait = repository_wait(state, None);
11494 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11495 Ok(lease) => lease,
11496 Err(e) => {
11497 state.save_under(home)?;
11498 return Err(e);
11499 }
11500 };
11501 let _source_outputs = if state.status == RunStatus::Merged
11505 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11506 {
11507 verify_merged_fold_sources(state, drop_winner, home).await?
11508 } else {
11509 Vec::new()
11510 };
11511 let root = state.worktree_root();
11512 let winner = state.tally.as_ref().map(|t| t.winner);
11513 let mut removed = Vec::new();
11514
11515 for i in 0..state.candidates.len() {
11516 let c = state.candidates[i].clone();
11517 let is_winner = Some(c.label) == winner;
11518 if is_winner && !drop_winner {
11519 continue;
11520 }
11521 let existed = c.worktree.exists();
11522 remove_preserved_worktree(&repo, &c.worktree).await?;
11525 if existed {
11526 removed.push(c.worktree.to_string_lossy().into_owned());
11527 }
11528 let handed_over = state.released_branches.contains(&c.branch);
11531 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11532 if !git::branch_delete(&repo, &c.branch).await? {
11533 bail!("candidate branch removal refused");
11534 }
11535 removed.push(c.branch.clone());
11536 }
11537 state.candidates[i].folded = true;
11538 }
11539
11540 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11541 let path = name.path();
11542 let keep = !drop_winner
11543 && winner.is_some_and(|w| {
11544 path.file_name()
11545 .is_some_and(|n| n == format!("cand-{w}").as_str())
11546 });
11547 if keep {
11548 continue;
11549 }
11550 remove_preserved_worktree(&repo, &path).await?;
11551 removed.push(path.to_string_lossy().into_owned());
11552 }
11553
11554 remove_if_empty(&root);
11563
11564 if state.enabled_worktree_config && drop_winner {
11565 git::release_worktree_config(&repo).await.ok();
11569 state.enabled_worktree_config = false;
11570 }
11571 state.save_under(home)?;
11572 Ok(removed)
11573}
11574
11575async fn verify_merged_fold_sources(
11576 state: &RunState,
11577 drop_winner: bool,
11578 home: &Path,
11579) -> Result<Vec<std::fs::File>> {
11580 let winner = state.tally.as_ref().map(|t| t.winner);
11581 let mut heads = BTreeSet::new();
11582 for candidate in &state.candidates {
11583 if Some(candidate.label) == winner && !drop_winner {
11584 continue;
11585 }
11586 match std::fs::symlink_metadata(&candidate.worktree) {
11587 Ok(_) => anyhow::ensure!(
11588 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11589 "candidate recovery source is outside the inventoried run root: {}",
11590 candidate.worktree.display()
11591 ),
11592 Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11593 Err(error) => {
11594 return Err(error).context(format!(
11595 "cannot inspect candidate recovery source: {}",
11596 candidate.worktree.display()
11597 ));
11598 }
11599 }
11600 if git::branch_exists(&state.repo, &candidate.branch).await? {
11601 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11602 heads.extend(
11603 git::git(
11604 &state.repo,
11605 &["reflog", "show", "--format=%H", &candidate.branch],
11606 )
11607 .await?
11608 .lines()
11609 .map(str::to_owned),
11610 );
11611 }
11612 }
11613 let mut cargo_guards = Vec::new();
11614 let entries = match std::fs::read_dir(state.worktree_root()) {
11615 Ok(entries) => Some(entries),
11616 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11617 Err(error) => return Err(error.into()),
11618 };
11619 for entry in entries.into_iter().flatten() {
11620 let path = entry?.path();
11621 let metadata = std::fs::symlink_metadata(&path)?;
11622 anyhow::ensure!(
11623 metadata.is_dir() && !metadata.file_type().is_symlink(),
11624 "non-directory or symlink recovery source retained: {}",
11625 path.display()
11626 );
11627 if !drop_winner
11628 && winner.is_some_and(|w| {
11629 path.file_name()
11630 .is_some_and(|name| name == format!("cand-{w}").as_str())
11631 })
11632 {
11633 continue;
11634 }
11635 anyhow::ensure!(
11636 path.join(".git").is_file(),
11637 "unregistered recovery source retained: {}",
11638 path.display()
11639 );
11640 anyhow::ensure!(
11641 git::git(
11642 &path,
11643 &[
11644 "status",
11645 "--porcelain",
11646 "--untracked-files=all",
11647 "--ignored",
11648 "--",
11649 ".",
11650 ":(exclude)target"
11651 ]
11652 )
11653 .await?
11654 .is_empty(),
11655 "source changes or ignored recovery files retained: {}",
11656 path.display()
11657 );
11658 match std::fs::symlink_metadata(path.join("target")) {
11659 Ok(_) => {
11660 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11661 }
11662 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11663 Err(error) => return Err(error.into()),
11664 }
11665 for checkpoint in [
11666 "rebase-merge",
11667 "rebase-apply",
11668 "sequencer",
11669 "CHERRY_PICK_HEAD",
11670 "MERGE_HEAD",
11671 "REBASE_HEAD",
11672 "REVERT_HEAD",
11673 "BISECT_LOG",
11674 "AUTO_MERGE",
11675 "MERGE_MSG",
11676 ] {
11677 let location = git::git(
11678 &path,
11679 &[
11680 "rev-parse",
11681 "--path-format=absolute",
11682 "--git-path",
11683 checkpoint,
11684 ],
11685 )
11686 .await?;
11687 anyhow::ensure!(
11688 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11689 if e.kind() == std::io::ErrorKind::NotFound),
11690 "Git recovery checkpoint retained: {checkpoint}"
11691 );
11692 }
11693 heads.insert(git::rev_parse(&path, "HEAD").await?);
11694 heads.extend(
11695 git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11696 .await?
11697 .lines()
11698 .map(str::to_owned),
11699 );
11700 if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11701 heads.insert(original);
11702 }
11703 }
11704 if !heads.is_empty() {
11705 let reference = format!(
11709 "refs/remotes/{}/{}",
11710 state.config.merge.remote, state.base_branch
11711 );
11712 let base = git::rev_parse(&state.repo, &reference)
11713 .await
11714 .context("cannot prove merged sources against the fetched base")?;
11715 let mut squash_winner = None;
11716 let mut checked_squash = false;
11717 for head in heads {
11718 if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11719 .await?
11720 .ok()
11721 {
11722 continue;
11723 }
11724 if !checked_squash {
11725 squash_winner = proven_squash_winner(state, &base).await?;
11726 checked_squash = true;
11727 }
11728 let incorporated = if let Some(winner) = &squash_winner {
11729 git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11730 .await?
11731 .ok()
11732 } else {
11733 false
11734 };
11735 anyhow::ensure!(
11736 incorporated,
11737 "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11738 );
11739 }
11740 }
11741 Ok(cargo_guards)
11742}
11743
11744async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11750 let Some(winner) = state.winner() else {
11751 return Ok(None);
11752 };
11753 let head = git::rev_parse(&state.repo, &winner.branch).await?;
11754 if !git::git_raw(
11755 &state.repo,
11756 &["merge-base", "--is-ancestor", &state.base_commit, &head],
11757 )
11758 .await?
11759 .ok()
11760 {
11761 return Ok(None);
11762 }
11763 let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11764 if patch.is_empty() {
11765 return Ok(None);
11766 }
11767 let range = format!("{}..{base}", state.base_commit);
11768 let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11769 for commit in commits.lines() {
11770 let parent = format!("{commit}^");
11771 if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11772 return Ok(Some(head));
11773 }
11774 }
11775 Ok(None)
11776}
11777
11778async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11779 git::git(
11780 repo,
11781 &[
11782 "-c",
11783 "core.quotePath=true",
11784 "diff",
11785 "--binary",
11786 "--full-index",
11787 "--no-renames",
11788 "--no-ext-diff",
11789 "--no-textconv",
11790 "--no-color",
11791 "--ignore-submodules=none",
11792 "--submodule=short",
11793 from,
11794 to,
11795 "--",
11796 ],
11797 )
11798 .await
11799}
11800
11801fn remove_if_empty(dir: &Path) {
11812 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11813 std::fs::remove_dir(dir).ok();
11814 }
11815}
11816
11817pub fn worst_open(state: &RunState) -> Option<Severity> {
11819 state
11820 .reviews
11821 .last()?
11822 .reviews
11823 .iter()
11824 .flat_map(|r| r.findings.iter())
11825 .map(|f| f.severity)
11826 .max()
11827}
11828
11829async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11833 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11834 git::acquire_worktree_config(repo).await?;
11835 state.enabled_worktree_config = true;
11836 }
11837 Ok(())
11838}
11839
11840#[cfg(test)]
11841mod tests {
11842 #[tokio::test]
11843 async fn launcher_base_fetch_requires_remote_success() {
11844 let tmp = tempfile::tempdir().unwrap();
11845 let repo = tmp.path().join("repo");
11846 let remote = tmp.path().join("remote");
11847 std::fs::create_dir(&repo).unwrap();
11848 std::fs::create_dir(&remote).unwrap();
11849 git::git(&remote, &["init", "--bare", "-b", "main"])
11850 .await
11851 .unwrap();
11852 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11853 git::git(
11854 &repo,
11855 &[
11856 "-c",
11857 "user.name=Fixture",
11858 "-c",
11859 "user.email=fixture@example.invalid",
11860 "commit",
11861 "--allow-empty",
11862 "-m",
11863 "fixture",
11864 ],
11865 )
11866 .await
11867 .unwrap();
11868 git::git(
11869 &repo,
11870 &["remote", "add", "origin", remote.to_str().unwrap()],
11871 )
11872 .await
11873 .unwrap();
11874 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11875 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11876 assert_eq!(
11877 resolve_base(&repo, "main", "origin", None)
11878 .await
11879 .unwrap()
11880 .commit,
11881 head
11882 );
11883 #[cfg(unix)]
11884 {
11885 let policy: crate::config::Coordination = toml::from_str(
11886 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11887 ).unwrap();
11888 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11889 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11890 let held = crate::repo_lock::SeatLease::operation_with(
11891 &repo,
11892 Some(&policy),
11893 &crate::repo_lock::WaitPolicy::default(),
11894 )
11895 .await
11896 .unwrap();
11897 assert!(held.is_some());
11898 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11899 tokio::pin!(fetch);
11900 assert!(
11901 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11902 .await
11903 .is_err()
11904 );
11905 assert!(
11906 !repo.join(".git/FETCH_HEAD").exists(),
11907 "no fetch before admission"
11908 );
11909 drop(held);
11910 let fetched = fetch.await.unwrap();
11911 assert_eq!(fetched.commit, head);
11912 assert_eq!(fetched.remote, "origin");
11913 assert_eq!(fetched.branch, "main");
11914 }
11915 git::git(
11917 &repo,
11918 &[
11919 "remote",
11920 "set-url",
11921 "origin",
11922 tmp.path().join("missing").to_str().unwrap(),
11923 ],
11924 )
11925 .await
11926 .unwrap();
11927 let error = resolve_base(&repo, "main", "origin", None)
11928 .await
11929 .unwrap_err()
11930 .to_string();
11931 assert!(error.contains("cannot read origin/main"));
11932 assert!(error.contains("refusing to branch off"));
11933 }
11934
11935 #[test]
11936 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11937 let leaked = vec!["chore(deps): update a crate".to_owned()];
11938 let own = "fix(daemon): apply a chosen action";
11939 assert!(should_retitle("", own, &leaked));
11940 assert!(should_retitle(
11941 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11942 own,
11943 &leaked
11944 ));
11945 assert!(should_retitle(
11946 "chore: land candidate A of run 1",
11947 own,
11948 &leaked
11949 ));
11950 assert!(should_retitle(
11951 "magi: candidate A (uncommitted work)",
11952 own,
11953 &leaked
11954 ));
11955 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11956 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11957 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11958 }
11959
11960 #[test]
11961 fn pr_merge_plan_creates_adopts_or_stops() {
11962 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11963 assert_eq!(
11964 pr_merge_plan(Ok(land::OpenPr::One {
11965 url: "u".into(),
11966 title: "t".into()
11967 })),
11968 PrPlan::Adopt {
11969 url: "u".into(),
11970 title: "t".into()
11971 }
11972 );
11973 let PrPlan::Stop(many) =
11974 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11975 else {
11976 panic!("many must stop");
11977 };
11978 assert!(many.contains('a') && many.contains('b'));
11979 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11980 panic!("a failed lookup must stop");
11981 };
11982 assert!(err.contains("bad token"));
11983 }
11984
11985 use super::*;
11986 use crate::run::GateStatus;
11987 use std::collections::BTreeMap;
11988 use std::time::Duration;
11989
11990 fn conductor() -> AgentSpec {
11991 AgentSpec {
11992 id: "conductor".to_owned(),
11993 kind: crate::config::AgentKind::Command,
11994 model: None,
11995 command: vec!["true".to_owned()],
11996 extra_args: Vec::new(),
11997 env: BTreeMap::new(),
11998 prompt_delivery: None,
11999 }
12000 }
12001
12002 fn spec(id: &str) -> AgentSpec {
12003 AgentSpec {
12004 id: id.to_owned(),
12005 kind: crate::config::AgentKind::Command,
12006 model: None,
12007 command: vec!["true".to_owned()],
12008 extra_args: Vec::new(),
12009 env: BTreeMap::new(),
12010 prompt_delivery: None,
12011 }
12012 }
12013
12014 fn ids(xs: &[&str]) -> BTreeSet<String> {
12015 xs.iter().map(|s| (*s).to_owned()).collect()
12016 }
12017
12018 #[test]
12019 fn pick_successor_skips_an_agent_another_seat_holds() {
12020 let roster = [spec("a"), spec("b"), spec("c")];
12022 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12023 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12024 }
12025
12026 #[test]
12027 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12028 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12030 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12031 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12032 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12033 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12034 }
12035
12036 #[test]
12037 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12038 let roster = [spec("a"), spec("b")];
12039 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12040 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12041 let carried = ids(&["b"]);
12042 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12043 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12044 }
12045
12046 #[test]
12047 fn pick_successor_returns_none_without_any_untried_successor() {
12048 let roster = [spec("a"), spec("b")];
12049 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12050 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12051 let carried = ids(&["a"]);
12052 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12053 }
12054
12055 #[test]
12056 fn pick_successor_rescue_avoids_another_seats_occupant() {
12057 let roster = [spec("a"), spec("b"), spec("c")];
12059 let carried = ids(&["b", "c"]);
12060 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12061 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12062 }
12063
12064 #[test]
12065 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12066 let roster = [spec("a"), spec("b"), spec("c")];
12067 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12068 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12069 }
12070
12071 #[test]
12072 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12073 let roster = [spec("a"), spec("b"), spec("c")];
12074 let failed = ids(&["a", "b", "c"]);
12075 let mut tried = ids(&["b"]);
12077 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12078 assert_eq!(first.id, "a");
12079 tried.insert(first.id.clone());
12080 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12081 assert_eq!(second.id, "c");
12082 tried.insert(second.id.clone());
12083 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12084 }
12085
12086 #[test]
12087 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12088 let roster = [spec("a"), spec("b")];
12089 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12090 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12091 }
12092
12093 #[test]
12094 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12095 let roster = [spec("a"), spec("b"), spec("c")];
12096 let h = SeatHistory {
12097 failed: ids(&["a"]),
12098 last_ok: Some("b".to_owned()),
12099 last_fail: None,
12100 };
12101 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12102 let gone = SeatHistory {
12104 last_ok: Some("zzz".to_owned()),
12105 ..h.clone()
12106 };
12107 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12108 let failed = SeatHistory {
12109 failed: ids(&["a", "b"]),
12110 last_ok: Some("b".to_owned()),
12111 last_fail: None,
12112 };
12113 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12114 }
12115
12116 #[test]
12117 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12118 let first = SeatState::new("review-1", "alpha", 7);
12119 let next = handover_seat("review-1", "gamma", 7);
12120 assert_ne!(first.claude_session, next.claude_session);
12121 }
12122
12123 #[test]
12124 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12125 let mut state = state_with_summary("x", "y");
12126 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12127 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12128 assert_ne!(a.claude_session, b.claude_session);
12129 }
12130
12131 #[test]
12132 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12133 let roster = [spec("a"), spec("b")];
12134 let h = SeatHistory {
12135 failed: ids(&["a", "b"]),
12136 ..SeatHistory::default()
12137 };
12138 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12139 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12140 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12141 }
12142
12143 #[test]
12150 fn failure_signature_ignores_numbers_and_paths() {
12151 assert_eq!(
12152 failure_signature("exited with Some(2) and no usable output"),
12153 failure_signature("exited with Some(137) and no usable output")
12154 );
12155 assert_eq!(
12156 failure_signature("process exited with no usable output (exit code 2)"),
12157 failure_signature("process exited with no usable output (exit code 137)")
12158 );
12159 for (old, new) in [
12160 (
12161 "exited with Some(2) and no usable output",
12162 "process exited with no usable output (exit code 2)",
12163 ),
12164 (
12165 "exited with None and no usable output",
12166 "process exited with no usable output (exit code unavailable)",
12167 ),
12168 ] {
12169 let previous = FailClass::Other(failure_signature(old));
12170 let current = FailClass::Other(failure_signature(new));
12171 assert_eq!(previous, current, "resumption retains the failure class");
12172 assert!(!should_hand_over(Some(&previous), ¤t));
12173 }
12174 assert_eq!(
12175 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12176 failure_signature("cannot open /var/x.txt: denied")
12177 );
12178 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12179 }
12180
12181 #[test]
12182 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12183 let other = FailClass::Other("x".into());
12184 assert!(should_hand_over(None, &FailClass::Quota));
12185 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12186 assert!(should_hand_over(
12187 Some(&FailClass::Timeout),
12188 &FailClass::Timeout
12189 ));
12190 assert!(should_hand_over(None, &other));
12191 assert!(!should_hand_over(Some(&other), &other));
12192 assert!(should_hand_over(
12193 Some(&other),
12194 &FailClass::Other("y".into())
12195 ));
12196 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12198 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12199 }
12200
12201 #[test]
12202 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12203 let a = SeatState::new("judge-1", "alpha", 7);
12204 let b = handover_seat("judge-1", "beta", 7);
12205 assert_ne!(a.claude_session, b.claude_session);
12206 assert_eq!(b.turns, 0);
12207 }
12208
12209 #[test]
12210 fn a_timeout_is_classified_apart_from_other_failures() {
12211 assert_eq!(
12212 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12213 Some(FailClass::Timeout)
12214 );
12215 assert!(matches!(
12216 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12217 Some(FailClass::Other(_))
12218 ));
12219 }
12220
12221 #[test]
12222 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12223 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12224 let tried = BTreeSet::from(["beta".to_owned()]);
12225 let next = next_untried_in_roster(&roster, 1, &tried);
12228 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12229 }
12230
12231 #[test]
12232 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12233 let roster = vec![spec("alpha"), spec("beta")];
12234 let tried = BTreeSet::from(["beta".to_owned()]);
12235 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12239 }
12240
12241 #[test]
12242 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12243 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12244 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12245 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12249 }
12250
12251 #[test]
12252 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12253 let roster = vec![spec("a"), spec("a"), spec("b")];
12254 let tried = BTreeSet::from(["a".to_owned()]);
12255 let next = next_untried_in_roster(&roster, 0, &tried);
12256 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12257 }
12258
12259 #[test]
12260 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12261 let roster = vec![spec("a"), spec("b")];
12262 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12263 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12264 }
12265
12266 #[test]
12267 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12268 let dir = tempfile::tempdir().unwrap();
12269 let bay = dir.path().join("ffff");
12270
12271 remove_if_empty(&bay);
12273 assert!(!bay.exists());
12274
12275 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12278 remove_if_empty(&bay);
12279 assert!(bay.exists(), "non-empty directory must survive");
12280
12281 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12283 remove_if_empty(&bay);
12284 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12285 }
12286
12287 #[test]
12296 fn a_full_panel_that_found_nothing_is_clean() {
12297 assert!(round_is_clean(
12298 0,
12299 true,
12300 2,
12301 2,
12302 0,
12303 IncompleteReviewPolicy::Block
12304 ));
12305 }
12306
12307 #[test]
12308 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12309 assert!(!round_is_clean(
12310 0,
12311 true,
12312 1,
12313 2,
12314 0,
12315 IncompleteReviewPolicy::Block
12316 ));
12317 }
12318
12319 #[test]
12320 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12321 assert!(!round_is_clean(
12322 1,
12323 true,
12324 1,
12325 2,
12326 0,
12327 IncompleteReviewPolicy::Warn
12328 ));
12329 }
12330
12331 #[test]
12332 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12333 assert!(round_is_clean(
12334 0,
12335 true,
12336 1,
12337 2,
12338 0,
12339 IncompleteReviewPolicy::Warn
12340 ));
12341 }
12342
12343 #[test]
12344 fn a_full_panel_with_an_open_finding_is_not_clean() {
12345 assert!(!round_is_clean(
12346 1,
12347 true,
12348 2,
12349 2,
12350 0,
12351 IncompleteReviewPolicy::Block
12352 ));
12353 }
12354
12355 #[test]
12356 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12357 assert!(!round_is_clean(
12358 0,
12359 false,
12360 2,
12361 2,
12362 0,
12363 IncompleteReviewPolicy::Block
12364 ));
12365 }
12366
12367 #[test]
12374 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12375 assert!(round_is_clean(
12378 0,
12379 true,
12380 1,
12381 2,
12382 1,
12383 IncompleteReviewPolicy::Block
12384 ));
12385 }
12386
12387 #[test]
12388 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12389 assert!(!round_is_clean(
12392 0,
12393 true,
12394 1,
12395 2,
12396 0,
12397 IncompleteReviewPolicy::Block
12398 ));
12399 }
12400
12401 #[test]
12402 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12403 assert!(!round_is_clean(
12404 1,
12405 true,
12406 1,
12407 2,
12408 1,
12409 IncompleteReviewPolicy::Block
12410 ));
12411 assert!(!round_is_clean(
12412 0,
12413 false,
12414 1,
12415 2,
12416 1,
12417 IncompleteReviewPolicy::Block
12418 ));
12419 }
12420
12421 #[test]
12422 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12423 assert!(!round_is_clean(
12427 0,
12428 true,
12429 0,
12430 2,
12431 2,
12432 IncompleteReviewPolicy::Block
12433 ));
12434 }
12435
12436 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12437 CommandOutcome {
12438 command: "test".to_owned(),
12439 code,
12440 output_tail: String::new(),
12441 duration_ms: 0,
12442 resource_blocked,
12443 }
12444 }
12445
12446 #[test]
12447 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12448 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12449 assert!(
12450 !verify_inconclusive(&[outcome(Some(1), false)]),
12451 "an ordinary failure is still evidence about the patch"
12452 );
12453 assert!(verify_inconclusive(&[outcome(None, true)]));
12454 assert!(
12455 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12456 "one inconclusive outcome taints the whole batch"
12457 );
12458 assert!(!verify_inconclusive(&[]));
12459 }
12460
12461 #[tokio::test]
12462 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12463 let calls = std::sync::atomic::AtomicUsize::new(0);
12467 let started = Instant::now();
12468 wait_for_pids_with(
12469 &[123],
12470 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12471 Duration::from_millis(5),
12472 Duration::from_secs(5),
12473 )
12474 .await;
12475 assert!(
12476 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12477 "must keep checking rather than deciding on the first answer"
12478 );
12479 assert!(
12480 started.elapsed() < Duration::from_secs(1),
12481 "must return the moment it is confirmed dead, not wait out the ceiling"
12482 );
12483 }
12484
12485 #[tokio::test]
12486 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12487 let started = Instant::now();
12488 wait_for_pids_with(
12489 &[123],
12490 |_| true, Duration::from_millis(5),
12492 Duration::from_millis(30),
12493 )
12494 .await;
12495 let elapsed = started.elapsed();
12496 assert!(
12497 elapsed >= Duration::from_millis(30),
12498 "must not give up before its own ceiling: {elapsed:?}"
12499 );
12500 assert!(
12501 elapsed < Duration::from_secs(1),
12502 "must not wait past its own ceiling either: {elapsed:?}"
12503 );
12504 }
12505
12506 #[tokio::test]
12507 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12508 let started = Instant::now();
12509 wait_for_pids_with(
12510 &[],
12511 |_| true,
12512 Duration::from_secs(5),
12513 Duration::from_secs(5),
12514 )
12515 .await;
12516 assert!(
12517 started.elapsed() < Duration::from_millis(200),
12518 "an empty pid list has nothing to confirm"
12519 );
12520 }
12521
12522 fn review_round(
12528 clean: bool,
12529 blocking: usize,
12530 answered: usize,
12531 expected: usize,
12532 progressed: bool,
12533 e2e_ok: bool,
12534 ) -> ReviewRound {
12535 ReviewRound {
12536 round: 1,
12537 head: "h".to_owned(),
12538 verified_head: None,
12539 verified_at: None,
12540 reviews: Vec::new(),
12541 e2e: vec![CommandOutcome {
12542 command: "test".to_owned(),
12543 code: Some(if e2e_ok { 0 } else { 1 }),
12544 output_tail: String::new(),
12545 duration_ms: 0,
12546 resource_blocked: false,
12547 }],
12548 verify_retried: false,
12549 e2e_deferred: false,
12550 e2e_defer_reason: None,
12551 fix: None,
12552 blocking,
12553 answered,
12554 expected,
12555 clean,
12556 progressed,
12557 vote_split: false,
12558 reconsideration: Vec::new(),
12559 verdict: None,
12560 }
12561 }
12562
12563 #[test]
12564 fn review_conclusion_is_none_when_nothing_has_run() {
12565 assert_eq!(review_conclusion(&[], 3), None);
12566 }
12567
12568 #[test]
12569 fn review_conclusion_is_none_while_rounds_remain() {
12570 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12571 assert_eq!(review_conclusion(&rounds, 3), None);
12572 }
12573
12574 #[test]
12575 fn review_conclusion_is_gating_once_a_round_is_clean() {
12576 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12577 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12578 }
12579
12580 #[test]
12581 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12582 let rounds = vec![
12583 review_round(false, 1, 2, 2, true, true),
12584 review_round(false, 1, 2, 2, true, true),
12585 ];
12586 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12587 }
12588
12589 #[test]
12590 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12591 let rounds = vec![
12592 review_round(false, 1, 2, 2, true, true),
12593 review_round(false, 1, 2, 2, true, false),
12594 ];
12595 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12596 }
12597
12598 #[test]
12599 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12600 let mut blocked = review_round(false, 1, 2, 2, true, false);
12607 blocked.e2e[0].resource_blocked = true;
12608 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12609 assert_eq!(review_conclusion(&rounds, 2), None);
12610 }
12611
12612 #[test]
12613 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12614 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12616 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12617 }
12618
12619 #[test]
12620 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12621 let rounds = vec![
12622 review_round(false, 1, 2, 2, false, true),
12623 review_round(false, 1, 2, 2, false, true),
12624 ];
12625 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12626 }
12627
12628 fn secs(n: u64) -> Duration {
12629 Duration::from_secs(n)
12630 }
12631
12632 #[tokio::test]
12635 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12636 ask_test_home();
12637 let tmp = tempfile::tempdir().unwrap();
12638 let repo = tmp.path().join("repo");
12639 std::fs::create_dir_all(&repo).unwrap();
12640 init_repo(&repo);
12641 let g = |args: &[&str]| {
12642 let out = std::process::Command::new("git")
12643 .args(args)
12644 .current_dir(&repo)
12645 .quiet()
12646 .output()
12647 .unwrap();
12648 assert!(
12649 out.status.success(),
12650 "{}",
12651 String::from_utf8_lossy(&out.stderr)
12652 );
12653 };
12654 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12655 g(&["checkout", "-b", "historical"]);
12656 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12657 g(&["add", "."]);
12658 g(&["commit", "-m", "old alternative"]);
12659 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12660 g(&["checkout", "-b", "magi/task-selected", &base]);
12661 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12662 g(&["add", "."]);
12663 g(&["commit", "-m", "measurement infrastructure"]);
12664 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12665 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12666 g(&["add", "."]);
12667 g(&["commit", "-m", "dependent implementation"]);
12668 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12669 g(&["checkout", "main"]);
12670 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12671 g(&["add", "."]);
12672 g(&["commit", "-m", "unrelated base change"]);
12673 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12674 let origin = tmp.path().join("origin.git");
12675 g(&[
12676 "clone",
12677 "--bare",
12678 repo.to_str().unwrap(),
12679 origin.to_str().unwrap(),
12680 ]);
12681 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12682 g(&["fetch", "origin"]);
12683 let selection = crate::pr_recovery::Selection {
12684 url: "https://example.test/project/pull/1".into(),
12685 branch: "magi/task-selected".into(),
12686 head: head.clone(),
12687 source_head: None,
12688 source_run: None,
12689 source_proof: None,
12690 disposition: Some(crate::pr_recovery::Disposition::Review),
12691 instruction: "preserve measured behavior".into(),
12692 approved_at: jiff::Timestamp::now(),
12693 };
12694 let mut template = runner_at(RunStatus::Reviewing);
12695 template.state.repo = repo.clone();
12696 template.state.base_commit = current.clone();
12697 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12698 template.state.origin = Some(crate::run::Origin::queue("selected"));
12699 template.state.pr_head = Some(selection.branch.clone());
12700 template.state.pr_recovery = Some(selection.clone());
12701 let mut runner =
12702 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12703 .await
12704 .unwrap();
12705 let wt = runner.state.winner().unwrap().worktree.clone();
12706 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12707 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12708 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12709 assert!(runner.state.seeds.is_empty());
12710 assert_eq!(runner.state.instruction, "preserve measured behavior");
12711 runner.sync_to_base().await.unwrap();
12712 assert_eq!(
12713 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12714 "prerequisite"
12715 );
12716 assert_eq!(
12717 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12718 "measured implementation"
12719 );
12720 assert_eq!(
12721 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12722 "current main change"
12723 );
12724 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12725 assert_eq!(
12726 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12727 head
12728 );
12729
12730 let mut partial = runner_at(RunStatus::Reviewing);
12732 partial.state.repo = repo.clone();
12733 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12734 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12735 run: Some("exit 1".into()),
12736 ..Default::default()
12737 }];
12738 let selection = runner.state.pr_recovery.clone().unwrap();
12739 partial.state.pr_recovery = Some(selection.clone());
12740 partial.state.pr_head = Some(selection.branch.clone());
12741 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12742 let id = partial.state.id.clone();
12743 let error = match Runner::open_selected(
12744 &repo,
12745 partial.state,
12746 partial.roles,
12747 1,
12748 runner.state.base_commit.clone(),
12749 selection,
12750 )
12751 .await
12752 {
12753 Ok(_) => panic!("setup must fail"),
12754 Err(e) => e,
12755 };
12756 assert_eq!(
12757 error
12758 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12759 .unwrap()
12760 .run,
12761 id
12762 );
12763 let failed = RunState::load(&id).unwrap();
12764 assert_eq!(failed.status, RunStatus::Blocked);
12765 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12766 assert!(Runner::resume(&id).is_err());
12767 runner.state.status = RunStatus::Ready;
12768 runner.state.driver_exited = true;
12769 runner.state.save().unwrap();
12770 let mut retry = runner_at(RunStatus::Reviewing);
12771 retry.state.repo = repo.clone();
12772 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12773 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12774 reselected.approved_at = jiff::Timestamp::now();
12775 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12776 retry.state.pr_head = Some(reselected.branch.clone());
12777 retry.state.pr_recovery = Some(reselected.clone());
12778 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12779 .await
12780 .unwrap();
12781 let restarted = Runner::open_selected(
12782 &repo,
12783 retry.state,
12784 retry.roles,
12785 1,
12786 runner.state.base_commit.clone(),
12787 reselected,
12788 )
12789 .await
12790 .unwrap();
12791 assert_eq!(
12792 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12793 .await
12794 .unwrap(),
12795 head
12796 );
12797 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12798 assert_eq!(
12799 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12800 head
12801 );
12802 }
12803
12804 #[tokio::test]
12805 #[cfg(unix)]
12806 async fn fallback_and_interrupted_restoration_require_coordination() {
12807 use std::os::fd::AsRawFd as _;
12808 ask_test_home();
12809 let tmp = tempfile::tempdir().unwrap();
12810 let mut runner = registered_runner(tmp.path()).await;
12811 runner.state.config.graph.timeout_resource_wait = 1;
12812 let winner = runner.state.candidates[0].clone();
12813 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12814 let leftover = winner.worktree.join("leftover");
12815 std::fs::write(&leftover, "untrusted edits").unwrap();
12816 let next = tmp.path().join("next-fixer");
12817 let mut failed = spec("first");
12818 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12819 let mut successor = spec("second");
12820 successor.command = vec![
12821 "sh".into(),
12822 "-c".into(),
12823 format!("touch '{}'; echo done", next.display()),
12824 ];
12825 let attempts = vec![
12826 (failed.clone(), "first".into()),
12827 (successor.clone(), "second".into()),
12828 ];
12829 let lock_path = runner.state.repo.join(".git/updater.lock");
12830 let external = std::fs::File::create(&lock_path).unwrap();
12831 assert_eq!(
12832 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12833 0
12834 );
12835 let artifacts = tmp.path().join("artifacts");
12836 let build = |spec, seat| SeatJob {
12837 spec,
12838 seat,
12839 cwd: winner.worktree.clone(),
12840 prompt: "fixture".into(),
12841 timeout: Duration::from_secs(10),
12842 allow_write: true,
12843 sessions: false,
12844 artifacts: artifacts.clone(),
12845 stem: "fix".into(),
12846 handover: None,
12847 };
12848 let (_, _, out) = runner
12849 .ask_fixer_among(
12850 attempts.clone(),
12851 "graft",
12852 None,
12853 Some((&winner.worktree, &head)),
12854 build,
12855 )
12856 .await;
12857 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12858 assert!(leftover.exists(), "contention must prevent clean");
12859 assert!(!next.exists());
12860 assert!(runner.state.handovers.is_empty());
12861 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12863 from_run: "earlier".into(),
12864 generation: 1,
12865 mode: crate::recovery::Mode::VerifyRepair,
12866 reasons: Vec::new(),
12867 reviewed_head: None,
12868 rounds: Vec::new(),
12869 open_findings: Vec::new(),
12870 evidence: None,
12871 criteria_inherited: true,
12872 repair: crate::recovery::Repair::Started {
12873 before: head.clone(),
12874 },
12875 });
12876 assert!(
12877 runner
12878 .recovery_repair(&winner)
12879 .await
12880 .unwrap_err()
12881 .is::<crate::repo_lock::ResourceBlocked>()
12882 );
12883 assert!(matches!(
12884 runner.state.recovery.as_ref().unwrap().repair,
12885 crate::recovery::Repair::Started { .. }
12886 ));
12887 assert!(leftover.exists());
12888 drop(external);
12889 runner.recovery_repair(&winner).await.unwrap();
12890 assert!(!leftover.exists());
12891 assert!(matches!(
12892 runner.state.recovery.as_ref().unwrap().repair,
12893 crate::recovery::Repair::Done { .. }
12894 ));
12895 std::fs::remove_file(&lock_path).unwrap();
12897 std::fs::create_dir(&lock_path).unwrap();
12898 std::fs::write(&leftover, "untrusted edits").unwrap();
12899 let (_, _, out) = runner
12900 .ask_fixer_among(
12901 attempts,
12902 "graft",
12903 None,
12904 Some((&winner.worktree, &head)),
12905 build,
12906 )
12907 .await;
12908 assert!(matches!(out, AgentOutcome::Failed(_)));
12909 assert!(leftover.exists());
12910 assert!(!next.exists());
12911 assert_eq!(
12912 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12913 head
12914 );
12915 }
12916
12917 #[cfg(unix)]
12918 async fn registered_runner(root: &Path) -> Runner {
12919 ask_test_home();
12920 let repo = root.join("repo");
12921 std::fs::create_dir_all(&repo).unwrap();
12922 init_repo(&repo);
12923 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12924 let mut runner = runner_at(RunStatus::Prep);
12925 runner.state.repo = repo.clone();
12926 runner.state.config.coordination = crate::config::test_coordination();
12927 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12928 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12929 runner.state.config.graph.advise = false;
12930 runner.roles.implementers = vec![spec("implement")];
12931 runner.roles.judges = vec![spec("judge")];
12932 runner.prep().await.unwrap();
12933 runner
12934 }
12935
12936 #[tokio::test]
12937 #[cfg(unix)]
12938 async fn parked_implementer_persists_completed_work() {
12939 for registered in [false, true] {
12940 let tmp = tempfile::tempdir().unwrap();
12941 let mut runner = registered_runner(tmp.path()).await;
12942 if !registered {
12943 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12944 }
12945 let marker = tmp.path().join("started");
12946 let release = tmp.path().join("release");
12947 let mut implementer = spec("implement");
12948 implementer.command = vec![
12949 "sh".into(),
12950 "-c".into(),
12951 format!(
12952 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12953 marker.display(),
12954 release.display()
12955 ),
12956 ];
12957 runner.roles.implementers = vec![implementer.clone()];
12958 runner.state.config.agents = vec![implementer];
12959 let pause = runner.pause.clone();
12960 let (result, ()) = tokio::join!(runner.implement(), async {
12961 tokio::time::timeout(Duration::from_secs(10), async {
12962 while !marker.exists() {
12963 tokio::time::sleep(Duration::from_millis(10)).await;
12964 }
12965 })
12966 .await
12967 .unwrap();
12968 pause.park();
12969 std::fs::write(&release, "go").unwrap();
12970 });
12971 result.unwrap();
12972 let candidate = &runner.state.candidates[0];
12973 assert!(candidate.commits > 0, "{candidate:?}");
12974 assert!(candidate.summary.contains("saved-result"));
12975 assert_eq!(candidate.files, 1);
12976 let before = candidate.commits;
12977 runner.implement().await.unwrap();
12978 assert_eq!(runner.state.candidates[0].commits, before);
12979 }
12980 }
12981
12982 #[tokio::test]
12983 #[cfg(unix)]
12984 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12985 let tmp = tempfile::tempdir().unwrap();
12986 let mut runner = registered_runner(tmp.path()).await;
12987 let winner = runner.state.candidates[0].clone();
12988 let cache = tmp.path().join("cache");
12989 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12990 runner.state.config.graph.timeout_resource_wait = 0;
12991 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12992 let crate::cache::AcquireOutcome::Acquired(held) =
12993 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12994 else {
12995 panic!("cache must be free")
12996 };
12997 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12998 from_run: "earlier".into(),
12999 generation: 1,
13000 mode: crate::recovery::Mode::VerifyRepair,
13001 reasons: Vec::new(),
13002 reviewed_head: None,
13003 rounds: Vec::new(),
13004 open_findings: Vec::new(),
13005 evidence: None,
13006 criteria_inherited: true,
13007 repair: crate::recovery::Repair::NotStarted,
13008 });
13009 runner.recovery_repair(&winner).await.unwrap();
13010 assert!(matches!(
13011 runner.state.recovery.as_ref().unwrap().repair,
13012 crate::recovery::Repair::WaitingForRestore { .. }
13013 ));
13014 drop(held);
13015 runner.state.resource_block = None;
13016 runner.state.config.graph.timeout_resource_wait = 1;
13017 runner.recovery_repair(&winner).await.unwrap();
13018 assert!(matches!(
13019 runner.state.recovery.as_ref().unwrap().repair,
13020 crate::recovery::Repair::Done { .. }
13021 ));
13022 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13023 }
13024
13025 #[tokio::test]
13026 #[cfg(unix)]
13027 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13028 use std::os::fd::AsRawFd as _;
13029 for graft in [false, true] {
13030 let tmp = tempfile::tempdir().unwrap();
13031 let mut runner = registered_runner(tmp.path()).await;
13032 runner.state.config.graph.timeout_resource_wait = 1;
13033 let winner = runner.state.candidates[0].clone();
13034 let marker = tmp.path().join("fixer-started");
13035 let mut fixer = spec("fixer");
13036 fixer.command = vec![
13037 "sh".into(),
13038 "-c".into(),
13039 format!("echo started >> '{}'; echo done", marker.display()),
13040 ];
13041 runner.roles.fixer = Some(vec![fixer]);
13042 if graft {
13043 let mut loser = winner.clone();
13044 loser.label = 'B';
13045 loser.index = 1;
13046 runner.state.candidates.push(loser);
13047 runner.state.config.graph.graft_runner_up = true;
13048 runner.state.tally = Some(Tally {
13049 first_choice: BTreeMap::new(),
13050 borda: BTreeMap::new(),
13051 winner: winner.label,
13052 rankings: 1,
13053 unanimous_initial: true,
13054 deliberated: false,
13055 changed_votes: 0,
13056 unanimous_final: true,
13057 tie_break: None,
13058 judges: 1,
13059 present: 1,
13060 quorum: 1,
13061 met_quorum: true,
13062 uncontested: None,
13063 });
13064 runner.state.judgements.push(crate::run::Judgement {
13065 judge: 1,
13066 seat: "judge-1".into(),
13067 agent: "judge".into(),
13068 ranking: vec![winner.label, 'B'],
13069 reasons: BTreeMap::new(),
13070 confidence: None,
13071 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13072 order: vec![0, 1],
13073 failed: None,
13074 duration_ms: 0,
13075 });
13076 } else {
13077 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13078 from_run: "earlier".into(),
13079 generation: 1,
13080 mode: crate::recovery::Mode::VerifyRepair,
13081 reasons: Vec::new(),
13082 reviewed_head: None,
13083 rounds: Vec::new(),
13084 open_findings: Vec::new(),
13085 evidence: None,
13086 criteria_inherited: true,
13087 repair: crate::recovery::Repair::NotStarted,
13088 });
13089 }
13090 let reader = crate::repo_lock::SeatLease::for_checkout(
13091 &winner.worktree,
13092 false,
13093 crate::config::test_coordination().as_ref(),
13094 )
13095 .await
13096 .unwrap();
13097 let pause = runner.pause.clone();
13098 let (result, ()) = tokio::join!(
13099 async {
13100 if graft {
13101 runner.graft().await
13102 } else {
13103 runner.recovery_repair(&winner).await
13104 }
13105 },
13106 async {
13107 tokio::time::sleep(Duration::from_millis(100)).await;
13108 pause.park();
13109 }
13110 );
13111 result.unwrap();
13112 assert!(!marker.exists(), "a blocked seat must not spawn");
13113 runner.state = RunState::load(&runner.state.id).unwrap();
13115 if graft {
13116 assert_eq!(
13117 runner.state.graft.as_ref().unwrap().outcome,
13118 GraftOutcome::WaitingForRestore
13119 );
13120 } else {
13121 assert!(matches!(
13122 runner.state.recovery.as_ref().unwrap().repair,
13123 crate::recovery::Repair::WaitingForRestore { .. }
13124 ));
13125 }
13126 drop(reader);
13127 runner.pause = Pause::new();
13128 runner.state.resource_block = None;
13129 let external =
13131 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13132 assert_eq!(
13133 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13134 0
13135 );
13136 let leftover = winner.worktree.join("leftover");
13137 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13138 let result = if graft {
13139 runner.graft().await
13140 } else {
13141 runner.recovery_repair(&winner).await
13142 };
13143 assert!(
13144 result
13145 .unwrap_err()
13146 .is::<crate::repo_lock::ResourceBlocked>()
13147 );
13148 assert!(leftover.exists());
13149 assert!(!marker.exists());
13150 drop(external);
13151 runner.state.resource_block = None;
13152 let result = if graft {
13153 runner.graft().await
13154 } else {
13155 runner.recovery_repair(&winner).await
13156 };
13157 result.unwrap();
13158 assert!(!leftover.exists(), "restore must precede retry");
13159 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13160 if graft {
13161 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13162 } else {
13163 assert!(matches!(
13164 runner.state.recovery.as_ref().unwrap().repair,
13165 crate::recovery::Repair::Done { .. }
13166 ));
13167 }
13168 }
13169 }
13170
13171 #[tokio::test]
13172 #[cfg(unix)]
13173 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13174 use std::os::fd::AsRawFd as _;
13175 let tmp = tempfile::tempdir().unwrap();
13176 let mut runner = registered_runner(tmp.path()).await;
13177 runner.state.config.graph.timeout_resource_wait = 1;
13178 let winner = runner.state.candidates[0].clone();
13179 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13180 runner.state.tally = Some(Tally {
13181 first_choice: BTreeMap::new(),
13182 borda: BTreeMap::new(),
13183 winner: winner.label,
13184 rankings: 0,
13185 unanimous_initial: false,
13186 deliberated: false,
13187 changed_votes: 0,
13188 unanimous_final: false,
13189 tie_break: None,
13190 judges: 0,
13191 present: 0,
13192 quorum: 0,
13193 met_quorum: true,
13194 uncontested: Some("fixture".into()),
13195 });
13196 runner.state.graft = Some(GraftRecord {
13197 pre: pre.clone(),
13198 post: None,
13199 lines: None,
13200 limit: None,
13201 agent: None,
13202 outcome: GraftOutcome::Running,
13203 requested: Vec::new(),
13204 adoption: None,
13205 adoption_error: None,
13206 });
13207 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13208 git::commit_all(&winner.worktree, "unfinished graft")
13209 .await
13210 .unwrap();
13211 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13212 let leftover = winner.worktree.join("leftover");
13213 std::fs::write(&leftover, "untrusted edits").unwrap();
13214 let path = runner.state.repo.join(".git/updater.lock");
13215 let external = std::fs::File::create(&path).unwrap();
13216 assert_eq!(
13217 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13218 0
13219 );
13220 assert!(
13221 runner
13222 .graft()
13223 .await
13224 .unwrap_err()
13225 .is::<crate::repo_lock::ResourceBlocked>()
13226 );
13227 assert!(matches!(
13228 runner.state.graft.as_ref().unwrap().outcome,
13229 GraftOutcome::Running
13230 ));
13231 assert_eq!(
13232 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13233 unfinished
13234 );
13235 assert!(leftover.exists());
13236 drop(external);
13237 std::fs::remove_file(&path).unwrap();
13238 std::fs::create_dir(&path).unwrap();
13239 assert!(runner.graft().await.is_err());
13240 assert!(matches!(
13241 runner.state.graft.as_ref().unwrap().outcome,
13242 GraftOutcome::Running
13243 ));
13244 assert_eq!(
13245 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13246 unfinished
13247 );
13248 assert!(leftover.exists());
13249 std::fs::remove_dir(&path).unwrap();
13250 runner.graft().await.unwrap();
13251 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13252 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13253 assert!(!leftover.exists());
13254 }
13255
13256 #[tokio::test]
13257 #[cfg(unix)]
13258 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13259 use std::os::fd::AsRawFd as _;
13260 let tmp = tempfile::tempdir().unwrap();
13261 let mut runner = registered_runner(tmp.path()).await;
13262 runner.state.prep_complete = Some(false);
13263 runner.state.config.graph.timeout_resource_wait = 1;
13264 runner.state.save().unwrap();
13265 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13266 assert_eq!(
13267 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13268 0
13269 );
13270 runner.execute().await.unwrap();
13271 let disk = RunState::load(&runner.state.id).unwrap();
13272 assert!(disk.driver_exited);
13273 assert_eq!(disk.driver_pid, None);
13274 assert_eq!(runner.state.status, RunStatus::Blocked);
13275 assert_eq!(
13276 runner.state.resource_block.as_ref().unwrap().kind,
13277 "repository coordination"
13278 );
13279 assert!(runner.state.prep_failure.is_none());
13280 assert!(runner.state.seats.is_empty());
13281 runner.pause.park();
13282 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13285 .await
13286 .unwrap()
13287 .unwrap();
13288 assert!(runner.state.parked);
13289 assert!(runner.state.prep_failure.is_none());
13290 let disk = RunState::load(&runner.state.id).unwrap();
13291 assert!(disk.driver_exited);
13292 assert_eq!(disk.driver_pid, None);
13293 }
13294
13295 #[tokio::test]
13296 #[cfg(unix)]
13297 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13298 use std::os::fd::AsRawFd as _;
13299 let tmp = tempfile::tempdir().unwrap();
13300 let mut runner = registered_runner(tmp.path()).await;
13301 runner.state.prep_complete = Some(false);
13302 runner.state.config.graph.timeout_resource_wait = 30;
13303 runner.state.save().unwrap();
13304 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13305 assert_eq!(
13306 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13307 0
13308 );
13309 assert!(
13310 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13311 .await
13312 .is_err()
13313 );
13314 let disk = RunState::load(&runner.state.id).unwrap();
13315 assert!(disk.driver_exited);
13316 assert_eq!(disk.driver_pid, None);
13317 assert!(disk.status.resumable());
13318 assert!(runner.state.driver_exited);
13319 assert_eq!(runner.state.driver_pid, None);
13320 runner.state.save().unwrap();
13321 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13322 }
13323
13324 #[tokio::test]
13325 #[cfg(unix)]
13326 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13327 let tmp = tempfile::tempdir().unwrap();
13328 let mut runner = registered_runner(tmp.path()).await;
13329 let winner = runner.state.candidates[0].clone();
13330 let cache = tmp.path().join("cache");
13331 let started = tmp.path().join("pre-gate-started");
13332 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13333 runner.state.config.verify.gate =
13334 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13335 let home = crate::run::home();
13336 let owner =
13337 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13338 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13339 crate::cache::AcquireOutcome::Acquired(g) => g,
13340 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13341 };
13342 let mut gate = Box::pin(runner.run_gate(&winner));
13343 assert!(
13344 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13345 .await
13346 .is_err()
13347 );
13348 assert!(!started.exists(), "pre-gate must wait for the cache too");
13349 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13353 let editor = crate::repo_lock::SeatLease::checkout_with(
13354 &winner.worktree,
13355 true,
13356 crate::config::test_coordination().as_ref(),
13357 &wait,
13358 )
13359 .await
13360 .unwrap();
13361 drop(editor);
13362 guard.release();
13363 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13364 .await
13365 .unwrap()
13366 .unwrap();
13367 drop(gate);
13368 assert!(outcomes.iter().all(CommandOutcome::ok));
13369 assert!(started.exists());
13370 }
13371
13372 #[tokio::test]
13373 #[cfg(unix)]
13374 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13375 use std::os::fd::AsRawFd as _;
13376
13377 ask_test_home();
13378 let tmp = tempfile::tempdir().unwrap();
13379 let repo = tmp.path().join("repo");
13380 std::fs::create_dir_all(&repo).unwrap();
13381 init_repo(&repo);
13382 let common = repo.join(".git");
13383 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13384 let mut runner = runner_at(RunStatus::Prep);
13385 runner.state.repo = repo.clone();
13386 runner.state.config.coordination = crate::config::test_coordination();
13387 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13388 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13389 runner.state.config.graph.advise = false;
13390 runner.roles.implementers = vec![spec("implement")];
13391 runner.roles.judges = vec![spec("judge")];
13392 runner.prep().await.unwrap();
13393 let winner = runner.state.candidates[0].clone();
13394 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13397 let completed = tmp.path().join("seat-completed");
13398 runner.roles.implementers[0].command = vec![
13399 "sh".into(),
13400 "-c".into(),
13401 format!("touch '{}'; echo completed", completed.display()),
13402 ];
13403 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13404 assert_eq!(
13405 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13406 0
13407 );
13408 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13409 let mut implement = Box::pin(runner.implement());
13410 tokio::select! {
13411 result = &mut implement => panic!("rescue must wait: {result:?}"),
13412 result = tokio::time::timeout(Duration::from_secs(10), async {
13413 while !completed.exists() {
13414 tokio::time::sleep(Duration::from_millis(20)).await;
13415 }
13416 }) => result.unwrap(),
13417 }
13418 assert!(
13419 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13420 .await
13421 .is_err()
13422 );
13423 assert_eq!(
13424 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13425 before
13426 );
13427 assert_eq!(
13428 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13429 0
13430 );
13431 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13432 .await
13433 .unwrap()
13434 .unwrap();
13435 drop(implement);
13436 assert_ne!(
13437 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13438 before
13439 );
13440 assert!(git::is_clean(&winner.worktree).await.unwrap());
13441
13442 let started = tmp.path().join("pre-gate-started");
13443 let proceed = tmp.path().join("pre-gate-proceed");
13444 let checked = tmp.path().join("gate-ran");
13445 runner.state.config.verify.pre_gate = vec![format!(
13446 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13447 started.display(),
13448 proceed.display()
13449 )];
13450 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13451 assert_eq!(
13452 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13453 0
13454 );
13455 let mut gate = Box::pin(runner.run_gate(&winner));
13456 assert!(
13457 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13458 .await
13459 .is_err()
13460 );
13461 assert!(!started.exists());
13462 assert!(!checked.exists());
13463 assert!(git::is_clean(&winner.worktree).await.unwrap());
13464 assert_eq!(
13465 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13466 0
13467 );
13468 tokio::select! {
13469 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13470 result = tokio::time::timeout(Duration::from_secs(10), async {
13471 while !started.exists() {
13472 tokio::time::sleep(Duration::from_millis(20)).await;
13473 }
13474 }) => result.unwrap(),
13475 }
13476 assert_eq!(
13477 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13478 -1,
13479 "mechanical edits must hold the real lock"
13480 );
13481 std::fs::write(proceed, "ready").unwrap();
13482 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13483 .await
13484 .unwrap()
13485 .unwrap();
13486 drop(gate);
13487 assert!(outcomes.iter().all(CommandOutcome::ok));
13488 assert!(checked.exists());
13489 assert!(runner.state.pre_gate_commit.is_some());
13490 assert!(git::is_clean(&winner.worktree).await.unwrap());
13491 assert_eq!(
13492 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13493 0,
13494 "gate completion must release coordination"
13495 );
13496 drop(outsider);
13497 std::fs::remove_file(common.join("updater.lock")).unwrap();
13498 std::fs::create_dir(common.join("updater.lock")).unwrap();
13499 std::fs::remove_file(&started).unwrap();
13500 std::fs::remove_file(&checked).unwrap();
13501 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13502 let error = runner.run_gate(&winner).await.unwrap_err();
13503 assert!(
13504 error
13505 .to_string()
13506 .contains("acquire mandatory repository lock")
13507 );
13508 assert!(
13509 !started.exists(),
13510 "failed acquisition must prevent pre-gate"
13511 );
13512 assert!(!checked.exists(), "failed acquisition must prevent gate");
13513 assert!(git::is_clean(&winner.worktree).await.unwrap());
13514 assert_eq!(
13515 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13516 head
13517 );
13518 }
13519
13520 #[tokio::test]
13521 #[cfg(unix)]
13522 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13523 use std::os::fd::AsRawFd as _;
13524 use std::time::Duration;
13525
13526 ask_test_home();
13527 let tmp = tempfile::tempdir().unwrap();
13528 let repo = tmp.path().join("repo");
13529 std::fs::create_dir_all(&repo).unwrap();
13530 init_repo(&repo);
13531 let common = repo.join(".git");
13532 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13533 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13534 assert_eq!(
13535 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13536 0
13537 );
13538 let mut runner = runner_at(RunStatus::Prep);
13539 runner.state.repo = repo.clone();
13540 runner.state.config.coordination = crate::config::test_coordination();
13541 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13542 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13543 runner.state.config.graph.advise = true;
13544 runner.state.config.graph.advisors = 1;
13545 runner.roles.implementers = vec![spec("implement")];
13546 runner.roles.judges = vec![spec("judge")];
13547 let root = runner.state.worktree_root();
13548 let started = tmp.path().join("setup-started");
13549 let proceed = tmp.path().join("setup-proceed");
13550 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13551 run: Some(format!(
13552 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13553 started.display(),
13554 proceed.display()
13555 )),
13556 ..Default::default()
13557 }];
13558 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13559 let config = git::git(&repo, &["config", "--local", "--list"])
13560 .await
13561 .unwrap();
13562 let mut prep = Box::pin(runner.prep());
13563 assert!(
13564 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13565 .await
13566 .is_err()
13567 );
13568 assert!(
13569 !root.exists(),
13570 "contention must prevent even checkout creation"
13571 );
13572 assert!(!started.exists(), "contention must prevent setup");
13573 assert_eq!(
13574 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13575 branches
13576 );
13577 assert_eq!(
13578 git::git(&repo, &["config", "--local", "--list"])
13579 .await
13580 .unwrap(),
13581 config
13582 );
13583 drop(lock);
13584 tokio::select! {
13585 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13586 result = tokio::time::timeout(Duration::from_secs(10), async {
13587 while !started.exists() {
13588 tokio::time::sleep(Duration::from_millis(20)).await;
13589 }
13590 }) => result.unwrap(),
13591 }
13592 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13593 assert_eq!(
13594 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13595 -1
13596 );
13597 assert!(
13598 tokio::time::timeout(
13599 Duration::from_millis(100),
13600 crate::repo_lock::SeatLease::for_checkout(
13601 &repo,
13602 true,
13603 crate::config::test_coordination().as_ref()
13604 )
13605 )
13606 .await
13607 .is_err(),
13608 "preparation must also exclude local editors"
13609 );
13610 std::fs::write(proceed, "ready").unwrap();
13611 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13612 .await
13613 .unwrap()
13614 .unwrap();
13615 drop(prep);
13616 assert_eq!(runner.state.prep_complete, Some(true));
13617 assert!(root.join("judge-1").exists());
13618 assert!(root.join("advisor-1").exists());
13619 assert_eq!(
13620 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13621 0
13622 );
13623 }
13624
13625 #[tokio::test]
13626 #[cfg(unix)]
13627 async fn preparation_lock_failure_prevents_repository_changes() {
13628 ask_test_home();
13629 let tmp = tempfile::tempdir().unwrap();
13630 let repo = tmp.path().join("repo");
13631 std::fs::create_dir_all(&repo).unwrap();
13632 init_repo(&repo);
13633 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13634 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13637 let mut runner = runner_at(RunStatus::Prep);
13638 runner.state.repo = repo.clone();
13639 runner.state.config.coordination = crate::config::test_coordination();
13640 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13641 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13642 runner.roles.implementers = vec![spec("implement")];
13643 let error = runner.prep().await.unwrap_err();
13644 assert!(
13645 error
13646 .to_string()
13647 .contains("acquire mandatory repository lock")
13648 );
13649 assert!(!runner.state.worktree_root().exists());
13650 assert!(runner.state.candidates.is_empty());
13651 assert!(!repo.join(".git/config.worktree").exists());
13652 assert_eq!(
13653 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13654 "* main"
13655 );
13656 }
13657
13658 #[tokio::test]
13659 async fn preparation_records_creation_intent_before_git_can_fail() {
13660 ask_test_home();
13661 let tmp = tempfile::tempdir().unwrap();
13662 let repo = tmp.path().join("repo");
13663 std::fs::create_dir_all(&repo).unwrap();
13664 init_repo(&repo);
13665 let mut runner = runner_at(RunStatus::Prep);
13666 runner.state.repo = repo.clone();
13667 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13668 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13669 runner.state.config.blind.commit_msg_hook = false;
13670 runner.roles.implementers = vec![spec("one")];
13671 let label = blind::assign_labels(1, runner.state.seed)[0];
13672 let branch = runner.state.branch_for(label);
13673 git::git(&repo, &["branch", &branch]).await.unwrap();
13676 assert!(runner.prep().await.is_err());
13677 let saved = RunState::load(&runner.state.id).unwrap();
13678 assert!(saved.candidates.is_empty());
13679 let event = saved
13680 .events
13681 .iter()
13682 .find(|e| e.node == "prep-intent")
13683 .unwrap();
13684 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13685 assert_eq!(intent["branch"], branch);
13686 assert_eq!(intent["start"], runner.state.base_commit);
13687 assert_eq!(
13688 intent["worktree"],
13689 runner
13690 .state
13691 .worktree_root()
13692 .join(format!("cand-{label}"))
13693 .to_string_lossy()
13694 .as_ref()
13695 );
13696 assert_eq!(intent["picks"], serde_json::json!([]));
13697 }
13698
13699 #[tokio::test]
13700 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13701 ask_test_home();
13702 let tmp = tempfile::tempdir().unwrap();
13703 let repo = tmp.path().join("repo");
13704 std::fs::create_dir_all(&repo).unwrap();
13705 init_repo(&repo);
13706 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13707 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13708 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13709 git::git(&repo, &["add", "."]).await.unwrap();
13710 git::git(&repo, &["commit", "-m", "required seed"])
13711 .await
13712 .unwrap();
13713 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13714 git::git(&repo, &["checkout", "main"]).await.unwrap();
13715 let mut runner = runner_at(RunStatus::Prep);
13716 runner.state.repo = repo.clone();
13717 runner.state.base_commit = base.clone();
13718 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13719 runner.state.config.graph.advise = false;
13720 runner.state.config.blind.commit_msg_hook = false;
13721 runner.state.seeds = vec![refs::Seed {
13722 token: seed.clone(),
13723 sha: seed.clone(),
13724 kind: refs::SeedKind::Unmerged,
13725 branch: false,
13726 detail: String::new(),
13727 }];
13728 runner.roles.implementers = vec![spec("one"), spec("two")];
13729 runner.roles.judges = vec![spec("judge")];
13730 let label = blind::assign_labels(2, runner.state.seed)[0];
13731 let branch = runner.state.branch_for(label);
13732 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13733 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13734 .await
13735 .unwrap();
13736 runner.state.candidates.push(Candidate {
13737 index: 0,
13738 label,
13739 agent: "one".into(),
13740 branch,
13741 worktree,
13742 summary: String::new(),
13743 stat: String::new(),
13744 files: 0,
13745 commits: 0,
13746 empty: false,
13747 failed: None,
13748 verified_noop: None,
13749 duration_ms: 0,
13750 folded: false,
13751 });
13752 runner.state.save().unwrap();
13753 runner.state = RunState::load(&runner.state.id).unwrap();
13755 assert_eq!(runner.state.prep_complete, Some(false));
13756 runner.prep().await.unwrap();
13757 assert_eq!(runner.state.prep_complete, Some(true));
13758 assert_eq!(runner.state.candidates.len(), 2);
13759 for candidate in &runner.state.candidates {
13760 assert_eq!(
13761 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13762 git::tree_of(&repo, &seed).await.unwrap()
13763 );
13764 assert_eq!(
13765 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13766 "required commit"
13767 );
13768 }
13769 assert!(runner.state.worktree_root().join("judge-1").exists());
13770 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13771 runner.prep().await.unwrap();
13772 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13773
13774 runner.state.prep_complete = Some(false);
13775 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13776 run: Some(format!(
13777 "test -f '{}'",
13778 tmp.path().join("dependency-ready").display()
13779 )),
13780 ..Default::default()
13781 }];
13782 assert!(runner.prep().await.is_err());
13783 assert!(runner.state.prep_failure.is_some());
13784 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13785 runner.state = RunState::load(&runner.state.id).unwrap();
13786 assert!(
13787 runner.prep().await.is_err(),
13788 "correcting the cause alone must not auto-retry"
13789 );
13790 let failed_at = runner.state.events.last().unwrap().at;
13791 assert!(
13792 runner.retry_preparation(failed_at).is_err(),
13793 "old approvals cannot unlock the hold"
13794 );
13795 let mut task = crate::queue::Task::new(
13796 "retry".into(),
13797 "retry setup".into(),
13798 repo.clone(),
13799 crate::queue::Source::Human,
13800 );
13801 task.start(runner.state.id.clone());
13802 task.hold_machine(Some("setup failed".into()));
13803 task.release_operator("cli", None);
13804 runner
13805 .retry_preparation(task.resumption.as_ref().unwrap().at)
13806 .unwrap();
13807 runner.prep().await.unwrap();
13808 assert!(runner.state.prep_failure.is_none());
13809 assert_eq!(runner.state.prep_complete, Some(true));
13810 assert_eq!(runner.state.candidates.len(), 2);
13811 assert!(runner.state.worktree_root().join("judge-1").exists());
13812 for candidate in &runner.state.candidates {
13813 assert_eq!(
13814 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13815 git::tree_of(&repo, &seed).await.unwrap()
13816 );
13817 }
13818 }
13819
13820 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13821 let mut heads = Vec::new();
13822 for candidate in candidates {
13823 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13824 }
13825 heads
13826 }
13827
13828 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13829 ask_test_home();
13830 let tmp = tempfile::tempdir().unwrap();
13831 let repo = tmp.path().join("repo");
13832 std::fs::create_dir(&repo).unwrap();
13833 init_repo(&repo);
13834 let mut runner = runner_at(RunStatus::Reviewing);
13835 runner.state.repo = repo.clone();
13836 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13837 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13838 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13839 let branch = format!("candidate-{label}");
13840 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13841 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13842 .await
13843 .unwrap();
13844 if label == 'B' {
13845 std::fs::write(worktree.join("change"), "winner").unwrap();
13846 git::rescue_commit(&worktree, "candidate change")
13847 .await
13848 .unwrap();
13849 }
13850 runner.state.candidates.push(Candidate {
13851 index,
13852 label,
13853 agent: "alpha".into(),
13854 branch,
13855 worktree,
13856 summary: "completed reply".into(),
13857 stat: String::new(),
13858 files: usize::from(label == 'B'),
13859 commits: usize::from(label == 'B'),
13860 empty: label == 'A',
13861 failed: None,
13862 verified_noop: None,
13863 duration_ms: 1,
13864 folded: false,
13865 });
13866 }
13867 runner.state.judge_skipped = true;
13868 runner.state.tally = Some(Tally {
13869 first_choice: BTreeMap::new(),
13870 borda: BTreeMap::new(),
13871 winner: 'B',
13872 rankings: 0,
13873 unanimous_initial: false,
13874 deliberated: false,
13875 changed_votes: 0,
13876 unanimous_final: false,
13877 tie_break: None,
13878 judges: 0,
13879 present: 0,
13880 quorum: 0,
13881 met_quorum: true,
13882 uncontested: Some("only B produced a change".into()),
13883 });
13884 runner.state.save().unwrap();
13885 (tmp, runner)
13886 }
13887
13888 #[tokio::test]
13889 #[cfg(unix)]
13890 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13891 for (solo, failed_checkpoint) in
13892 [(true, false), (false, false), (true, true), (false, true)]
13893 {
13894 let (_tmp, mut runner) = preservation_resume_fixture().await;
13895 if solo {
13896 runner.state.candidates.retain(|c| c.label == 'A');
13897 }
13898 runner.state.candidates[0].empty = false;
13899 runner.state.candidates[0].summary.clear();
13900 runner.state.tally = None;
13901 runner.state.judge_skipped = false;
13902 runner.state.status = RunStatus::Implementing;
13903 runner.state.config.disk.min_free_bytes = u64::MAX;
13904 let mut implementer = spec("alpha");
13905 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13906 runner.roles.implementers = vec![implementer];
13907 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13908 if failed_checkpoint {
13909 std::fs::create_dir_all(&patch).unwrap();
13910 }
13911 runner.state.save().unwrap();
13912 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13913 runner.state = RunState::load(&runner.state.id).unwrap();
13914 assert_eq!(runner.state.status, RunStatus::Blocked);
13915 assert!(runner.state.interrupted_implementations.contains(&'A'));
13916 assert_eq!(runner.state.candidates[0].commits, 0);
13917 assert!(!runner.state.candidates[0].empty);
13918 assert!(runner.state.candidates[0].failed.is_none());
13919 if failed_checkpoint {
13920 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13921 std::fs::remove_dir(patch).unwrap();
13922 }
13923 runner.state.resource_block = None;
13924 runner.state.config.disk.min_free_bytes = 0;
13925 runner.roles.implementers[0].command = vec![
13926 "sh".into(),
13927 "-c".into(),
13928 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13929 ];
13930 runner.implement().await.unwrap();
13931 assert!(runner.state.interrupted_implementations.is_empty());
13932 assert!(runner.state.candidate_preservation.is_empty());
13933 assert_eq!(runner.state.candidates[0].commits, 1);
13934 assert!(!runner.state.candidates[0].empty);
13935 assert!(
13936 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13937 .unwrap()
13938 .contains("retried empty interrupted seat")
13939 );
13940 if !solo {
13941 assert_eq!(runner.state.candidates[1].commits, 1);
13942 }
13943 }
13944 }
13945
13946 #[tokio::test]
13947 #[cfg(unix)]
13948 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13949 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13950 let (_tmp, mut runner) = preservation_resume_fixture().await;
13951 runner.state.candidates.retain(|c| c.label == 'A');
13952 runner.state.candidates[0].empty = false;
13953 runner.state.candidates[0].summary.clear();
13954 runner.state.tally = None;
13955 runner.state.judge_skipped = false;
13956 runner.state.status = RunStatus::Implementing;
13957 runner.state.config.disk.min_free_bytes = u64::MAX;
13960 let wt = runner.state.candidates[0].worktree.clone();
13961 std::fs::write(
13962 wt.join("interrupted-source"),
13963 "retain implementation work\n",
13964 )
13965 .unwrap();
13966 if committed {
13967 git::rescue_commit(&wt, "implementation checkpoint")
13968 .await
13969 .unwrap();
13970 }
13971 let lock = git::git(
13972 &wt,
13973 &[
13974 "rev-parse",
13975 "--path-format=absolute",
13976 "--git-path",
13977 "index.lock",
13978 ],
13979 )
13980 .await
13981 .unwrap();
13982 if block_rescue {
13983 std::fs::write(&lock, "fixture-owned lock").unwrap();
13984 }
13985 let mut implementer = spec("alpha");
13986 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13987 runner.roles.implementers = vec![implementer];
13988 runner.state.save().unwrap();
13989 let result = runner.implement().await;
13990 assert_eq!(result.is_err(), block_rescue);
13991 assert_eq!(
13992 runner.state.resource_block.as_ref().unwrap().kind,
13993 "disk pressure"
13994 );
13995 assert!(
13996 runner
13997 .state
13998 .events
13999 .iter()
14000 .any(|e| e.message.contains("agent execution interrupted"))
14001 );
14002 assert!(
14003 runner
14004 .state
14005 .dir()
14006 .join("artifacts/candidate-A-recovery/staged.patch")
14007 .exists()
14008 );
14009 if block_rescue {
14010 let saved = RunState::load(&runner.state.id).unwrap();
14011 assert!(saved.candidate_preservation.contains_key(&'A'));
14012 assert!(!saved.candidates[0].empty);
14013 assert_eq!(
14014 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14015 "retain implementation work\n"
14016 );
14017 std::fs::remove_file(lock).unwrap();
14018 runner.state = saved;
14019 } else {
14020 assert_eq!(runner.state.status, RunStatus::Blocked);
14021 assert_eq!(runner.state.candidates[0].commits, 1);
14022 }
14023 runner.roles.implementers.clear();
14026 runner.state.config.disk.min_free_bytes = 0;
14027 runner.state.resource_block = None;
14028 runner.implement().await.unwrap();
14029 assert!(runner.state.candidate_preservation.is_empty());
14030 assert_eq!(runner.state.candidates[0].commits, 1);
14031 assert!(!runner.state.candidates[0].empty);
14032 assert!(git::is_clean(&wt).await.unwrap());
14033 assert!(
14034 crate::run::read_artifact(&runner.state, "cand-A.patch")
14035 .unwrap()
14036 .contains("retain implementation work")
14037 );
14038 }
14039 }
14040
14041 #[tokio::test]
14042 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14043 let (_tmp, mut runner) = preservation_resume_fixture().await;
14044 runner.implement().await.unwrap();
14046 assert!(runner.state.candidate_preservation.is_empty());
14047 assert!(
14048 !runner
14049 .state
14050 .dir()
14051 .join("artifacts/candidate-A-recovery")
14052 .exists()
14053 );
14054 runner.fold_losers().await.unwrap();
14055 assert!(!runner.state.candidates[0].worktree.exists());
14056 for schema in [14, crate::run::SCHEMA] {
14057 runner.state.schema = schema;
14058 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14059 runner.state.save().unwrap();
14060 runner.state = RunState::load(&runner.state.id).unwrap();
14061 runner.implement().await.unwrap();
14062 assert!(runner.state.candidate_preservation.is_empty());
14063 assert!(runner.state.candidates[0].folded);
14064 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14065 }
14066 }
14067
14068 #[tokio::test]
14069 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14070 let (_tmp, mut runner) = preservation_resume_fixture().await;
14071 let wt = runner.state.candidates[0].worktree.clone();
14072 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14073 runner.state.schema = 14;
14074 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14075 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14076 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14077 for _ in 0..2 {
14078 assert!(runner.implement().await.is_err());
14079 assert_eq!(runner.state.status, RunStatus::Blocked);
14080 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14081 assert!(
14082 runner.state.candidates[0].empty,
14083 "old verdict must remain unusable"
14084 );
14085 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14086 assert!(runner.state.judge_skipped);
14087 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14088 assert!(
14089 crate::run::read_artifact(&runner.state, "cand-A.patch")
14090 .unwrap()
14091 .contains("owned edits")
14092 );
14093 assert!(wt.exists());
14094 assert_eq!(
14095 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14096 .await
14097 .unwrap(),
14098 1
14099 );
14100 runner.state = RunState::load(&runner.state.id).unwrap();
14101 }
14102 }
14103
14104 #[tokio::test]
14105 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14106 let (_tmp, mut runner) = preservation_resume_fixture().await;
14107 let wt = runner.state.candidates[0].worktree.clone();
14108 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14109 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14110 let index_lock = PathBuf::from(
14111 git::git(
14112 &wt,
14113 &[
14114 "rev-parse",
14115 "--path-format=absolute",
14116 "--git-path",
14117 "index.lock",
14118 ],
14119 )
14120 .await
14121 .unwrap(),
14122 );
14123 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14124 runner.state.schema = 14;
14125 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14126 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14127 assert!(runner.implement().await.is_err());
14128 assert!(runner.after_implement().is_err());
14129 assert!(runner.fold_losers().await.is_err());
14130 assert_eq!(
14131 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14132 "failed seat edits"
14133 );
14134 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14135 runner.state = RunState::load(&runner.state.id).unwrap();
14136 std::fs::remove_file(index_lock).unwrap();
14137 assert!(runner.implement().await.is_err());
14139 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14140 assert!(runner.state.candidates[0].empty);
14141 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14142 assert!(
14143 crate::run::read_artifact(&runner.state, "cand-A.patch")
14144 .unwrap()
14145 .contains("failed seat edits")
14146 );
14147 assert!(wt.exists());
14148 }
14149
14150 #[tokio::test]
14151 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14152 let (_tmp, mut runner) = preservation_resume_fixture().await;
14153 let repo = runner.state.repo.clone();
14154 let common = git::git(
14155 &repo,
14156 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14157 )
14158 .await
14159 .unwrap();
14160 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14161 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14162 assert!(runner.fold_losers().await.is_err());
14163 assert!(!runner.state.candidates[0].worktree.exists());
14164 assert!(!runner.state.candidates[0].folded);
14165 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14166 runner.state = RunState::load(&runner.state.id).unwrap();
14167 std::fs::remove_file(branch_lock).unwrap();
14168 let judge = runner.state.worktree_root().join("judge-1");
14169 git::worktree_add_detached(&repo, &judge, "main")
14170 .await
14171 .unwrap();
14172 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14173 runner.roles.judges = vec![spec("judge")];
14174 assert!(runner.fold_losers().await.is_err());
14175 runner.state = RunState::load(&runner.state.id).unwrap();
14176 assert!(
14177 runner.state.candidates[0].folded,
14178 "candidate progress saved before judge failure"
14179 );
14180 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14181 assert!(judge.join("uncommitted").exists());
14182 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14183 runner.fold_losers().await.unwrap();
14184 assert!(!judge.exists());
14185 runner.fold_losers().await.unwrap();
14186 }
14187
14188 #[tokio::test]
14189 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14190 let (_tmp, mut runner) = preservation_resume_fixture().await;
14191 runner.state.candidates.retain(|c| c.label == 'B');
14192 runner.state.tally = None;
14193 runner.state.judge_skipped = false;
14194 runner.state.prep_complete = Some(true);
14195 runner.state.config.graph.advise = false;
14196 runner.state.config.graph.selected_implementer = Some("alpha".into());
14197 let wt = runner.state.candidates[0].worktree.clone();
14198 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14199 runner
14200 .state
14201 .candidate_preservation
14202 .insert('B', "awaiting persistence".into());
14203 let index_lock = git::git(
14204 &wt,
14205 &[
14206 "rev-parse",
14207 "--path-format=absolute",
14208 "--git-path",
14209 "index.lock",
14210 ],
14211 )
14212 .await
14213 .unwrap();
14214 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14215 runner.state.save().unwrap();
14216 assert!(runner.execute_graph().await.is_err());
14217 assert_eq!(runner.state.status, RunStatus::Blocked);
14218 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14219 std::fs::remove_file(index_lock).unwrap();
14220 for _ in 0..2 {
14223 runner.state = RunState::load(&runner.state.id).unwrap();
14224 runner.pause.park();
14225 runner.execute_graph().await.unwrap();
14226 assert_eq!(runner.state.status, RunStatus::Implementing);
14227 assert!(runner.state.candidate_preservation.is_empty());
14228 assert_eq!(runner.state.candidates[0].commits, 2);
14229 assert!(runner.state.tally.is_none());
14230 }
14231 runner.implement().await.unwrap();
14232 assert_eq!(runner.state.status, RunStatus::Judging);
14233 runner.state.status = RunStatus::Blocked;
14235 runner.state.candidates[0].failed = Some("implementation failed".into());
14236 runner.state.save().unwrap();
14237 let error = runner.execute_graph().await.unwrap_err();
14238 assert!(
14239 error
14240 .to_string()
14241 .contains("selected implementation is held")
14242 );
14243 }
14244
14245 #[tokio::test]
14246 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14247 let (_tmp, mut runner) = preservation_resume_fixture().await;
14248 let repo = runner.state.repo.clone();
14249 let candidate = runner.state.candidates[0].clone();
14250 let registered = git::worktree_holding(&repo, &candidate.branch)
14253 .await
14254 .unwrap()
14255 .unwrap();
14256 assert_eq!(
14257 registered.canonicalize().unwrap(),
14258 candidate.worktree.canonicalize().unwrap()
14259 );
14260 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14261 assert_eq!(
14262 git::worktree_holding(&repo, &candidate.branch)
14263 .await
14264 .unwrap(),
14265 Some(registered)
14266 );
14267 assert!(
14268 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14269 "the stale registration initially keeps the branch checked out"
14270 );
14271 runner.fold_losers().await.unwrap();
14272 assert!(runner.state.candidates[0].folded);
14273 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14274 assert_eq!(
14275 git::worktree_holding(&repo, &candidate.branch)
14276 .await
14277 .unwrap(),
14278 None
14279 );
14280 runner.state = RunState::load(&runner.state.id).unwrap();
14281 runner.fold_losers().await.unwrap();
14282 }
14283
14284 #[tokio::test]
14285 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14286 let (_tmp, mut runner) = preservation_resume_fixture().await;
14287 let repo = runner.state.repo.clone();
14288 let candidate = runner.state.candidates[0].clone();
14289 let winner = runner.state.candidates[1].clone();
14290 let registered = git::worktree_holding(&repo, &candidate.branch)
14291 .await
14292 .unwrap()
14293 .unwrap();
14294 assert_eq!(
14295 registered.canonicalize().unwrap(),
14296 candidate.worktree.canonicalize().unwrap()
14297 );
14298 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14299 assert_eq!(
14300 git::worktree_holding(&repo, &candidate.branch)
14301 .await
14302 .unwrap(),
14303 Some(registered)
14304 );
14305 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14306
14307 for _ in 0..2 {
14308 fold_run(&mut runner.state, false, &crate::run::home())
14309 .await
14310 .unwrap();
14311 runner.state = RunState::load(&runner.state.id).unwrap();
14312 assert!(runner.state.candidates[0].folded);
14313 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14314 assert_eq!(
14315 git::worktree_holding(&repo, &candidate.branch)
14316 .await
14317 .unwrap(),
14318 None
14319 );
14320 assert!(!runner.state.candidates[1].folded);
14321 assert!(winner.worktree.exists());
14322 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14323 }
14324 }
14325
14326 #[tokio::test]
14327 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14328 ask_test_home();
14329 let tmp = tempfile::tempdir().unwrap();
14330 let repo = tmp.path().join("repo");
14331 std::fs::create_dir(&repo).unwrap();
14332 init_repo(&repo);
14333 let mut runner = runner_at(RunStatus::Implementing);
14334 runner.state.repo = repo.clone();
14335 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14336 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14337 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14338 let branch = format!("candidate-{label}");
14339 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14340 git::worktree_add_branch(&repo, &wt, &branch, "main")
14341 .await
14342 .unwrap();
14343 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14344 git::git(&wt, &["add", "README.md"]).await.unwrap();
14345 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14346 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14347 runner.state.candidates.push(Candidate {
14348 index,
14349 label,
14350 agent: "alpha".to_owned(),
14351 branch,
14352 worktree: wt,
14353 summary: "completed implementation".to_owned(),
14354 stat: String::new(),
14355 files: 0,
14356 commits: 0,
14357 empty: false,
14358 failed: None,
14359 verified_noop: None,
14360 duration_ms: 42,
14361 folded: false,
14362 });
14363 runner
14364 .state
14365 .candidate_preservation
14366 .insert(label, "awaiting persistence".to_owned());
14367 }
14368 let wt = runner.state.candidates[0].worktree.clone();
14369 let index_lock = git::git(
14370 &wt,
14371 &[
14372 "rev-parse",
14373 "--path-format=absolute",
14374 "--git-path",
14375 "index.lock",
14376 ],
14377 )
14378 .await
14379 .unwrap();
14380 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14381 runner.state.save().unwrap();
14382 assert!(runner.persist_candidates().await.is_err());
14383 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14384 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14385 assert_eq!(
14386 runner.state.candidates[1].commits, 1,
14387 "other results are still collected"
14388 );
14389 assert!(
14390 !runner.state.candidates[0].empty,
14391 "failed rescue is not no change"
14392 );
14393 assert_eq!(
14394 runner.state.candidates[0].summary,
14395 "completed implementation"
14396 );
14397 assert!(runner.after_implement().is_err());
14398 let home = tmp.path().join("fold-home");
14399 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14400 assert!(wt.join("new-file").exists());
14401 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14402 assert!(recovery.join("staged.patch").exists());
14403 assert!(recovery.join("unstaged.patch").exists());
14404 assert_eq!(
14405 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14406 "untracked edits\n"
14407 );
14408 runner.state.begin_pass();
14409 runner.state.end_pass_parked(ParkCause::Upgrade {
14410 from: "1.0".into(),
14411 to: Some("1.1".into()),
14412 });
14413 runner.state.save().unwrap();
14414 runner.state = RunState::load(&runner.state.id).unwrap();
14415 assert!(matches!(
14416 runner.state.passes.last().unwrap().end,
14417 Some(crate::run::PassEnd::Parked {
14418 cause: ParkCause::Upgrade { .. }
14419 })
14420 ));
14421 std::fs::remove_file(index_lock).unwrap();
14422 runner.implement().await.unwrap();
14424 assert!(runner.state.candidate_preservation.is_empty());
14425 assert!(
14426 runner
14427 .state
14428 .candidates
14429 .iter()
14430 .all(|c| c.commits == 1 && !c.empty)
14431 );
14432 assert!(git::is_clean(&wt).await.unwrap());
14433 assert!(
14434 crate::run::read_artifact(&runner.state, "cand-A.patch")
14435 .unwrap()
14436 .contains("later tracked edits")
14437 );
14438 }
14439
14440 #[tokio::test]
14441 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14442 ask_test_home();
14443 let tmp = tempfile::tempdir().unwrap();
14444 let repo = tmp.path().join("repo");
14445 std::fs::create_dir(&repo).unwrap();
14446 init_repo(&repo);
14447 let wt = tmp.path().join("candidate");
14448 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14449 .await
14450 .unwrap();
14451 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14452 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14453 assert!(wt.join("uncommitted").exists());
14454 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14455 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14456 let mut runner = runner_at(RunStatus::Implementing);
14457 runner.state.repo = repo.clone();
14458 runner.state.base_commit = "missing-base".to_owned();
14459 runner.state.candidates.push(Candidate {
14460 index: 0,
14461 label: 'A',
14462 agent: "alpha".into(),
14463 branch: "candidate".into(),
14464 worktree: wt.clone(),
14465 summary: "completed".into(),
14466 stat: String::new(),
14467 files: 0,
14468 commits: 0,
14469 empty: false,
14470 failed: None,
14471 verified_noop: None,
14472 duration_ms: 1,
14473 folded: false,
14474 });
14475 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14476 assert!(
14477 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14478 .await
14479 .is_err()
14480 );
14481 assert!(!runner.state.candidates[0].folded);
14482 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14483 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14484 runner
14485 .state
14486 .candidate_preservation
14487 .insert('A', "pending".into());
14488 assert!(runner.persist_candidates().await.is_err());
14489 assert!(!runner.state.candidates[0].empty);
14490 assert!(!runner.state.candidates[0].folded);
14491 assert!(wt.exists());
14492 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14493 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14494 std::fs::create_dir_all(&patch_path).unwrap();
14495 assert!(
14496 runner.persist_candidates().await.is_err(),
14497 "artifact failure stays pending"
14498 );
14499 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14500 assert!(!runner.state.candidates[0].empty);
14501 std::fs::remove_dir(patch_path).unwrap();
14502 runner.persist_candidates().await.unwrap();
14503 assert!(
14504 runner.state.candidates[0].empty,
14505 "only successful inspection confirms no change"
14506 );
14507 }
14508
14509 fn init_repo(dir: &Path) {
14510 let run = |args: &[&str]| {
14511 let out = std::process::Command::new("git")
14512 .args(args)
14513 .current_dir(dir)
14514 .quiet()
14515 .output()
14516 .expect("spawn git");
14517 assert!(
14518 out.status.success(),
14519 "git {args:?} failed: {}",
14520 String::from_utf8_lossy(&out.stderr)
14521 );
14522 };
14523 run(&["init", "-b", "main"]);
14524 run(&["config", "user.name", "magi test"]);
14525 run(&["config", "user.email", "magi@example.com"]);
14526 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14527 run(&["add", "-A"]);
14528 run(&["commit", "-m", "init"]);
14529 }
14530
14531 fn ask_test_home() {
14539 crate::run::pin_test_home();
14540 }
14541
14542 fn runner_at(status: RunStatus) -> Runner {
14545 let mut state = RunState::new(
14546 PathBuf::from("/nonexistent/repo"),
14547 "main".to_owned(),
14548 "deadbeef".to_owned(),
14549 "task".to_owned(),
14550 Config::default(),
14551 );
14552 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14555 state.id = format!(
14556 "20000101-000000-{:04x}",
14557 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14558 );
14559 state.status = status;
14560 Runner {
14561 state,
14562 roles: ResolvedRoles {
14563 implementers: Vec::new(),
14564 judges: Vec::new(),
14565 reviewers: Vec::new(),
14566 fixer: None,
14567 conductor: conductor(),
14568 implementer_roster: Vec::new(),
14569 judge_roster: Vec::new(),
14570 reviewer_roster: Vec::new(),
14571 },
14572 sem: Arc::new(Semaphore::new(1)),
14573 pause: Pause::new(),
14574 interrupt: Pause::new(),
14575 }
14576 }
14577
14578 #[test]
14579 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14580 crate::run::pin_test_home();
14581 let mut runner = runner_at(RunStatus::Gating);
14582 runner.state.save().unwrap();
14583 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14586 assert!(runner.claim_driver().is_err());
14587 assert!(runner.state.driver_exited);
14588 assert_eq!(runner.state.driver_pid, None);
14589 let disk = RunState::load(&runner.state.id).unwrap();
14590 assert!(disk.driver_exited);
14591 assert_eq!(disk.driver_pid, None);
14592 assert_eq!(disk.status, RunStatus::Gating);
14593 }
14594
14595 #[tokio::test]
14596 async fn an_early_graph_error_releases_its_driver() {
14597 crate::run::pin_test_home();
14598 let mut runner = runner_at(RunStatus::Blocked);
14599 runner.state.config.graph.selected_implementer = Some("selected".into());
14600 runner.state.save().unwrap();
14601 assert!(runner.execute().await.is_err());
14602 let disk = RunState::load(&runner.state.id).unwrap();
14603 assert!(disk.driver_exited);
14604 assert_eq!(disk.driver_pid, None);
14605 assert_eq!(disk.status, RunStatus::Blocked);
14606 }
14607
14608 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14609 crate::run::MergeRepair {
14610 question: "repair-question".into(),
14611 head: "old-head".into(),
14612 pr: "https://github.com/o/r/pull/42".into(),
14613 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14614 gate: vec![outcome(Some(0), false)],
14615 gate_head: Some("old-head".into()),
14616 gate_fixes: Vec::new(),
14617 pre_gate: Vec::new(),
14618 phase,
14619 failure: None,
14620 }
14621 }
14622
14623 #[tokio::test]
14624 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14625 crate::run::pin_test_home();
14626 let mut runner = runner_at(RunStatus::Landing);
14627 runner.state.merge_repairs.push(merge_repair_request(
14628 crate::run::MergeRepairPhase::Requested,
14629 ));
14630 runner.state.save().unwrap();
14631 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14632 assert!(
14633 runner
14634 .execute()
14635 .await
14636 .unwrap_err()
14637 .to_string()
14638 .contains("already running")
14639 );
14640 assert_eq!(
14641 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14642 crate::run::MergeRepairPhase::Requested
14643 );
14644 drop(claim);
14645 let mut disk = runner.state.clone();
14646 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14647 disk.save().unwrap();
14648 assert!(
14649 runner
14650 .execute()
14651 .await
14652 .unwrap_err()
14653 .to_string()
14654 .contains("decision changed")
14655 );
14656 assert_eq!(
14657 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14658 crate::run::MergeRepairPhase::Complete
14659 );
14660 }
14661
14662 #[test]
14663 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14664 crate::run::pin_test_home();
14665 let mut runner = runner_at(RunStatus::Landing);
14666 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14667 runner.state.reviews = request.reviews.clone();
14668 runner.state.gate = request.gate.clone();
14669 runner.state.gate_ran = true;
14670 runner.state.land_approval = Some(crate::run::LandApproval {
14671 question: "old".into(),
14672 head: "old-head".into(),
14673 });
14674 runner.state.merge_repairs.push(request);
14675 runner.begin_merge_repair_review(0).unwrap();
14676 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14677 assert!(!runner.state.gate_status().ok());
14678 assert!(runner.state.land_approval.is_none());
14679 let disk = RunState::load(&runner.state.id).unwrap();
14680 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14681 assert!(disk.merge_repairs[0].gate[0].ok());
14682 assert_eq!(
14683 disk.merge_repairs[0].phase,
14684 crate::run::MergeRepairPhase::Reviewing
14685 );
14686 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14687 assert!(disk.reviews.is_empty());
14688 }
14689
14690 #[tokio::test]
14691 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14692 crate::run::pin_test_home();
14693 for phase in [
14694 crate::run::MergeRepairPhase::Fixing,
14695 crate::run::MergeRepairPhase::Requested,
14696 ] {
14697 let mut runner = runner_at(RunStatus::Landing);
14698 runner.state.config.graph.land_rounds = 0;
14699 runner.state.merge_repairs.push(merge_repair_request(phase));
14700 runner.state.save().unwrap();
14701 assert!(!runner.repair_merge_review().await.unwrap());
14702 assert_eq!(runner.state.status, RunStatus::Blocked);
14703 assert_eq!(
14704 runner.state.merge_repairs[0].phase,
14705 crate::run::MergeRepairPhase::Failed
14706 );
14707 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14708 assert!(
14709 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14710 "interrupted"
14711 } else {
14712 "budget exhausted"
14713 })
14714 );
14715 assert!(runner.state.merge.is_none());
14716 }
14717 }
14718
14719 #[test]
14723 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14724 crate::run::pin_test_home();
14725 let mut runner = runner_at(RunStatus::Implementing);
14726 let interrupt = Pause::new();
14727 runner.watch_interrupt(interrupt.clone());
14728
14729 interrupt.park_because("task a1b2 asked to run first");
14730
14731 assert!(runner.park_here().expect("park_here"));
14732 assert!(runner.state.parked);
14733 let last = runner.state.events.last().expect("a park event");
14734 assert_eq!(last.node, "park");
14735 assert!(
14736 last.message.contains("task a1b2 asked to run first"),
14737 "expected the interrupt reason in {:?}",
14738 last.message
14739 );
14740 }
14741
14742 #[test]
14745 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14746 crate::run::pin_test_home();
14747 let mut runner = runner_at(RunStatus::Implementing);
14748 runner.state.begin_pass();
14749 let shutdown = Pause::new();
14750 runner.on_pause(shutdown.clone());
14751 let upgrade = ParkCause::Upgrade {
14752 from: "0.121.0".to_owned(),
14753 to: Some("0.121.1".to_owned()),
14754 };
14755 shutdown.park_for(upgrade.clone());
14756 shutdown.park_for(ParkCause::Operator);
14757
14758 assert!(runner.park_here().expect("park_here"));
14759 assert_eq!(
14760 runner.state.passes[0].end,
14761 Some(crate::run::PassEnd::Parked { cause: upgrade })
14762 );
14763 let last = runner.state.events.last().expect("a park event");
14764 assert!(last.message.contains("0.121.0"), "{}", last.message);
14765 }
14766
14767 #[test]
14775 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14776 crate::run::pin_test_home();
14777 let mut runner = runner_at(RunStatus::Implementing);
14778 let shutdown = Pause::new();
14779 runner.on_pause(shutdown.clone());
14780 let interrupt = Pause::new();
14781 runner.watch_interrupt(interrupt.clone());
14782
14783 assert!(!runner.park_here().expect("park_here"));
14785 assert!(!runner.state.parked);
14786
14787 interrupt.park_because("test");
14789 assert!(!shutdown.parked());
14790 assert!(runner.park_here().expect("park_here"));
14791 }
14792
14793 #[tokio::test]
14807 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14808 crate::run::pin_test_home();
14809 let mut runner = runner_at(RunStatus::Implementing);
14810 let interrupt = Pause::new();
14811 runner.watch_interrupt(interrupt.clone());
14812
14813 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14814 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14815
14816 let node = async move {
14820 started_tx.send(()).expect("send started");
14821 finish_rx.await.expect("recv finish");
14822 "node finished"
14823 };
14824
14825 let interrupter = async move {
14826 started_rx.await.expect("recv started");
14827 interrupt.park_because("higher-priority task waiting");
14829 tokio::task::yield_now().await;
14833 finish_tx.send(()).expect("send finish");
14834 };
14835
14836 let (node_result, ()) = tokio::join!(node, interrupter);
14837 assert_eq!(
14838 node_result, "node finished",
14839 "the in-flight call ran to completion"
14840 );
14841
14842 assert!(runner.park_here().expect("park_here"));
14845 assert!(runner.state.parked);
14846 }
14847
14848 #[test]
14854 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14855 crate::run::pin_test_home();
14856 let mut runner = runner_at(RunStatus::Judging);
14857 runner.state.config.agents = vec![conductor()];
14861 runner.state.candidates = vec![Candidate {
14862 index: 0,
14863 label: 'A',
14864 agent: "alpha".to_owned(),
14865 branch: "magi/x/A".to_owned(),
14866 worktree: PathBuf::from("/nonexistent/worktree"),
14867 summary: "did the thing".to_owned(),
14868 stat: "1 file changed".to_owned(),
14869 files: 1,
14870 commits: 1,
14871 empty: false,
14872 failed: None,
14873 verified_noop: None,
14874 duration_ms: 1234,
14875 folded: false,
14876 }];
14877 let run_id = runner.state.id.clone();
14878
14879 let interrupt = Pause::new();
14880 runner.watch_interrupt(interrupt.clone());
14881 interrupt.park_because("task c3d4 asked to run first");
14882 assert!(runner.park_here().expect("park_here"));
14883
14884 let resumed = Runner::resume(&run_id).expect("resume");
14885 assert_eq!(resumed.state.candidates.len(), 1);
14886 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14887 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14888 assert_eq!(resumed.state.status, runner.state.status);
14889 assert!(
14890 resumed.state.parked,
14891 "still parked until `execute` actually walks the graph again"
14892 );
14893 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14894 }
14895
14896 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14898 let mut q = ask::Question::new(
14899 run.to_owned(),
14900 "implement".to_owned(),
14901 "impl-A".to_owned(),
14902 "Which storage backend should the cache use?".to_owned(),
14903 String::new(),
14904 vec!["SQLite".to_owned(), "Redis".to_owned()],
14905 );
14906 store.put(&mut q).unwrap();
14907 q
14908 }
14909
14910 #[test]
14911 fn a_failed_runs_open_question_is_abandoned() {
14912 ask_test_home();
14913 let store = ask::Questions::open();
14914 let mut runner = runner_at(RunStatus::Failed);
14915 let run = runner.state.id.clone();
14916 let q = ask_open_question(&store, &run);
14917
14918 runner.settle_questions();
14919
14920 let back = store.get(&q.id).unwrap();
14921 assert!(
14922 !back.status.open(),
14923 "the seat that asked died with the run; nobody is left to read an answer"
14924 );
14925 assert!(
14926 back.detail.contains(&run) && back.detail.contains("failed"),
14927 "the reason names what the run became, not just that it is gone: {}",
14928 back.detail
14929 );
14930 }
14931
14932 #[test]
14933 fn a_merged_runs_open_question_is_abandoned_too() {
14934 ask_test_home();
14935 let store = ask::Questions::open();
14936 for status in [RunStatus::Merged, RunStatus::Ready] {
14939 let mut runner = runner_at(status);
14940 let run = runner.state.id.clone();
14941 let q = ask_open_question(&store, &run);
14942
14943 runner.settle_questions();
14944
14945 let back = store.get(&q.id).unwrap();
14946 assert!(
14947 !back.status.open(),
14948 "{status:?} run's question must not outlive the run"
14949 );
14950 }
14951 }
14952
14953 #[test]
14954 fn a_still_resumable_runs_open_question_is_left_alone() {
14955 ask_test_home();
14956 let store = ask::Questions::open();
14957 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14963 let mut runner = runner_at(status);
14964 let run = runner.state.id.clone();
14965 let q = ask_open_question(&store, &run);
14966
14967 runner.settle_questions();
14968
14969 let back = store.get(&q.id).unwrap();
14970 assert!(
14971 back.status.open(),
14972 "{status:?} is still alive; the question must still be waiting"
14973 );
14974 }
14975 }
14976
14977 #[test]
14978 fn settle_questions_never_touches_an_already_answered_question() {
14979 ask_test_home();
14980 let store = ask::Questions::open();
14981 let mut runner = runner_at(RunStatus::Failed);
14982 let run = runner.state.id.clone();
14983 let mut q = ask_open_question(&store, &run);
14984 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14985 .unwrap();
14986 store.put(&mut q).unwrap();
14987
14988 runner.settle_questions();
14993 runner.settle_questions();
14994
14995 let back = store.get(&q.id).unwrap();
14996 assert_eq!(
14997 back.status,
14998 ask::QuestionStatus::Answered,
14999 "a real answer is a decision on record, never overwritten by a sweep"
15000 );
15001 }
15002
15003 #[tokio::test]
15014 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15015 crate::run::pin_test_home();
15016 let tmp = tempfile::tempdir().expect("tempdir");
15017 let repo = tmp.path().join("repo");
15018 std::fs::create_dir_all(&repo).unwrap();
15019 init_repo(&repo);
15020
15021 let mut config = Config::default();
15022 config.graph.worktree_root = Some(tmp.path().join("wt"));
15023
15024 let mut state = RunState::new(
15025 repo.clone(),
15026 "main".to_owned(),
15027 "deadbeef".to_owned(),
15028 "task".to_owned(),
15029 config,
15030 );
15031 let root = state.worktree_root();
15032 let wt_a = root.join("cand-A");
15033 let wt_b = root.join("cand-B");
15034 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15035 .await
15036 .expect("worktree A");
15037 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15038 .await
15039 .expect("worktree B");
15040
15041 state.candidates = vec![
15042 Candidate {
15043 index: 0,
15044 label: 'A',
15045 agent: "alpha".to_owned(),
15046 branch: "magi/x/A".to_owned(),
15047 worktree: wt_a.clone(),
15048 summary: String::new(),
15049 stat: String::new(),
15050 files: 0,
15051 commits: 0,
15052 empty: false,
15053 failed: None,
15054 verified_noop: None,
15055 duration_ms: 0,
15056 folded: false,
15057 },
15058 Candidate {
15059 index: 1,
15060 label: 'B',
15061 agent: "beta".to_owned(),
15062 branch: "magi/x/B".to_owned(),
15063 worktree: wt_b.clone(),
15064 summary: String::new(),
15065 stat: String::new(),
15066 files: 0,
15067 commits: 0,
15068 empty: false,
15069 failed: None,
15070 verified_noop: None,
15071 duration_ms: 0,
15072 folded: false,
15073 },
15074 ];
15075 state.tally = Some(Tally {
15076 first_choice: BTreeMap::from([('A', 1)]),
15077 borda: BTreeMap::new(),
15078 winner: 'A',
15079 rankings: 1,
15080 unanimous_initial: true,
15081 deliberated: false,
15082 changed_votes: 0,
15083 unanimous_final: true,
15084 tie_break: None,
15085 judges: 1,
15086 present: 1,
15087 quorum: 1,
15088 met_quorum: true,
15089 uncontested: None,
15090 });
15091 state.status = RunStatus::Ready;
15092
15093 fold_run(&mut state, false, &crate::run::home())
15094 .await
15095 .expect("fold_run");
15096
15097 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15098 assert!(
15099 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15100 "the unmerged winner's branch survives"
15101 );
15102 assert!(
15103 !state.candidates[0].folded,
15104 "the winner is not marked folded"
15105 );
15106
15107 assert!(!wt_b.exists(), "the loser's worktree is removed");
15108 assert!(
15109 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15110 "the loser's branch is removed"
15111 );
15112 assert!(state.candidates[1].folded, "the loser is marked folded");
15113 }
15114
15115 #[tokio::test]
15116 #[cfg(unix)]
15117 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15118 use std::os::fd::AsRawFd as _;
15119 use std::time::Duration;
15120
15121 let tmp = tempfile::tempdir().unwrap();
15122 let repo = tmp.path().join("repo");
15123 std::fs::create_dir_all(&repo).unwrap();
15124 init_repo(&repo);
15125 let common = repo.join(".git");
15126 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15127 let home = tmp.path().join("home");
15128 let mut config = Config::default();
15129 config.coordination = crate::config::test_coordination();
15130 config.graph.worktree_root = Some(tmp.path().join("wt"));
15131 config.graph.timeout_resource_wait = 1;
15132 let mut state = RunState::new(
15133 repo.clone(),
15134 "main".into(),
15135 "deadbeef".into(),
15136 "task".into(),
15137 config,
15138 );
15139 let root = state.worktree_root();
15140 let candidate = root.join("cand-A");
15141 let reviewer = root.join("review-1");
15142 let branch = state.branch_for('A');
15143 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15144 .await
15145 .unwrap();
15146 git::worktree_add_detached(&repo, &reviewer, "main")
15147 .await
15148 .unwrap();
15149 git::acquire_worktree_config(&repo).await.unwrap();
15150 state.enabled_worktree_config = true;
15151 state.status = RunStatus::Ready;
15152 state.candidates.push(Candidate {
15153 index: 0,
15154 label: 'A',
15155 agent: "implement".into(),
15156 branch: branch.clone(),
15157 worktree: candidate.clone(),
15158 summary: String::new(),
15159 stat: String::new(),
15160 files: 0,
15161 commits: 0,
15162 empty: false,
15163 failed: None,
15164 verified_noop: None,
15165 duration_ms: 0,
15166 folded: false,
15167 });
15168 state.save_under(&home).unwrap();
15169 let config_before = std::fs::read(common.join("config")).unwrap();
15170 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15171 .await
15172 .unwrap();
15173 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15176 assert_eq!(
15177 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15178 0
15179 );
15180 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15181 assert!(
15182 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15183 .await
15184 .is_err()
15185 );
15186 assert!(candidate.exists());
15187 assert!(reviewer.exists());
15188 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15189 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15190 assert_eq!(
15191 git::git(&repo, &["worktree", "list", "--porcelain"])
15192 .await
15193 .unwrap(),
15194 worktrees_before
15195 );
15196 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15197 .await
15198 .unwrap()
15199 .unwrap_err();
15200 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15201 drop(fold);
15202 assert!(candidate.exists() && reviewer.exists());
15203 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15204 assert_eq!(
15205 state.resource_block.as_ref().unwrap().kind,
15206 "repository coordination"
15207 );
15208 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15209 drop(external);
15210 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15211 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15212 .await
15213 .unwrap()
15214 .unwrap();
15215 drop(fold);
15216 assert!(removed.contains(&branch));
15217 assert!(!root.exists());
15218 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15219 assert!(state.candidates[0].folded);
15220 assert!(!state.enabled_worktree_config);
15221 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15222 let enabled = git::git_raw(
15223 &repo,
15224 &["config", "--local", "--get", "extensions.worktreeConfig"],
15225 )
15226 .await
15227 .unwrap();
15228 assert!(!enabled.ok(), "the last config reference was released");
15229 }
15230
15231 #[tokio::test]
15234 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15235 let tmp = tempfile::tempdir().expect("tempdir");
15236 let repo = tmp.path().join("repo");
15237 std::fs::create_dir_all(&repo).unwrap();
15238 init_repo(&repo);
15239 let home = tmp.path().join("home");
15240
15241 let mut config = Config::default();
15242 config.graph.worktree_root = Some(tmp.path().join("wt"));
15243 let mut state = RunState::new(
15244 repo.clone(),
15245 "main".to_owned(),
15246 "deadbeef".to_owned(),
15247 "task".to_owned(),
15248 config,
15249 );
15250 git::git(&repo, &["branch", "magi/x/A", "main"])
15252 .await
15253 .expect("branch");
15254 state.candidates = vec![Candidate {
15255 index: 0,
15256 label: 'A',
15257 agent: "alpha".to_owned(),
15258 branch: "magi/x/A".to_owned(),
15259 worktree: state.worktree_root().join("cand-A"),
15260 summary: String::new(),
15261 stat: String::new(),
15262 files: 0,
15263 commits: 0,
15264 empty: false,
15265 failed: None,
15266 verified_noop: None,
15267 duration_ms: 0,
15268 folded: true,
15269 }];
15270 state.released_to = Some("20260901-000000-new1".to_owned());
15271 state.released_branches = vec!["magi/x/A".to_owned()];
15272
15273 fold_run(&mut state, true, &home).await.expect("fold_run");
15274
15275 assert!(
15276 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15277 "the handed-over branch survives a fold"
15278 );
15279 }
15280
15281 #[tokio::test]
15285 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15286 let tmp = tempfile::tempdir().expect("tempdir");
15287 let repo = tmp.path().join("repo");
15288 std::fs::create_dir_all(&repo).unwrap();
15289 init_repo(&repo);
15290 let run = |args: &[&str]| {
15291 let out = std::process::Command::new("git")
15292 .quiet()
15293 .args(args)
15294 .current_dir(&repo)
15295 .output()
15296 .expect("spawn git");
15297 assert!(out.status.success(), "git {args:?}");
15298 };
15299 run(&["branch", "magi/x/A"]);
15300 run(&["checkout", "-q", "-b", "magi/x/B"]);
15301 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15302 run(&["add", "-A"]);
15303 run(&["commit", "-q", "-m", "work"]);
15304 run(&["checkout", "-q", "main"]);
15305
15306 let probe = RunState::new(
15310 repo.clone(),
15311 "main".to_owned(),
15312 "deadbeef".to_owned(),
15313 "task".to_owned(),
15314 Config::default(),
15315 );
15316 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15317 let bare = tmp.path().join("origin.git");
15318 let out = std::process::Command::new("git")
15319 .quiet()
15320 .args(["init", "-q", "--bare"])
15321 .arg(&bare)
15322 .output()
15323 .expect("spawn git");
15324 assert!(out.status.success(), "git init --bare");
15325 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15326 run(&["push", "-q", "origin", "main"]);
15327
15328 let mut state = RunState::new(
15329 repo.clone(),
15330 "main".to_owned(),
15331 "deadbeef".to_owned(),
15332 "task".to_owned(),
15333 Config::default(),
15334 );
15335 state.seeds = vec![refs::Seed {
15336 token: "magi/27b2/A".to_owned(),
15337 kind: refs::SeedKind::Unresolved,
15338 sha: String::new(),
15339 branch: true,
15340 detail: "no branch or commit named magi/27b2/A".to_owned(),
15341 }];
15342
15343 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15344 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15345 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15346 let detail = empty_candidate_detail(&state, "main");
15347 assert!(detail.starts_with("empty candidate"), "{detail}");
15348 assert!(detail.contains("magi/27b2/A"), "{detail}");
15349 }
15350
15351 #[tokio::test]
15360 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15361 let tmp = tempfile::tempdir().expect("tempdir");
15362 let repo = tmp.path().join("repo");
15363 std::fs::create_dir_all(&repo).unwrap();
15364 init_repo(&repo);
15365
15366 let mut config = Config::default();
15367 config.merge.mode = MergeMode::Local;
15368
15369 let mut state = RunState::new(
15370 repo.clone(),
15371 "main".to_owned(),
15372 "deadbeef".to_owned(),
15373 "task".to_owned(),
15374 config,
15375 );
15376 state.candidates = vec![Candidate {
15377 index: 0,
15378 label: 'A',
15379 agent: "alpha".to_owned(),
15380 branch: "does-not-exist".to_owned(),
15381 worktree: repo.clone(),
15382 summary: String::new(),
15383 stat: String::new(),
15384 files: 0,
15385 commits: 0,
15386 empty: false,
15387 failed: None,
15388 verified_noop: None,
15389 duration_ms: 0,
15390 folded: false,
15391 }];
15392 state.tally = Some(Tally {
15393 first_choice: BTreeMap::from([('A', 1)]),
15394 borda: BTreeMap::new(),
15395 winner: 'A',
15396 rankings: 1,
15397 unanimous_initial: true,
15398 deliberated: false,
15399 changed_votes: 0,
15400 unanimous_final: true,
15401 tie_break: None,
15402 judges: 0,
15403 present: 0,
15404 quorum: 0,
15405 met_quorum: true,
15406 uncontested: Some("only candidate A produced a change".to_owned()),
15407 });
15408 state.reviews = vec![ReviewRound {
15409 round: 1,
15410 head: "deadbeef".to_owned(),
15411 verified_head: None,
15412 verified_at: None,
15413 reviews: Vec::new(),
15414 e2e: Vec::new(),
15415 fix: None,
15416 blocking: 0,
15417 answered: 0,
15418 expected: 0,
15419 clean: true,
15420 verify_retried: false,
15421 e2e_deferred: false,
15422 e2e_defer_reason: None,
15423 progressed: false,
15424 vote_split: false,
15425 reconsideration: Vec::new(),
15426 verdict: None,
15427 }];
15428 state.gate = vec![CommandOutcome {
15429 command: "test".to_owned(),
15430 code: Some(0),
15431 output_tail: String::new(),
15432 duration_ms: 0,
15433 resource_blocked: false,
15434 }];
15435 state.gate_ran = true;
15436 state.status = RunStatus::Ready;
15441 state.merge = Some(MergeOutcome {
15442 mode: MergeMode::Local,
15443 ok: false,
15444 detail: "already concluded".to_owned(),
15445 empty: false,
15446 });
15447
15448 let mut runner = Runner {
15449 state,
15450 roles: ResolvedRoles {
15451 implementers: Vec::new(),
15452 judges: Vec::new(),
15453 reviewers: Vec::new(),
15454 fixer: None,
15455 conductor: conductor(),
15456 implementer_roster: Vec::new(),
15457 judge_roster: Vec::new(),
15458 reviewer_roster: Vec::new(),
15459 },
15460 sem: Arc::new(Semaphore::new(1)),
15461 pause: Pause::new(),
15462 interrupt: Pause::new(),
15463 };
15464
15465 runner.merge().await.expect("merge");
15466
15467 assert_eq!(
15468 runner.state.status,
15469 RunStatus::Ready,
15470 "a concluded run's status must not change on reentry"
15471 );
15472 assert_eq!(
15473 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15474 Some("already concluded"),
15475 "merge must not run again once the node already recorded an outcome"
15476 );
15477 }
15478
15479 #[tokio::test]
15488 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15489 let tmp = tempfile::tempdir().expect("tempdir");
15490 let repo = tmp.path().join("repo");
15491 std::fs::create_dir_all(&repo).unwrap();
15492 init_repo(&repo);
15493
15494 let mut config = Config::default();
15495 config.merge.mode = MergeMode::Local;
15496
15497 let mut state = RunState::new(
15498 repo.clone(),
15499 "main".to_owned(),
15500 "deadbeef".to_owned(),
15501 "task".to_owned(),
15502 config,
15503 );
15504 state.candidates = vec![Candidate {
15505 index: 0,
15506 label: 'A',
15507 agent: "alpha".to_owned(),
15508 branch: "does-not-exist".to_owned(),
15509 worktree: repo.clone(),
15510 summary: String::new(),
15511 stat: String::new(),
15512 files: 0,
15513 commits: 0,
15514 empty: false,
15515 failed: None,
15516 verified_noop: None,
15517 duration_ms: 0,
15518 folded: false,
15519 }];
15520 state.tally = Some(Tally {
15521 first_choice: BTreeMap::from([('A', 1)]),
15522 borda: BTreeMap::new(),
15523 winner: 'A',
15524 rankings: 1,
15525 unanimous_initial: true,
15526 deliberated: false,
15527 changed_votes: 0,
15528 unanimous_final: true,
15529 tie_break: None,
15530 judges: 0,
15531 present: 0,
15532 quorum: 0,
15533 met_quorum: true,
15534 uncontested: Some("only candidate A produced a change".to_owned()),
15535 });
15536 state.reviews = vec![ReviewRound {
15537 round: 1,
15538 head: "deadbeef".to_owned(),
15539 verified_head: None,
15540 verified_at: None,
15541 reviews: Vec::new(),
15542 e2e: Vec::new(),
15543 fix: None,
15544 blocking: 0,
15545 answered: 0,
15546 expected: 0,
15547 clean: true,
15548 verify_retried: false,
15549 e2e_deferred: false,
15550 e2e_defer_reason: None,
15551 progressed: false,
15552 vote_split: false,
15553 reconsideration: Vec::new(),
15554 verdict: None,
15555 }];
15556 state.gate = Vec::new();
15558 state.gate_ran = false;
15559 state.status = RunStatus::Gating;
15560
15561 let mut runner = Runner {
15562 state,
15563 roles: ResolvedRoles {
15564 implementers: Vec::new(),
15565 judges: Vec::new(),
15566 reviewers: Vec::new(),
15567 fixer: None,
15568 conductor: conductor(),
15569 implementer_roster: Vec::new(),
15570 judge_roster: Vec::new(),
15571 reviewer_roster: Vec::new(),
15572 },
15573 sem: Arc::new(Semaphore::new(1)),
15574 pause: Pause::new(),
15575 interrupt: Pause::new(),
15576 };
15577
15578 runner.merge().await.expect("merge");
15579
15580 assert!(
15581 runner.state.merge.is_none(),
15582 "an empty gate must never be read as a passing one: {:?}",
15583 runner.state.merge
15584 );
15585 }
15586
15587 #[tokio::test]
15594 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15595 let tmp = tempfile::tempdir().expect("tempdir");
15596 let repo = tmp.path().join("repo");
15597 std::fs::create_dir_all(&repo).unwrap();
15598 init_repo(&repo);
15599
15600 let config = Config::default();
15602
15603 let mut state = RunState::new(
15604 repo.clone(),
15605 "main".to_owned(),
15606 "deadbeef".to_owned(),
15607 "task".to_owned(),
15608 config,
15609 );
15610 state.candidates = vec![Candidate {
15611 index: 0,
15612 label: 'A',
15613 agent: "alpha".to_owned(),
15614 branch: "does-not-exist".to_owned(),
15615 worktree: repo.clone(),
15616 summary: String::new(),
15617 stat: String::new(),
15618 files: 0,
15619 commits: 0,
15620 empty: false,
15621 failed: None,
15622 verified_noop: None,
15623 duration_ms: 0,
15624 folded: false,
15625 }];
15626 state.tally = Some(Tally {
15627 first_choice: BTreeMap::from([('A', 1)]),
15628 borda: BTreeMap::new(),
15629 winner: 'A',
15630 rankings: 1,
15631 unanimous_initial: true,
15632 deliberated: false,
15633 changed_votes: 0,
15634 unanimous_final: true,
15635 tie_break: None,
15636 judges: 0,
15637 present: 0,
15638 quorum: 0,
15639 met_quorum: true,
15640 uncontested: Some("only candidate A produced a change".to_owned()),
15641 });
15642 state.reviews = vec![ReviewRound {
15643 round: 1,
15644 head: "deadbeef".to_owned(),
15645 verified_head: None,
15646 verified_at: None,
15647 reviews: Vec::new(),
15648 e2e: Vec::new(),
15649 fix: None,
15650 blocking: 0,
15651 answered: 0,
15652 expected: 0,
15653 clean: true,
15654 verify_retried: false,
15655 e2e_deferred: false,
15656 e2e_defer_reason: None,
15657 progressed: false,
15658 vote_split: false,
15659 reconsideration: Vec::new(),
15660 verdict: None,
15661 }];
15662
15663 let mut runner = Runner {
15664 state,
15665 roles: ResolvedRoles {
15666 implementers: Vec::new(),
15667 judges: Vec::new(),
15668 reviewers: Vec::new(),
15669 fixer: None,
15670 conductor: conductor(),
15671 implementer_roster: Vec::new(),
15672 judge_roster: Vec::new(),
15673 reviewer_roster: Vec::new(),
15674 },
15675 sem: Arc::new(Semaphore::new(1)),
15676 pause: Pause::new(),
15677 interrupt: Pause::new(),
15678 };
15679
15680 runner.gate().await.expect("gate");
15681 assert!(
15682 runner.state.gate_ran,
15683 "zero configured commands is still a real attempt, not an unrun gate"
15684 );
15685 assert!(runner.state.gate.is_empty());
15686 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15687 assert_ne!(
15688 runner.state.status,
15689 RunStatus::Blocked,
15690 "a gate with nothing to check must not read as failed"
15691 );
15692
15693 runner.merge().await.expect("merge");
15694 assert_eq!(
15695 runner.state.status,
15696 RunStatus::Ready,
15697 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15698 );
15699 }
15700
15701 #[tokio::test]
15712 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15713 crate::run::pin_test_home();
15714 let home = crate::run::home();
15715
15716 let tmp = tempfile::tempdir().expect("tempdir");
15717 let repo = tmp.path().join("repo");
15718 std::fs::create_dir_all(&repo).unwrap();
15719 init_repo(&repo);
15720 let cache_dir = tmp.path().join("target");
15723
15724 let mut config = Config::default();
15725 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15726 config.graph.timeout_verify = Some(2);
15729 config.graph.timeout_resource_wait = 2;
15730
15731 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15732 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15733 .expect("no io error acquiring directly")
15734 {
15735 crate::cache::AcquireOutcome::Acquired(g) => g,
15736 crate::cache::AcquireOutcome::Busy(b) => {
15737 panic!("expected the direct acquire to win the lease first: {b:?}")
15738 }
15739 };
15740
15741 let mut state = RunState::new(
15742 repo.clone(),
15743 "main".to_owned(),
15744 "deadbeef".to_owned(),
15745 "task".to_owned(),
15746 config,
15747 );
15748 state.candidates = vec![Candidate {
15749 index: 0,
15750 label: 'A',
15751 agent: "alpha".to_owned(),
15752 branch: "does-not-exist".to_owned(),
15753 worktree: repo.clone(),
15754 summary: String::new(),
15755 stat: String::new(),
15756 files: 0,
15757 commits: 0,
15758 empty: false,
15759 failed: None,
15760 verified_noop: None,
15761 duration_ms: 0,
15762 folded: false,
15763 }];
15764 state.tally = Some(Tally {
15765 first_choice: BTreeMap::from([('A', 1)]),
15766 borda: BTreeMap::new(),
15767 winner: 'A',
15768 rankings: 1,
15769 unanimous_initial: true,
15770 deliberated: false,
15771 changed_votes: 0,
15772 unanimous_final: true,
15773 tie_break: None,
15774 judges: 0,
15775 present: 0,
15776 quorum: 0,
15777 met_quorum: true,
15778 uncontested: Some("only candidate A produced a change".to_owned()),
15779 });
15780 state.reviews = vec![ReviewRound {
15781 round: 1,
15782 head: "deadbeef".to_owned(),
15783 verified_head: None,
15784 verified_at: None,
15785 reviews: Vec::new(),
15786 e2e: Vec::new(),
15787 fix: None,
15788 blocking: 0,
15789 answered: 0,
15790 expected: 0,
15791 clean: true,
15792 verify_retried: false,
15793 e2e_deferred: false,
15794 e2e_defer_reason: None,
15795 progressed: false,
15796 vote_split: false,
15797 reconsideration: Vec::new(),
15798 verdict: None,
15799 }];
15800
15801 let mut runner = Runner {
15802 state,
15803 roles: ResolvedRoles {
15804 implementers: Vec::new(),
15805 judges: Vec::new(),
15806 reviewers: Vec::new(),
15807 fixer: None,
15808 conductor: conductor(),
15809 implementer_roster: Vec::new(),
15810 judge_roster: Vec::new(),
15811 reviewer_roster: Vec::new(),
15812 },
15813 sem: Arc::new(Semaphore::new(1)),
15814 pause: Pause::new(),
15815 interrupt: Pause::new(),
15816 };
15817
15818 let started = std::time::Instant::now();
15819 runner.gate().await.expect("gate");
15820 assert!(
15821 started.elapsed() < Duration::from_secs(1),
15822 "a gate with nothing to run must never wait on a lease it never needed"
15823 );
15824 assert!(
15825 runner.state.gate_ran,
15826 "zero commands is still a real, immediate attempt"
15827 );
15828 assert!(runner.state.gate.is_empty());
15829 assert_ne!(
15830 runner.state.status,
15831 RunStatus::Blocked,
15832 "must not read as resource-blocked on a lease it never asked for"
15833 );
15834 }
15835
15836 #[cfg(unix)]
15837 #[tokio::test]
15838 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15839 crate::run::pin_test_home();
15840 let tmp = tempfile::tempdir().unwrap();
15841 let mut config = Config::default();
15842 config.disk.min_free_bytes = 0;
15843 let mut state = RunState::new(
15844 tmp.path().to_owned(),
15845 "main".into(),
15846 "base".into(),
15847 "verification evidence".into(),
15848 config,
15849 );
15850 let (outcomes, pids) = run_commands(
15851 &mut state,
15852 "gate",
15853 "gate",
15854 1,
15855 &["sh".into(), "-c".into()],
15856 &[
15857 "printf before-timeout; exec sleep 30".into(),
15858 "kill -TERM $$".into(),
15859 "exit 7".into(),
15860 "printf success".into(),
15861 ],
15862 tmp.path(),
15863 Duration::from_secs(1),
15864 )
15865 .await;
15866 wait_for_timed_out_children_to_die(&pids).await;
15867 assert_eq!(
15868 pids.len(),
15869 1,
15870 "unleased timeout retains its death-check PID"
15871 );
15872 assert_eq!(outcomes.len(), 4);
15873 assert_eq!(outcomes[0].code, None);
15874 assert!(outcomes[0].output_tail.contains("before-timeout"));
15875 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15876 assert_eq!(outcomes[1].code, None);
15877 assert!(outcomes[1].output_tail.contains("command terminated:"));
15878 assert!(!outcomes[1].output_tail.contains("timed out"));
15879 assert_eq!(outcomes[2].code, Some(7));
15880 assert!(!outcomes[2].output_tail.contains("timed out"));
15881 assert_eq!(outcomes[3].code, Some(0));
15882 assert_eq!(outcomes[3].output_tail, "success");
15883 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15884 }
15885
15886 #[tokio::test]
15896 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15897 crate::run::pin_test_home();
15898
15899 let tmp = tempfile::tempdir().expect("tempdir");
15900 let repo = tmp.path().join("repo");
15901 std::fs::create_dir_all(&repo).unwrap();
15902 init_repo(&repo);
15903
15904 let mut config = Config::default();
15905 config.verify.gate = vec![
15906 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15907 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15908 .to_owned(),
15909 ];
15910
15911 let mut state = RunState::new(
15912 repo.clone(),
15913 "main".to_owned(),
15914 "deadbeef".to_owned(),
15915 "task".to_owned(),
15916 config,
15917 );
15918 let run_id = state.id.clone();
15919 state.candidates = vec![Candidate {
15920 index: 0,
15921 label: 'A',
15922 agent: "alpha".to_owned(),
15923 branch: "does-not-exist".to_owned(),
15924 worktree: repo.clone(),
15925 summary: String::new(),
15926 stat: String::new(),
15927 files: 0,
15928 commits: 0,
15929 empty: false,
15930 failed: None,
15931 verified_noop: None,
15932 duration_ms: 0,
15933 folded: false,
15934 }];
15935 state.tally = Some(Tally {
15936 first_choice: BTreeMap::from([('A', 1)]),
15937 borda: BTreeMap::new(),
15938 winner: 'A',
15939 rankings: 1,
15940 unanimous_initial: true,
15941 deliberated: false,
15942 changed_votes: 0,
15943 unanimous_final: true,
15944 tie_break: None,
15945 judges: 0,
15946 present: 0,
15947 quorum: 0,
15948 met_quorum: true,
15949 uncontested: Some("only candidate A produced a change".to_owned()),
15950 });
15951 state.reviews = vec![ReviewRound {
15952 round: 1,
15953 head: "deadbeef".to_owned(),
15954 verified_head: None,
15955 verified_at: None,
15956 reviews: Vec::new(),
15957 e2e: Vec::new(),
15958 fix: None,
15959 blocking: 0,
15960 answered: 0,
15961 expected: 0,
15962 clean: true,
15963 verify_retried: false,
15964 e2e_deferred: false,
15965 e2e_defer_reason: None,
15966 progressed: false,
15967 vote_split: false,
15968 reconsideration: Vec::new(),
15969 verdict: None,
15970 }];
15971
15972 let mut runner = Runner {
15973 state,
15974 roles: ResolvedRoles {
15975 implementers: Vec::new(),
15976 judges: Vec::new(),
15977 reviewers: Vec::new(),
15978 fixer: None,
15979 conductor: conductor(),
15980 implementer_roster: Vec::new(),
15981 judge_roster: Vec::new(),
15982 reviewer_roster: Vec::new(),
15983 },
15984 sem: Arc::new(Semaphore::new(1)),
15985 pause: Pause::new(),
15986 interrupt: Pause::new(),
15987 };
15988
15989 let started_marker = repo.join("started.marker");
15990 let release_marker = repo.join("release.marker");
15991 let poller = tokio::spawn(async move {
15992 for _ in 0..100 {
15997 if started_marker.exists()
15998 && let Ok(s) = crate::run::RunState::load(&run_id)
15999 && let Some(a) = s.active.get("gate")
16000 {
16001 std::fs::write(&release_marker, b"go").expect("release marker");
16002 return Some(a.clone());
16003 }
16004 tokio::time::sleep(Duration::from_millis(50)).await;
16005 }
16006 None
16007 });
16008
16009 runner.gate().await.expect("gate");
16010 let captured = poller.await.expect("poller task");
16011 let captured = captured.expect(
16012 "the poller never saw a `gate` task entry in run.json while the command was \
16013 still blocked on its own release marker",
16014 );
16015
16016 assert_eq!(captured.task.as_deref(), Some("gate"));
16017 assert_eq!(captured.node, "gate");
16018 assert_eq!(captured.index, Some(1));
16019 assert_eq!(captured.total, Some(1));
16020 assert!(
16021 captured
16022 .command
16023 .as_deref()
16024 .is_some_and(|c| c.contains("started.marker")),
16025 "{captured:?}"
16026 );
16027
16028 assert!(
16029 runner.state.active.is_empty(),
16030 "the entry must be cleared once the command actually finished: {:?}",
16031 runner.state.active
16032 );
16033 assert!(runner.state.gate_ran);
16034 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16035 }
16036
16037 #[tokio::test]
16040 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16041 use crate::verdict::{Finding, ReviewVote, Severity};
16042 crate::run::pin_test_home();
16043 let tmp = tempfile::tempdir().expect("tempdir");
16044 let repo = tmp.path().join("repo");
16045 std::fs::create_dir_all(&repo).unwrap();
16046 init_repo(&repo);
16047
16048 for (severity, vote, expect) in [
16049 (Severity::Major, ReviewVote::Reject, true),
16050 (Severity::Minor, ReviewVote::Reject, false),
16051 (Severity::Major, ReviewVote::Approve, false),
16052 ] {
16053 let mut round = review_round(false, 1, 1, 1, false, true);
16054 round.reviews = vec![ReviewRecord {
16055 reviewer: 1,
16056 agent: "alpha".to_owned(),
16057 summary: String::new(),
16058 findings: vec![Finding {
16059 id: "R1-1-1".to_owned(),
16060 severity,
16061 file: None,
16062 line: None,
16063 title: "t".to_owned(),
16064 detail: String::new(),
16065 }],
16066 vote: Some(vote),
16067 failed: None,
16068 duration_ms: 0,
16069 attempts: 0,
16070 }];
16071 let mut state = RunState::new(
16072 repo.clone(),
16073 "main".to_owned(),
16074 "deadbeef".to_owned(),
16075 "task".to_owned(),
16076 Config::default(),
16077 );
16078 state.reviews = vec![round];
16079 let mut runner = Runner {
16080 state,
16081 roles: ResolvedRoles {
16082 implementers: Vec::new(),
16083 judges: Vec::new(),
16084 reviewers: Vec::new(),
16085 fixer: None,
16086 conductor: conductor(),
16087 implementer_roster: Vec::new(),
16088 judge_roster: Vec::new(),
16089 reviewer_roster: Vec::new(),
16090 },
16091 sem: Arc::new(Semaphore::new(1)),
16092 pause: Pause::new(),
16093 interrupt: Pause::new(),
16094 };
16095 let shell = runner.state.config.shell();
16096 runner
16097 .stop_reviewing("round budget spent", &shell, &repo)
16098 .await
16099 .expect("stop_reviewing");
16100 assert_eq!(runner.state.status, RunStatus::Gating);
16101 assert_eq!(
16102 runner.state.contested_handoff.is_some(),
16103 expect,
16104 "{severity:?} + {vote:?}"
16105 );
16106 }
16107 }
16108
16109 #[tokio::test]
16122 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16123 crate::run::pin_test_home();
16124 let home = crate::run::home();
16125
16126 let tmp = tempfile::tempdir().expect("tempdir");
16127 let repo = tmp.path().join("repo");
16128 std::fs::create_dir_all(&repo).unwrap();
16129 init_repo(&repo);
16130 let head = crate::git::rev_parse(&repo, "HEAD")
16131 .await
16132 .expect("rev-parse");
16133 let cache_dir = tmp.path().join("target");
16136
16137 let mut config = Config::default();
16138 config.verify.e2e = vec![format!(
16139 "CARGO_TARGET_DIR='{}' test -f README.md",
16140 cache_dir.display()
16141 )];
16142 config.graph.review_rounds = 1;
16143 config.graph.timeout_verify = Some(2);
16146 config.graph.timeout_resource_wait = 2;
16147
16148 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16149 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16150 .expect("no io error acquiring directly")
16151 {
16152 crate::cache::AcquireOutcome::Acquired(g) => g,
16153 crate::cache::AcquireOutcome::Busy(b) => {
16154 panic!("expected the direct acquire to win the lease first: {b:?}")
16155 }
16156 };
16157
16158 let mut state = RunState::new(
16159 repo.clone(),
16160 "main".to_owned(),
16161 head.clone(),
16162 "task".to_owned(),
16163 config,
16164 );
16165 state.candidates = vec![Candidate {
16166 index: 0,
16167 label: 'A',
16168 agent: "alpha".to_owned(),
16169 branch: "does-not-exist".to_owned(),
16170 worktree: repo.clone(),
16171 summary: String::new(),
16172 stat: String::new(),
16173 files: 0,
16174 commits: 0,
16175 empty: false,
16176 failed: None,
16177 verified_noop: None,
16178 duration_ms: 0,
16179 folded: false,
16180 }];
16181 state.tally = Some(Tally {
16182 first_choice: BTreeMap::from([('A', 1)]),
16183 borda: BTreeMap::new(),
16184 winner: 'A',
16185 rankings: 1,
16186 unanimous_initial: true,
16187 deliberated: false,
16188 changed_votes: 0,
16189 unanimous_final: true,
16190 tie_break: None,
16191 judges: 0,
16192 present: 0,
16193 quorum: 0,
16194 met_quorum: true,
16195 uncontested: Some("only candidate A produced a change".to_owned()),
16196 });
16197 state.reviews = vec![ReviewRound {
16201 round: 1,
16202 head: head.clone(),
16203 verified_head: None,
16204 verified_at: None,
16205 reviews: Vec::new(),
16206 e2e: Vec::new(),
16207 fix: None,
16208 blocking: 1,
16209 answered: 1,
16210 expected: 1,
16211 clean: false,
16212 verify_retried: false,
16213 e2e_deferred: true,
16214 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16215 progressed: false,
16216 vote_split: false,
16217 reconsideration: Vec::new(),
16218 verdict: None,
16219 }];
16220
16221 let mut runner = Runner {
16222 state,
16223 roles: ResolvedRoles {
16224 implementers: Vec::new(),
16225 judges: Vec::new(),
16226 reviewers: Vec::new(),
16227 fixer: None,
16228 conductor: conductor(),
16229 implementer_roster: Vec::new(),
16230 judge_roster: Vec::new(),
16231 reviewer_roster: Vec::new(),
16232 },
16233 sem: Arc::new(Semaphore::new(1)),
16234 pause: Pause::new(),
16235 interrupt: Pause::new(),
16236 };
16237
16238 let shell = runner.state.config.shell();
16239 runner
16240 .stop_reviewing("round budget spent", &shell, &repo)
16241 .await
16242 .expect("stop_reviewing");
16243
16244 let last = runner.state.reviews.last().expect("round record");
16245 assert_eq!(
16246 last.e2e_status(),
16247 E2eStatus::ResourceBlocked,
16248 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16249 failed: {last:?}"
16250 );
16251 assert_eq!(
16252 last.verified_head.as_deref(),
16253 Some(head.as_str()),
16254 "which commit this attempt targeted is known even though nothing finished checking \
16255 it"
16256 );
16257 let first_attempt_at = last
16258 .verified_at
16259 .expect("when this attempt ran is known too");
16260 assert_ne!(
16261 runner.state.status,
16262 RunStatus::Blocked,
16263 "contention is evidence about the machine, not the patch — it must not settle the \
16264 run as blocked: {:?}",
16265 runner.state.status
16266 );
16267 assert!(
16268 !runner
16269 .state
16270 .events
16271 .iter()
16272 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16273 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16274 runner.state.events
16275 );
16276
16277 runner
16282 .stop_reviewing("round budget spent", &shell, &repo)
16283 .await
16284 .expect("stop_reviewing retry");
16285 assert_eq!(
16286 runner.state.reviews.len(),
16287 1,
16288 "no new round was started: {:?}",
16289 runner.state.reviews
16290 );
16291 let last = runner.state.reviews.last().expect("round record");
16292 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16293 assert!(
16294 last.verified_at.expect("still known") > first_attempt_at,
16295 "a second reentry must be a fresh attempt, not a stale copy of the first"
16296 );
16297 assert_ne!(runner.state.status, RunStatus::Blocked);
16298
16299 held.release();
16300 }
16301
16302 #[tokio::test]
16314 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16315 crate::run::pin_test_home();
16316 let home = crate::run::home();
16317
16318 let tmp = tempfile::tempdir().expect("tempdir");
16319 let repo = tmp.path().join("repo");
16320 std::fs::create_dir_all(&repo).unwrap();
16321 init_repo(&repo);
16322 let head = crate::git::rev_parse(&repo, "HEAD")
16323 .await
16324 .expect("rev-parse");
16325 let cache_dir = tmp.path().join("target");
16326
16327 let mut config = Config::default();
16328 config.verify.e2e = vec![format!(
16329 "CARGO_TARGET_DIR='{}' test -f README.md",
16330 cache_dir.display()
16331 )];
16332 config.graph.review_rounds = 1;
16333 config.graph.timeout_verify = Some(2);
16334 config.graph.timeout_resource_wait = 2;
16335
16336 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16337 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16338 .expect("no io error acquiring directly")
16339 {
16340 crate::cache::AcquireOutcome::Acquired(g) => g,
16341 crate::cache::AcquireOutcome::Busy(b) => {
16342 panic!("expected the direct acquire to win the lease first: {b:?}")
16343 }
16344 };
16345
16346 let mut state = RunState::new(
16347 repo.clone(),
16348 "main".to_owned(),
16349 head.clone(),
16350 "task".to_owned(),
16351 config,
16352 );
16353 state.candidates = vec![Candidate {
16354 index: 0,
16355 label: 'A',
16356 agent: "alpha".to_owned(),
16357 branch: "does-not-exist".to_owned(),
16358 worktree: repo.clone(),
16359 summary: String::new(),
16360 stat: String::new(),
16361 files: 0,
16362 commits: 0,
16363 empty: false,
16364 failed: None,
16365 verified_noop: None,
16366 duration_ms: 0,
16367 folded: false,
16368 }];
16369 state.tally = Some(Tally {
16370 first_choice: BTreeMap::from([('A', 1)]),
16371 borda: BTreeMap::new(),
16372 winner: 'A',
16373 rankings: 1,
16374 unanimous_initial: true,
16375 deliberated: false,
16376 changed_votes: 0,
16377 unanimous_final: true,
16378 tie_break: None,
16379 judges: 0,
16380 present: 0,
16381 quorum: 0,
16382 met_quorum: true,
16383 uncontested: Some("only candidate A produced a change".to_owned()),
16384 });
16385 state.reviews = vec![ReviewRound {
16389 round: 1,
16390 head: head.clone(),
16391 verified_head: Some(head.clone()),
16392 verified_at: Some(jiff::Timestamp::now()),
16393 reviews: Vec::new(),
16394 e2e: vec![CommandOutcome {
16395 command: format!(
16396 "CARGO_TARGET_DIR='{}' test -f README.md",
16397 cache_dir.display()
16398 ),
16399 code: None,
16400 output_tail: "waiting for the shared build cache".to_owned(),
16401 duration_ms: 0,
16402 resource_blocked: true,
16403 }],
16404 fix: None,
16405 blocking: 1,
16406 answered: 1,
16407 expected: 1,
16408 clean: false,
16409 verify_retried: false,
16410 e2e_deferred: false,
16411 e2e_defer_reason: None,
16412 progressed: false,
16413 vote_split: false,
16414 reconsideration: Vec::new(),
16415 verdict: None,
16416 }];
16417
16418 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16419 let mut runner = Runner {
16420 state,
16421 roles: ResolvedRoles {
16422 implementers: Vec::new(),
16423 judges: Vec::new(),
16424 reviewers: Vec::new(),
16425 fixer: None,
16426 conductor: conductor(),
16427 implementer_roster: Vec::new(),
16428 judge_roster: Vec::new(),
16429 reviewer_roster: Vec::new(),
16430 },
16431 sem: Arc::new(Semaphore::new(1)),
16432 pause: Pause::new(),
16433 interrupt: Pause::new(),
16434 };
16435
16436 runner.review_loop().await.expect("review_loop");
16441
16442 assert_eq!(
16443 runner.state.reviews.len(),
16444 1,
16445 "no new round was started on top of the unresolved one: {:?}",
16446 runner.state.reviews
16447 );
16448 let last = &runner.state.reviews[0];
16449 assert_eq!(
16450 last.e2e_status(),
16451 E2eStatus::ResourceBlocked,
16452 "still contended: {last:?}"
16453 );
16454 assert!(
16455 last.verified_at.expect("still known") > first_attempt_at,
16456 "review_loop must have actually retried the check, not left it exactly as found"
16457 );
16458 assert_ne!(
16459 runner.state.status,
16460 RunStatus::Blocked,
16461 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16462 runner.state.status
16463 );
16464
16465 held.release();
16466 }
16467
16468 #[tokio::test]
16469 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16470 crate::run::pin_test_home();
16471 for status in [RunStatus::Blocked, RunStatus::Gating] {
16472 let mut runner = runner_at(status);
16473 runner.state.config.graph.land = true;
16474 runner.state.gate_ran = true;
16475 runner.state.merge = Some(MergeOutcome {
16476 mode: MergeMode::Pr,
16477 ok: false,
16478 detail: "the fixer produced no commit".into(),
16479 empty: false,
16480 });
16481 let url = "https://example.invalid/o/r/pull/1";
16482 runner.state.pr = Some(crate::run::PrRecord {
16483 url: url.into(),
16484 number: 1,
16485 state: "open".into(),
16486 checks: "red".into(),
16487 round: 1,
16488 rounds: 1,
16489 red_at_merge: Vec::new(),
16490 });
16491 runner.state.save().unwrap();
16492 runner.execute_graph().await.unwrap();
16496 assert_eq!(runner.state.status, RunStatus::Blocked);
16497 assert!(runner.state.land_sync_pending);
16498 assert!(runner.state.events.iter().any(|event|
16499 event.node == "land" && event.message == format!("watching {url}")));
16500 assert_eq!(
16501 runner.state.merge.as_ref().unwrap().detail,
16502 "the fixer produced no commit"
16503 );
16504 }
16505 }
16506
16507 #[tokio::test]
16508 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16509 crate::run::pin_test_home();
16510 let tmp = tempfile::tempdir().expect("tempdir");
16511 let repo = tmp.path().join("repo");
16512 std::fs::create_dir_all(&repo).unwrap();
16513 init_repo(&repo);
16514
16515 let mut config = Config::default();
16516 config.merge.mode = MergeMode::Pr;
16517 config.graph.land = true;
16518 config.graph.land_approval = false;
16519
16520 let mut state = RunState::new(
16521 repo.clone(),
16522 "main".to_owned(),
16523 "deadbeef".to_owned(),
16524 "task".to_owned(),
16525 config,
16526 );
16527 state.candidates = vec![Candidate {
16528 index: 0,
16529 label: 'A',
16530 agent: "alpha".to_owned(),
16531 branch: "does-not-exist".to_owned(),
16532 worktree: repo.clone(),
16533 summary: String::new(),
16534 stat: String::new(),
16535 files: 0,
16536 commits: 0,
16537 empty: false,
16538 failed: None,
16539 verified_noop: None,
16540 duration_ms: 0,
16541 folded: false,
16542 }];
16543 state.tally = Some(Tally {
16544 first_choice: BTreeMap::from([('A', 1)]),
16545 borda: BTreeMap::new(),
16546 winner: 'A',
16547 rankings: 1,
16548 unanimous_initial: true,
16549 deliberated: false,
16550 changed_votes: 0,
16551 unanimous_final: true,
16552 tie_break: None,
16553 judges: 0,
16554 present: 0,
16555 quorum: 0,
16556 met_quorum: true,
16557 uncontested: Some("only candidate A produced a change".to_owned()),
16558 });
16559 state.reviews = vec![ReviewRound {
16560 round: 1,
16561 head: "deadbeef".to_owned(),
16562 verified_head: None,
16563 verified_at: None,
16564 reviews: Vec::new(),
16565 e2e: Vec::new(),
16566 fix: None,
16567 blocking: 0,
16568 answered: 0,
16569 expected: 0,
16570 clean: true,
16571 verify_retried: false,
16572 e2e_deferred: false,
16573 e2e_defer_reason: None,
16574 progressed: false,
16575 vote_split: false,
16576 reconsideration: Vec::new(),
16577 verdict: None,
16578 }];
16579 state.gate = vec![CommandOutcome {
16580 command: "test".to_owned(),
16581 code: Some(0),
16582 output_tail: String::new(),
16583 duration_ms: 0,
16584 resource_blocked: false,
16585 }];
16586 state.gate_ran = true;
16587 state.status = RunStatus::Landing;
16591 state.merge = Some(MergeOutcome {
16592 mode: MergeMode::Pr,
16593 ok: true,
16594 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16595 empty: false,
16596 });
16597
16598 ask_test_home();
16602 state.save().expect("save landing run");
16605 let store = ask::Questions::open();
16606 let q = ask_open_question(&store, &state.id);
16607
16608 let mut runner = Runner {
16609 state,
16610 roles: ResolvedRoles {
16611 implementers: Vec::new(),
16612 judges: Vec::new(),
16613 reviewers: Vec::new(),
16614 fixer: None,
16615 conductor: conductor(),
16616 implementer_roster: Vec::new(),
16617 judge_roster: Vec::new(),
16618 reviewer_roster: Vec::new(),
16619 },
16620 sem: Arc::new(Semaphore::new(1)),
16621 pause: Pause::new(),
16622 interrupt: Pause::new(),
16623 };
16624
16625 runner.execute().await.expect("execute");
16630
16631 assert_eq!(
16632 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16633 Some("https://example.invalid/x/y/pull/1"),
16634 "reentry must not push again or open a second pull request over the \
16635 one `land` is already watching"
16636 );
16637 assert_ne!(
16638 runner.state.status,
16639 RunStatus::Landing,
16640 "land could not actually reach the fake pull request, so it must \
16641 have given up rather than left the run silently parked forever"
16642 );
16643 assert_eq!(runner.state.status, RunStatus::Blocked);
16647 assert!(
16648 store.get(&q.id).unwrap().status.open(),
16649 "Blocked is still alive; settle_questions must have been a no-op here"
16650 );
16651 }
16652
16653 fn state_with_round(round: ReviewRound) -> RunState {
16654 let mut s = RunState::new(
16655 PathBuf::from("/repo"),
16656 "main".to_owned(),
16657 "abc1234".to_owned(),
16658 "add retries".to_owned(),
16659 Config::default(),
16660 );
16661 s.reviews = vec![round];
16662 s
16663 }
16664
16665 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16666 crate::verdict::Finding {
16667 id: id.to_owned(),
16668 severity,
16669 file: None,
16670 line: None,
16671 title: title.to_owned(),
16672 detail: String::new(),
16673 }
16674 }
16675
16676 #[test]
16677 fn pr_body_names_open_findings_and_declined_ones() {
16678 let round = ReviewRound {
16679 round: 2,
16680 head: "deadbee".to_owned(),
16681 verified_head: None,
16682 verified_at: None,
16683 reviews: vec![ReviewRecord {
16684 attempts: 0,
16685 reviewer: 1,
16686 agent: "alpha".to_owned(),
16687 summary: String::new(),
16688 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16689 vote: None,
16690 failed: None,
16691 duration_ms: 0,
16692 }],
16693 e2e: vec![CommandOutcome {
16694 command: "cargo test".to_owned(),
16695 code: Some(0),
16696 output_tail: String::new(),
16697 duration_ms: 0,
16698 resource_blocked: false,
16699 }],
16700 verify_retried: false,
16701 e2e_deferred: false,
16702 e2e_defer_reason: None,
16703 fix: Some(FixRecord {
16704 agent: "alpha".to_owned(),
16705 addressed: Vec::new(),
16706 rejected: vec![crate::verdict::Rejection {
16707 id: "R1-1-1".to_owned(),
16708 why: "not reachable from any caller".to_owned(),
16709 }],
16710 notes: String::new(),
16711 committed: true,
16712 failed: None,
16713 duration_ms: 0,
16714 continuation: None,
16715 }),
16716 blocking: 0,
16717 answered: 1,
16718 expected: 1,
16719 clean: false,
16720 progressed: true,
16721 vote_split: false,
16722 reconsideration: Vec::new(),
16723 verdict: None,
16724 };
16725 let state = state_with_round(round);
16726 let body = pr_message(&state, 'A').body;
16727
16728 assert!(body.contains("add retries"), "the task must still be there");
16729 assert!(body.contains("R2-1-1"), "{body}");
16730 assert!(!body.contains("unused import"), "{body}");
16731 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16732 assert!(
16733 !body.contains("not reachable from any caller"),
16734 "the reason it was declined: {body}"
16735 );
16736 }
16737
16738 #[test]
16739 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16740 let round = ReviewRound {
16741 round: 1,
16742 head: "deadbee".to_owned(),
16743 verified_head: None,
16744 verified_at: None,
16745 reviews: vec![ReviewRecord {
16746 attempts: 0,
16747 reviewer: 1,
16748 agent: "alpha".to_owned(),
16749 summary: String::new(),
16750 findings: Vec::new(),
16751 vote: None,
16752 failed: None,
16753 duration_ms: 0,
16754 }],
16755 e2e: Vec::new(),
16756 verify_retried: false,
16757 e2e_deferred: false,
16758 e2e_defer_reason: None,
16759 fix: None,
16760 blocking: 0,
16761 answered: 1,
16762 expected: 1,
16763 clean: true,
16764 progressed: false,
16765 vote_split: false,
16766 reconsideration: Vec::new(),
16767 verdict: None,
16768 };
16769 let state = state_with_round(round);
16770 let body = pr_message(&state, 'A').body;
16771 assert!(!body.contains("Open review findings"), "{body}");
16772 assert!(!body.contains("Declined"), "{body}");
16773 }
16774
16775 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16776 let mut state = RunState::new(
16777 PathBuf::from("/repo"),
16778 "main".to_owned(),
16779 "abc1234".to_owned(),
16780 instruction.to_owned(),
16781 Config::default(),
16782 );
16783 state.candidates.push(Candidate {
16784 index: 0,
16785 label: 'A',
16786 agent: "alpha".to_owned(),
16787 branch: "magi/x/A".to_owned(),
16788 worktree: PathBuf::from("/wt"),
16789 summary: summary.to_owned(),
16790 stat: String::new(),
16791 files: 1,
16792 commits: 1,
16793 empty: false,
16794 failed: None,
16795 verified_noop: None,
16796 folded: false,
16797 duration_ms: 0,
16798 });
16799 state
16800 }
16801
16802 fn review_state(subjects: &[&str]) -> RunState {
16803 let mut state = state_with_summary(
16804 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16805 "",
16806 );
16807 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16808 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16809 state
16810 }
16811
16812 #[tokio::test]
16815 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16816 ask_test_home();
16817 let tmp = tempfile::tempdir().unwrap();
16818 let repo = tmp.path().join("repo");
16819 std::fs::create_dir_all(&repo).unwrap();
16820 init_repo(&repo);
16821 let origin = tmp.path().join("origin.git");
16822 let g = |dir: &Path, args: &[&str]| {
16823 let out = std::process::Command::new("git")
16824 .args(args)
16825 .current_dir(dir)
16826 .quiet()
16827 .output()
16828 .expect("spawn git");
16829 assert!(
16830 out.status.success(),
16831 "git {args:?}: {}",
16832 String::from_utf8_lossy(&out.stderr)
16833 );
16834 };
16835 g(
16836 tmp.path(),
16837 &[
16838 "clone",
16839 "--bare",
16840 "-q",
16841 repo.to_str().unwrap(),
16842 origin.to_str().unwrap(),
16843 ],
16844 );
16845 g(
16846 &repo,
16847 &["remote", "add", "origin", origin.to_str().unwrap()],
16848 );
16849 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16850
16851 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16853 g(&repo, &["add", "-A"]);
16854 g(
16855 &repo,
16856 &["commit", "-q", "-m", "chore(deps): update a crate"],
16857 );
16858 g(&repo, &["push", "-q", "origin", "main"]);
16859 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16860 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16861 g(&repo, &["add", "-A"]);
16862 g(
16863 &repo,
16864 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16865 );
16866 g(&repo, &["checkout", "-q", "main"]);
16867
16868 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16869 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16870 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16872 assert_eq!(fallback, c1);
16873
16874 let mut state = review_state(&[
16875 "chore(deps): update a crate",
16876 "fix(daemon): apply a chosen action",
16877 ]);
16878 state.repo = repo.clone();
16879 state.base_branch = "main".to_owned();
16880 state.base_commit = c1;
16881 refresh_reviewed_commits(&mut state, "feat/own").await;
16882 assert_eq!(
16883 state.reviewed_commits,
16884 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16885 );
16886 assert_eq!(
16887 review_title(&state).as_deref(),
16888 Some("fix(daemon): apply a chosen action")
16889 );
16890 assert_eq!(
16891 leaked_subjects(&state, "feat/own").await,
16892 Some(vec!["chore(deps): update a crate".to_owned()])
16893 );
16894 state.config.merge.remote = "nowhere".to_owned();
16897 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16898 }
16899
16900 struct SummaryFixture {
16904 _tmp: tempfile::TempDir,
16905 repo: PathBuf,
16906 reply: PathBuf,
16907 runner: Runner,
16908 }
16909
16910 fn summary_fixture() -> SummaryFixture {
16911 ask_test_home();
16912 let tmp = tempfile::tempdir().unwrap();
16913 let repo = tmp.path().join("repo");
16914 std::fs::create_dir_all(&repo).unwrap();
16915 init_repo(&repo);
16916 let origin = tmp.path().join("origin.git");
16917 let g = |args: &[&str]| {
16918 let out = std::process::Command::new("git")
16919 .args(args)
16920 .current_dir(&repo)
16921 .quiet()
16922 .output()
16923 .expect("spawn git");
16924 assert!(
16925 out.status.success(),
16926 "git {args:?}: {}",
16927 String::from_utf8_lossy(&out.stderr)
16928 );
16929 };
16930 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16931 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16932 g(&["push", "-q", "origin", "main"]);
16933 g(&["checkout", "-q", "-b", "feat/own"]);
16934 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16935 g(&["add", "-A"]);
16936 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16937 let reply = tmp.path().join("reply.txt");
16938 std::fs::write(
16939 &reply,
16940 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16941 )
16942 .unwrap();
16943 let mut runner = runner_at(RunStatus::Gating);
16944 let mut state = review_state(&["feat(own): add the own file"]);
16945 state.repo = repo.clone();
16946 state.base_branch = "main".to_owned();
16947 state.candidates[0].branch = "feat/own".to_owned();
16948 state.candidates[0].worktree = repo.clone();
16949 let mut author = spec("alpha");
16950 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16951 author
16952 .env
16953 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16954 state.config.agents = vec![author];
16955 runner.state = state;
16956 SummaryFixture {
16957 _tmp: tmp,
16958 repo,
16959 reply,
16960 runner,
16961 }
16962 }
16963
16964 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16965 let winner = f.runner.state.candidates[0].clone();
16966 f.runner
16967 .guarded_pr_message(&winner, None, true)
16968 .await
16969 .unwrap()
16970 .expect("an English description never parks")
16971 }
16972
16973 fn clean_round_with_minor_findings() -> ReviewRound {
16974 let mut round = passing_round(2, None);
16975 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16976 round.reviews = vec![ReviewRecord {
16977 attempts: 0,
16978 reviewer: 1,
16979 agent: "alpha".into(),
16980 summary: "内部レビュー".into(),
16981 findings: vec![
16982 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16983 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16984 ],
16985 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16986 failed: None,
16987 duration_ms: 0,
16988 }];
16989 round
16990 }
16991
16992 #[tokio::test]
16993 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16994 for review_only in [false, true] {
16995 let mut f = summary_fixture();
16996 if !review_only {
16997 f.runner.state.candidates[0].agent = "alpha".into();
16998 }
16999 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17000 assert!(f.runner.state.reviews[0].clean);
17001 assert!(f.runner.state.reviews[0].fix.is_none());
17002 assert!(f.runner.state.open_findings().is_empty());
17003 assert_eq!(
17004 crate::pr_outcome::required_finding_ids(&f.runner.state),
17005 ["R2-1-1", "R2-1-2"]
17006 );
17007 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17008 let evidence = crate::pr_outcome::summary_evidence(
17009 &f.runner.state,
17010 crate::pr_outcome::FinalHead::At(&head),
17011 );
17012 assert!(
17013 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17014 );
17015 let omitted = posted(&mut f).await;
17017 assert!(omitted.body.contains("No description"));
17018 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17019 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17020 assert!(f.runner.state.change_summary.is_none());
17021 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17022 let described = posted(&mut f).await;
17023 assert!(described.body.contains("improve documentation for callers"));
17024 assert!(described.body.contains("remaining edge case"));
17025 assert!(described.body.contains("## Open review findings"));
17026 assert!(!described.body.contains("未修正"));
17027 }
17028 }
17029
17030 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17031 let mut f = summary_fixture();
17032 if !review_only {
17033 f.runner.state.candidates[0].agent = "alpha".into();
17034 }
17035 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17036 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17037 fixed
17038 .fix
17039 .as_mut()
17040 .unwrap()
17041 .rejected
17042 .push(crate::verdict::Rejection {
17043 id: "R1-1-1".into(),
17044 why: "呼び出されないコードのため辞退".into(),
17045 });
17046 let mut clean = clean_round_with_minor_findings();
17047 clean.verified_head = Some(head.clone());
17048 f.runner.state.reviews = vec![fixed, clean];
17049 ok_gate(&mut f.runner.state, Some(&head));
17050 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17051 f.runner.state.config.agents.push(scripted("beta", reply));
17052 f.runner.state.config.roles.language_judge =
17053 Some(crate::config::AgentChoice::One("beta".into()));
17054 f
17055 }
17056
17057 #[tokio::test]
17058 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17059 for review_only in [false, true] {
17060 let mut f = rewrite_fixture_with_findings(review_only,
17061 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17062 let m = posted(&mut f).await;
17063 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17064 let expected =
17065 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17066 assert!(m.body.contains(&expected));
17067 assert!(m.body.contains("## Review rounds"));
17068 assert!(
17069 m.body
17070 .contains("Gate (`verify.gate`): passed on the final commit")
17071 );
17072 assert!(
17073 m.body
17074 .contains("Tests (`verify.e2e`): passed on the final commit")
17075 );
17076 let c = f.runner.state.change_summary.as_ref().unwrap();
17077 assert!(
17078 m.body
17079 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17080 );
17081 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17082 assert!(crate::pr_outcome::covers_required_findings(
17083 &f.runner.state,
17084 &m.body
17085 ));
17086 assert!(m.body.contains("documentation remains incomplete"));
17087 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17088 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17089 assert!(f.runner.state.github_text.is_none());
17090 }
17091 }
17092
17093 #[tokio::test]
17094 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17095 for review_only in [false, true] {
17096 for reply in [
17097 "TITLE: fix: public description\nA short English description.",
17098 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17099 ] {
17100 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17101 let winner = f.runner.state.candidates[0].clone();
17102 let posted = f
17103 .runner
17104 .guarded_pr_message(&winner, None, true)
17105 .await
17106 .unwrap();
17107 assert!(posted.is_none());
17108 assert!(f.runner.state.parked);
17109 let withheld = f.runner.state.github_text.as_ref().unwrap();
17110 assert!(
17111 withheld.rewrite.is_none(),
17112 "an incomplete rewrite must not be selectable"
17113 );
17114 let q = ask::Questions::open()
17115 .get(withheld.question.as_ref().unwrap())
17116 .unwrap();
17117 assert!(
17118 !q.choices
17119 .contains(&crate::github_text::USE_REWRITE.to_owned())
17120 );
17121 assert!(
17122 q.detail.contains("R1-1-1")
17123 && q.detail.contains("R2-1-1")
17124 && q.detail.contains("R2-1-2")
17125 );
17126 }
17127 }
17128 }
17129
17130 #[tokio::test]
17131 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17132 for review_only in [false, true] {
17133 let mut f = summary_fixture();
17134 if !review_only {
17135 f.runner.state.candidates[0].agent = "alpha".into();
17136 }
17137 f.runner.state.instruction = "内部タスクの説明".into();
17138 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17139 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17140 f.runner.state.reviews = vec![
17141 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17142 round_with_notes(2, Some("二回目の内部修正報告です。")),
17143 passing_round(3, Some(&head)),
17144 ];
17145 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17146 ok_gate(&mut f.runner.state, Some(&head));
17147 let captured = f.repo.join("public-prompt.txt");
17148 let author = &mut f.runner.state.config.agents[0];
17149 author.command = vec![
17150 "sh".into(),
17151 "-c".into(),
17152 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17153 ];
17154 author.env.insert(
17155 "PROMPT_FILE".into(),
17156 captured.to_string_lossy().into_owned(),
17157 );
17158 let m = posted(&mut f).await;
17159 assert!(m.body.contains("Adds `own.txt`"));
17160 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17161 assert!(m.body.contains("R1-1-1"));
17162 assert!(
17163 m.body
17164 .contains("Gate (`verify.gate`): passed on the final commit")
17165 );
17166 assert!(
17167 m.body
17168 .contains("Tests (`verify.e2e`): passed on the final commit")
17169 );
17170 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17171 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17172 let prompt = std::fs::read_to_string(captured).unwrap();
17173 assert!(prompt.contains(&format!(
17174 "git diff {} {head}",
17175 f.runner.state.change_summary.as_ref().unwrap().base
17176 )));
17177 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17178 assert_eq!(
17179 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17180 "二回目の内部修正報告です。"
17181 );
17182 }
17183 }
17184
17185 #[tokio::test]
17186 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17187 for review_only in [false, true] {
17188 let mut f = summary_fixture();
17189 if !review_only {
17190 f.runner.state.candidates[0].agent = "alpha".into();
17191 }
17192 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17193 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17194 std::fs::write(&f.reply, "no usable description").unwrap();
17195 let failed = posted(&mut f).await;
17196 assert!(
17197 failed
17198 .body
17199 .contains("No description of what this change does could be generated")
17200 );
17201 assert!(failed.body.is_ascii());
17202 assert!(f.runner.state.github_text.is_none());
17203 assert!(f.runner.state.change_summary.is_none());
17204 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17205 let evidence = crate::pr_outcome::summary_evidence(
17206 &f.runner.state,
17207 crate::pr_outcome::FinalHead::At(&head),
17208 );
17209 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17210 assert!(
17211 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17212 .exists()
17213 );
17214 f.runner.state =
17215 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17216 std::fs::write(
17217 &f.reply,
17218 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17219 )
17220 .unwrap();
17221 let retried = posted(&mut f).await;
17222 assert!(retried.body.contains("Adds the committed own file."));
17223 assert_eq!(
17224 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17225 "日本語の修正報告"
17226 );
17227 }
17228 }
17229
17230 #[tokio::test]
17231 async fn same_head_verification_and_findings_invalidate_public_prose() {
17232 let mut f = summary_fixture();
17233 posted(&mut f).await;
17234 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17235 let original = f
17236 .runner
17237 .state
17238 .change_summary
17239 .as_ref()
17240 .unwrap()
17241 .evidence
17242 .clone();
17243 ok_gate(&mut f.runner.state, Some(&head));
17244 std::fs::write(
17245 &f.reply,
17246 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17247 )
17248 .unwrap();
17249 let verified = posted(&mut f).await;
17250 assert!(verified.body.contains("now verified"));
17251 assert_ne!(
17252 f.runner.state.change_summary.as_ref().unwrap().evidence,
17253 original
17254 );
17255 let mut round = round_with_notes(1, Some("内部報告"));
17256 round.reviews.push(ReviewRecord {
17257 attempts: 0,
17258 reviewer: 1,
17259 agent: "alpha".into(),
17260 summary: "内部レビュー".into(),
17261 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17262 vote: None,
17263 failed: None,
17264 duration_ms: 0,
17265 });
17266 round
17267 .fix
17268 .as_mut()
17269 .unwrap()
17270 .rejected
17271 .push(crate::verdict::Rejection {
17272 id: "R1-1-1".into(),
17273 why: "呼び出し元から到達できない".into(),
17274 });
17275 f.runner.state.reviews = vec![round];
17276 let omitted = posted(&mut f).await;
17278 assert!(omitted.body.contains("No description"));
17279 assert!(omitted.body.contains("R1-1-1"));
17280 assert!(!omitted.body.contains("到達"));
17281 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17282 let described = posted(&mut f).await;
17283 assert!(described.body.contains("unreachable from callers"));
17284 assert!(described.body.contains("R1-1-1"));
17285 assert!(!described.body.contains("到達"));
17286 let evidence = f
17287 .runner
17288 .state
17289 .change_summary
17290 .as_ref()
17291 .unwrap()
17292 .evidence
17293 .clone();
17294 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17295 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17296 assert!(posted(&mut f).await.body.contains("rationale updated"));
17297 assert_ne!(
17298 f.runner.state.change_summary.as_ref().unwrap().evidence,
17299 evidence
17300 );
17301 f.runner
17303 .state
17304 .change_summary
17305 .as_mut()
17306 .unwrap()
17307 .evidence
17308 .clear();
17309 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17310 assert!(posted(&mut f).await.body.contains("current specification"));
17311 }
17312
17313 #[tokio::test]
17314 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17315 let mut f = summary_fixture();
17316 let m = posted(&mut f).await;
17317 assert_eq!(m.title, "feat(own): add the own file");
17318 assert!(
17319 m.body
17320 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17321 "{}",
17322 m.body
17323 );
17324 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17325 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17326 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17327 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17328 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17329 let kept = f.runner.state.change_summary.as_ref().unwrap();
17330 assert_eq!(
17331 (kept.head.as_str(), kept.base.as_str()),
17332 (head.as_str(), base.as_str())
17333 );
17334 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17335 }
17336
17337 #[tokio::test]
17338 async fn review_only_summary_receives_the_configured_github_language() {
17339 let mut f = summary_fixture();
17340 f.runner.state.config.graph.github_language = "ja".into();
17341 let captured = f.repo.join("summary-prompt.txt");
17342 let author = &mut f.runner.state.config.agents[0];
17343 author.command = vec![
17344 "sh".into(),
17345 "-c".into(),
17346 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17347 ];
17348 author.env.insert(
17349 "PROMPT_FILE".into(),
17350 captured.to_string_lossy().into_owned(),
17351 );
17352 std::fs::write(
17353 &f.reply,
17354 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17355 )
17356 .unwrap();
17357 let m = posted(&mut f).await;
17358 let prompt = std::fs::read_to_string(captured).unwrap();
17359 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17360 assert!(!prompt.contains("in English"));
17361 assert_eq!(m.title, "feat(own): ファイルを追加");
17362 assert!(m.body.contains("own.txt を追加します。"));
17363 }
17364
17365 #[tokio::test]
17366 async fn review_fixes_come_after_the_whole_change_summary() {
17367 let mut f = summary_fixture();
17368 f.runner.state.reviews = vec![
17369 round_with_notes(1, Some("handled the PATH case")),
17370 round_with_notes(2, None),
17371 ];
17372 let m = posted(&mut f).await;
17373 let summary = m.body.find("## Summary").expect("summary");
17374 let fixes = m.body.find("## Review rounds").expect("review rounds");
17375 assert!(summary < fixes, "{}", m.body);
17376 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17377 assert!(m.body.contains("## Final verification"), "{}", m.body);
17378 }
17379
17380 #[tokio::test]
17381 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17382 let mut f = summary_fixture();
17383 let first = posted(&mut f).await;
17384 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17385 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17387 let again = posted(&mut f).await;
17388 assert_eq!((again.title, again.body), (first.title, first.body));
17389
17390 let g = |args: &[&str]| {
17391 let out = std::process::Command::new("git")
17392 .args(args)
17393 .current_dir(&f.repo)
17394 .quiet()
17395 .output()
17396 .expect("spawn git");
17397 assert!(out.status.success(), "git {args:?}");
17398 };
17399 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17400 g(&["add", "-A"]);
17401 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17402 let second = posted(&mut f).await;
17403 let kept = f.runner.state.change_summary.clone().unwrap();
17404 assert_ne!(kept.head, first_head);
17405 assert!(second.body.contains("Not read yet."), "{}", second.body);
17406 assert!(!second.body.contains("so the first summary is read"));
17407 assert_eq!(second.title, "feat(own): other");
17408 }
17409
17410 #[tokio::test]
17411 async fn an_unavailable_summary_is_stated_and_never_invented() {
17412 let mut f = summary_fixture();
17413 std::fs::write(&f.reply, "no title line here").unwrap();
17414 let m = posted(&mut f).await;
17415 assert!(
17416 m.body
17417 .contains("No description of what this change does could be generated"),
17418 "{}",
17419 m.body
17420 );
17421 assert!(m.body.contains("internal reports remain in local records"));
17422 assert!(f.runner.state.change_summary.is_none());
17423 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17424 let winner = f.runner.state.candidates[0].clone();
17426 let preview = f
17427 .runner
17428 .guarded_pr_message(&winner, None, false)
17429 .await
17430 .unwrap()
17431 .unwrap();
17432 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17433 }
17434
17435 #[test]
17436 fn pr_message_review_single_commit_uses_its_subject() {
17437 let state = review_state(&["feat(nats): per-role user"]);
17438 let m = pr_message(&state, 'A');
17439 assert_eq!(m.title, "feat(nats): per-role user");
17440 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17441 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17442 }
17443
17444 #[test]
17445 fn pr_message_review_multi_commit_takes_the_oldest() {
17446 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17447 let m = pr_message(&state, 'A');
17448 assert_eq!(m.title, "feat: the change");
17449 for s in ["feat: the change", "fix: typo", "fix: again"] {
17450 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17451 }
17452 }
17453
17454 #[test]
17455 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17456 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17457 let mut state = review_state(&[first, "fix: later fixup"]);
17458 state.candidates[0].branch = "機能/ログイン".to_owned();
17459 let m = pr_message(&state, 'A');
17460 assert!(
17461 m.title.starts_with("chore: land candidate A of run"),
17462 "{}",
17463 m.title
17464 );
17465 }
17466 }
17467
17468 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17469 BranchFacts {
17470 commits: commits
17471 .iter()
17472 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17473 .collect(),
17474 stat: stat.to_owned(),
17475 }
17476 }
17477
17478 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17479 let mut r = review_round(true, 0, 1, 1, true, true);
17480 r.round = round;
17481 r.fix = notes.map(|n| FixRecord {
17482 agent: "fixer".to_owned(),
17483 addressed: Vec::new(),
17484 rejected: Vec::new(),
17485 notes: n.to_owned(),
17486 committed: true,
17487 failed: None,
17488 duration_ms: 0,
17489 continuation: None,
17490 });
17491 r
17492 }
17493
17494 #[test]
17495 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17496 let state = review_state(&["ignored"]);
17497 let f = facts(
17498 &[
17499 (
17500 "fix(login): resolve PATH on macOS",
17501 "Login shells skip rc files.",
17502 ),
17503 ("fix: address review", ""),
17504 ],
17505 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17506 );
17507 let m = pr_message_with(&state, 'A', Some(&f));
17508 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17509 assert!(
17510 m.body
17511 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17512 "{}",
17513 m.body
17514 );
17515 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17516 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17517 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17518 for banned in [
17519 "Review the work already",
17520 "no task statement",
17521 "Original task",
17522 ] {
17523 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17524 }
17525 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17526 }
17527
17528 #[test]
17529 fn pr_message_review_truncates_a_huge_first_commit_body() {
17530 let state = review_state(&["ignored"]);
17531 let f = facts(
17532 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17533 "s",
17534 );
17535 let m = pr_message_with(&state, 'A', Some(&f));
17536 assert!(m.body.len() < 30_000, "{}", m.body.len());
17537 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17538 assert!(
17539 m.body.contains("- ... 1 more commit(s)"),
17540 "{}",
17541 m.body.len()
17542 );
17543 assert!(m.body.ends_with("magi:candidate-a\n"));
17544 }
17545
17546 #[test]
17547 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17548 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17549 assert_eq!(m.title, "feat: x");
17550 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17551 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17552 assert!(!m.body.contains("no task statement"), "{}", m.body);
17553 }
17554
17555 #[test]
17556 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17557 let mut state = review_state(&["日本語の件名"]);
17558 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17559 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17560 state.candidates[0].branch = "機能/ログイン".to_owned();
17561 assert!(
17562 pr_message(&state, 'A')
17563 .title
17564 .starts_with("chore: land candidate A")
17565 );
17566 }
17567
17568 #[test]
17569 fn pr_message_review_fixes_survive_a_clean_final_round() {
17570 let mut state = review_state(&["feat: x"]);
17571 state.reviews = vec![
17572 round_with_notes(1, Some("handled the PATH case")),
17573 round_with_notes(2, None),
17574 ];
17575 let body = pr_message(&state, 'A').body;
17576 assert!(
17577 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17578 "{body}"
17579 );
17580 assert!(body.contains("### Round 1"), "{body}");
17581
17582 state.reviews = vec![
17583 round_with_notes(1, Some("first fix")),
17584 round_with_notes(2, Some("")),
17585 round_with_notes(3, Some("second fix")),
17586 round_with_notes(4, None),
17587 ];
17588 let body = pr_message(&state, 'A').body;
17589 assert!(
17590 body.contains("### Round 1") && !body.contains("first fix"),
17591 "{body}"
17592 );
17593 assert!(
17594 body.contains("### Round 3") && !body.contains("second fix"),
17595 "{body}"
17596 );
17597 assert!(body.contains("### Round 2"), "{body}");
17598 }
17599
17600 const FINAL: &str = "abcdef1234567890";
17601
17602 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17603 let mut r = round_with_notes(round, None);
17604 r.verified_head = verified.map(str::to_owned);
17605 r
17606 }
17607
17608 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17609 state.gate = vec![CommandOutcome {
17610 command: "gate".to_owned(),
17611 code: Some(0),
17612 output_tail: String::new(),
17613 duration_ms: 0,
17614 resource_blocked: false,
17615 }];
17616 state.gate_ran = true;
17617 state.gate_head = head.map(str::to_owned);
17618 }
17619
17620 fn body_at(state: &RunState) -> String {
17621 pr_message_raw(
17622 state,
17623 'A',
17624 None,
17625 crate::pr_outcome::FinalHead::At(FINAL),
17626 ChangeNote::NotAsked,
17627 )
17628 .body
17629 }
17630
17631 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17632
17633 #[test]
17634 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17635 let mut state = review_state(&["feat: x"]);
17636 state.reviews = vec![
17637 round_with_notes(1, Some(SANDBOXED)),
17638 passing_round(2, Some(FINAL)),
17639 ];
17640 ok_gate(&mut state, Some(FINAL));
17641 let body = body_at(&state);
17642 assert!(!body.contains("Handled the PATH case."), "{body}");
17643 assert!(!body.contains("could not commit"), "{body}");
17644 assert!(!body.contains("could not run"), "{body}");
17645 assert!(
17646 body.contains("The fix is committed on the branch"),
17647 "{body}"
17648 );
17649 assert!(!body.contains("sandbox"), "{body}");
17650 assert!(
17651 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17652 "{body}"
17653 );
17654 assert!(
17655 body.contains("Gate (`verify.gate`): passed on the final commit"),
17656 "{body}"
17657 );
17658 assert!(
17660 state.reviews[0]
17661 .fix
17662 .as_ref()
17663 .unwrap()
17664 .notes
17665 .contains("could not commit")
17666 );
17667 }
17668
17669 #[test]
17670 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17671 let mut state = review_state(&["feat: x"]);
17672 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17673 r1.fix.as_mut().unwrap().committed = false;
17674 let mut r2 = passing_round(2, Some(FINAL));
17675 r2.e2e[0].code = Some(1);
17676 state.reviews = vec![r1, r2];
17677 let body = body_at(&state);
17678 assert!(!body.contains("could not commit"), "{body}");
17679 assert!(!body.contains("could not run"), "{body}");
17680 assert!(body.contains("This round produced no commit"), "{body}");
17681 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17682 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17683 assert!(!body.contains("passed"), "{body}");
17684 }
17685
17686 #[test]
17687 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17688 let mut state = review_state(&["feat: x"]);
17689 state.reviews = vec![
17690 round_with_notes(1, Some(SANDBOXED)),
17691 passing_round(2, Some("0123456789abcdef")),
17692 ];
17693 ok_gate(&mut state, Some("0123456789abcdef"));
17694 let body = body_at(&state);
17695 assert!(!body.contains("HTTP tests could not run"), "{body}");
17696 assert!(!body.contains("could not commit"), "{body}");
17698 assert!(
17699 body.contains("passed on an earlier commit (`0123456`)"),
17700 "{body}"
17701 );
17702 assert!(!body.contains("passed on the final commit"), "{body}");
17703
17704 ok_gate(&mut state, None);
17706 let body = body_at(&state);
17707 assert!(
17708 body.contains("passed on a commit that was not recorded"),
17709 "{body}"
17710 );
17711 }
17712
17713 #[test]
17714 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17715 let state = state_with_summary(
17716 "add retries to the client",
17717 "TITLE: feat: retries\n\nDid it.",
17718 );
17719 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17720 assert_eq!(m.title, "feat: retries");
17721 assert!(
17722 m.body.contains("<summary>Original task</summary>"),
17723 "{}",
17724 m.body
17725 );
17726 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17727 assert!(
17728 m.body
17729 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17730 "{}",
17731 m.body
17732 );
17733 }
17734
17735 #[test]
17736 fn pr_message_review_bounds_a_long_english_subject() {
17737 let long = format!("feat: {}", "word ".repeat(100));
17738 let m = pr_message(&review_state(&[&long]), 'A');
17739 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17740 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17741 }
17742
17743 #[test]
17744 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17745 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17746 let m = pr_message(&state, 'A');
17747 assert_eq!(m.title, "add retries");
17748 assert!(m.body.contains("<summary>Original task</summary>"));
17749 assert!(!m.body.contains("Commits under review"));
17750 assert_eq!(landing_subject_source(&state), state.instruction);
17751 }
17752
17753 #[test]
17754 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17755 let state = review_state(&["feat: the change", "fix: typo"]);
17756 let source = landing_subject_source(&state);
17757 assert_eq!(land::merge_subject("", &source), "feat: the change");
17758 assert_eq!(
17759 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17760 "feat: the change"
17761 );
17762 assert_eq!(
17764 land::merge_subject("feat: renamed by hand", &source),
17765 "feat: renamed by hand"
17766 );
17767 let blank = review_state(&["日本語"]);
17768 assert!(
17769 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17770 );
17771 }
17772
17773 #[test]
17774 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17775 let state = review_state(&["feat: the change"]);
17776 let source = landing_subject_source(&state);
17777 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17778 assert_eq!(
17779 land::merge_subject(landing_title(&state, old), &source),
17780 "feat: the change"
17781 );
17782 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17783 let task = state_with_summary("add retries", "");
17784 assert_eq!(landing_title(&task, old), old);
17785 }
17786
17787 #[test]
17788 fn pr_message_describes_the_change_not_the_task() {
17789 let state = state_with_summary(
17790 "今回やってほしいこと: results projector を直す",
17791 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17792 );
17793 let m = pr_message(&state, 'A');
17794 assert_eq!(m.title, "fix(web): batch the runs list reads");
17795 assert!(
17796 m.body.starts_with("## Summary\n\n- reads run.json once"),
17797 "{}",
17798 m.body
17799 );
17800 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17801 let task_at = m.body.find("今回やってほしいこと").unwrap();
17802 let details_at = m.body.find("<details>").unwrap();
17803 assert!(
17804 details_at < task_at,
17805 "the task lives inside <details>: {}",
17806 m.body
17807 );
17808 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17809 assert!(m.body.contains("magi:candidate-a"));
17810 }
17811
17812 #[test]
17813 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17814 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17815 let m = pr_message(&state, 'A');
17816 assert_eq!(m.title, "add retries");
17817 assert!(
17818 m.body.contains("## Summary\n\n- did some things"),
17819 "{}",
17820 m.body
17821 );
17822
17823 let none = RunState::new(
17824 PathBuf::from("/repo"),
17825 "main".to_owned(),
17826 "abc1234".to_owned(),
17827 "add retries".to_owned(),
17828 Config::default(),
17829 );
17830 let m = pr_message(&none, 'A');
17831 assert_eq!(m.title, "add retries");
17832 assert!(!m.body.contains("## Summary"), "{}", m.body);
17833 }
17834
17835 #[test]
17836 fn pr_message_refuses_the_candidate_commit_subject() {
17837 for bad in [
17838 "TITLE: magi: candidate A (uncommitted work)",
17839 "TITLE: chore: stuff (uncommitted work)",
17840 "TITLE: ",
17841 ] {
17842 let state = state_with_summary("add retries", bad);
17843 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17844 }
17845 }
17846
17847 #[test]
17848 fn pr_message_bounds_a_very_long_task_and_title() {
17849 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17850 let state = state_with_summary(&long, "- nothing");
17851 let m = pr_message(&state, 'A');
17852 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17853 assert!(!m.title.contains('\n'));
17854
17855 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17856 let m = pr_message(&state, 'A');
17857 assert!(m.title.starts_with("feat: "));
17858 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17859 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17860 }
17861
17862 fn long_title_of(instruction: &str) -> String {
17863 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17864 }
17865
17866 #[test]
17867 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17868 let first = "Make the landing path keep a readable title for long tasks";
17869 let line = format!(
17870 "{first}. {}",
17871 "And then keep going with more words ".repeat(20)
17872 );
17873 let t = long_title_of(&line);
17874 assert_eq!(t, first);
17875 assert!(!t.starts_with("chore: land"));
17876 }
17877
17878 #[test]
17879 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17880 let line = "word ".repeat(200);
17881 let t = long_title_of(&line);
17882 assert!(t.ends_with("word..."), "{t}");
17883 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17884 }
17885
17886 #[test]
17887 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17888 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17889 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17890 }
17891 }
17892
17893 #[test]
17894 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17895 let mut state = state_with_summary("再試行を追加する", "- nothing");
17896 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17897 state.config.graph.github_language = "ja".into();
17898 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17899 }
17900
17901 #[test]
17902 fn pr_message_title_limit_is_exact() {
17903 let at = "a".repeat(PR_TITLE_MAX);
17904 assert_eq!(long_title_of(&at), at);
17905 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17906 assert!(over.ends_with("..."), "{over}");
17907 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17908 }
17909
17910 #[test]
17911 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17912 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17913 let t = long_title_of(&line);
17914 assert!(t.ends_with("..."), "{t}");
17915 assert!(t.is_ascii(), "{t}");
17916 }
17917
17918 #[test]
17919 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17920 let line = format!(
17921 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17922 "plus more filler words ".repeat(20)
17923 );
17924 let t = long_title_of(&line);
17925 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17926 }
17927
17928 #[test]
17929 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17930 let line = format!("\n\n# {}", "title words ".repeat(40));
17931 let state = state_with_summary(&line, "- nothing");
17932 let m = pr_message(&state, 'A');
17933 assert!(m.title.starts_with("title words"), "{}", m.title);
17934 assert_eq!(
17935 land::merge_subject(&m.title, &landing_subject_source(&state)),
17936 m.title
17937 );
17938 assert_eq!(
17940 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17941 "feat: renamed by hand"
17942 );
17943 }
17944
17945 #[tokio::test]
17946 async fn github_text_rewrite_is_bounded_and_falls_back() {
17947 crate::run::pin_test_home();
17948 for (reply, accepted) in [
17949 (
17950 "TITLE: fix: retries\nAdd retries for failed requests.",
17951 true,
17952 ),
17953 (
17954 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17955 false,
17956 ),
17957 (
17958 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17959 true,
17960 ),
17961 (
17962 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17963 true,
17964 ),
17965 (
17966 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17967 true,
17968 ),
17969 (
17970 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17971 false,
17972 ),
17973 ("TITLE: fix: retries\n", false),
17974 ] {
17975 let dir = tempfile::tempdir().unwrap();
17976 let mut runner = runner_at(RunStatus::Gating);
17977 runner.state = state_with_summary(
17978 "add retries",
17979 "TITLE: fix: retries\n日本語の説明を書きます。",
17980 );
17981 runner.state.repo = dir.path().to_owned();
17982 runner.state.candidates[0].worktree = dir.path().to_owned();
17983 let mut author = spec("alpha");
17984 author.command = vec![
17985 "sh".into(),
17986 "-c".into(),
17987 "printf '%s' \"$REWRITE_REPLY\"".into(),
17988 ];
17989 author.env.insert("REWRITE_REPLY".into(), reply.into());
17990 runner.state.config.agents = vec![author];
17991 seed_public_summary(&mut runner);
17992 let winner = runner.state.candidates[0].clone();
17993 let first = runner
17994 .guarded_pr_message(&winner, None, true)
17995 .await
17996 .unwrap();
17997 let message = if accepted {
17998 first.expect("a passing rewrite never asks")
17999 } else {
18000 assert!(first.is_none());
18002 assert!(runner.state.parked);
18003 let store = ask::Questions::open();
18004 let asked: Vec<_> = store
18005 .list()
18006 .into_iter()
18007 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18008 .collect();
18009 assert_eq!(asked.len(), 1);
18010 assert!(!asked[0].detail.contains("secret"));
18011 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18012 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18013 assert_eq!(
18014 asked[0]
18015 .choices
18016 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18017 rewrite.is_some()
18018 );
18019 store
18022 .update(&asked[0].id, |q| {
18023 q.abandon("test");
18024 Ok(())
18025 })
18026 .unwrap();
18027 let again = runner
18028 .guarded_pr_message(&winner, None, true)
18029 .await
18030 .unwrap()
18031 .expect("an abandoned question falls back");
18032 let n = store
18033 .list()
18034 .into_iter()
18035 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18036 .count();
18037 assert_eq!(n, 1, "asked once per rejected text");
18038 again
18039 };
18040 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18041 assert!(!message.body.contains("secret"));
18042 assert!(!message.body.contains("prefix"));
18043 assert!(!message.title.contains("secret"));
18044 assert_eq!(
18045 message.body.contains("Add retries for failed requests."),
18046 accepted
18047 );
18048 assert_eq!(
18049 runner.state.seats["github-text-rewrite"].turns, 1,
18050 "only one rewrite invocation"
18051 );
18052 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18053 if accepted {
18054 assert!(runner.state.github_text.is_none());
18055 assert!(!runner.state.parked);
18056 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18057 assert!(message.body.contains("## Final verification"));
18058 assert!(message.body.contains("Evidence: final diff"));
18059 assert!(
18060 message
18061 .body
18062 .contains(&format!("magi:run/{}", runner.state.id))
18063 );
18064 }
18065 if !accepted {
18066 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18067 assert!(
18068 message
18069 .body
18070 .contains(&format!("magi:run/{}", runner.state.id))
18071 );
18072 }
18073 }
18074 }
18075
18076 fn scripted(id: &str, reply: &str) -> AgentSpec {
18077 let mut a = spec(id);
18078 a.command = vec![
18079 "sh".into(),
18080 "-c".into(),
18081 "printf '%s' \"$REWRITE_REPLY\"".into(),
18082 ];
18083 a.env.insert("REWRITE_REPLY".into(), reply.into());
18084 a
18085 }
18086
18087 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18088 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18089
18090 fn seed_public_summary(runner: &mut Runner) {
18092 let repo = &runner.state.repo;
18093 let g = |args: &[&str]| {
18094 let out = std::process::Command::new("git")
18095 .args(args)
18096 .current_dir(repo)
18097 .quiet()
18098 .output()
18099 .unwrap();
18100 assert!(out.status.success(), "git {args:?}");
18101 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18102 };
18103 g(&["init", "-q", "-b", "main"]);
18104 g(&[
18105 "-c",
18106 "user.name=Test",
18107 "-c",
18108 "user.email=test@example.invalid",
18109 "commit",
18110 "-q",
18111 "--allow-empty",
18112 "-m",
18113 "feat: fixture",
18114 ]);
18115 let head = g(&["rev-parse", "HEAD"]);
18116 g(&["update-ref", "refs/remotes/origin/main", &head]);
18117 let evidence = crate::pr_outcome::summary_evidence(
18118 &runner.state,
18119 crate::pr_outcome::FinalHead::At(&head),
18120 );
18121 let generated = &runner.state.candidates[0].summary;
18122 runner.state.change_summary = Some(crate::run::ChangeSummary {
18123 base: head.clone(),
18124 head,
18125 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18126 title: summary_title(generated).unwrap(),
18127 body: summary_without_title(generated),
18128 });
18129 }
18130
18131 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18132 let mut runner = runner_at(RunStatus::Gating);
18133 runner.state = state_with_summary("add retries", JA_SUMMARY);
18134 if review_only {
18135 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18136 }
18137 runner.state.repo = dir.to_owned();
18138 runner.state.candidates[0].worktree = dir.to_owned();
18139 runner.state.config.agents = agents;
18140 seed_public_summary(&mut runner);
18141 runner
18142 }
18143
18144 fn gate_questions(runner: &Runner) -> usize {
18145 ask::Questions::open()
18146 .list()
18147 .into_iter()
18148 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18149 .count()
18150 }
18151
18152 #[tokio::test]
18153 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18154 crate::run::pin_test_home();
18155 let dir = tempfile::tempdir().unwrap();
18156 let mut runner = rewrite_runner(
18157 dir.path(),
18158 true,
18159 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18160 );
18161 runner.state.config.roles.language_judge =
18162 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18163 let winner = runner.state.candidates[0].clone();
18164 let msg = runner
18165 .guarded_pr_message(&winner, None, true)
18166 .await
18167 .unwrap()
18168 .expect("an English rewrite posts without asking");
18169 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18170 assert!(msg.body.contains("<details>kept</details>"));
18171 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18172 assert_eq!(gate_questions(&runner), 0);
18173 assert!(runner.state.github_text.is_none());
18174 assert!(!runner.state.seats.contains_key("impl-A"));
18175 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18176 }
18177
18178 #[tokio::test]
18179 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18180 crate::run::pin_test_home();
18181 let dir = tempfile::tempdir().unwrap();
18182 let mut runner = rewrite_runner(
18183 dir.path(),
18184 false,
18185 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18186 );
18187 runner.state.config.roles.language_judge =
18188 Some(crate::config::AgentChoice::One("beta".into()));
18189 runner.state.seats.insert(
18190 "impl-A".into(),
18191 SeatState::new("impl-A", "alpha", runner.state.seed),
18192 );
18193 let winner = runner.state.candidates[0].clone();
18194 let msg = runner
18195 .guarded_pr_message(&winner, None, true)
18196 .await
18197 .unwrap()
18198 .expect("rewritten");
18199 assert!(msg.body.contains("<details>kept</details>"));
18200 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18201 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18202 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18203 }
18204
18205 #[tokio::test]
18206 async fn rewrite_reuses_a_resumable_implementer_session() {
18207 crate::run::pin_test_home();
18208 let dir = tempfile::tempdir().unwrap();
18209 let mut runner = rewrite_runner(
18210 dir.path(),
18211 false,
18212 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18213 );
18214 runner.state.config.roles.language_judge =
18215 Some(crate::config::AgentChoice::One("beta".into()));
18216 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18217 seat.turns = 1;
18218 runner.state.seats.insert("impl-A".into(), seat);
18219 let winner = runner.state.candidates[0].clone();
18220 runner
18221 .guarded_pr_message(&winner, None, true)
18222 .await
18223 .unwrap()
18224 .expect("rewritten");
18225 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18226 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18227 }
18228
18229 #[tokio::test]
18230 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18231 crate::run::pin_test_home();
18232 let dir = tempfile::tempdir().unwrap();
18233 let mut broken = spec("bad");
18234 broken.command = vec!["false".into()];
18235 let mut runner = rewrite_runner(
18236 dir.path(),
18237 true,
18238 vec![
18239 broken,
18240 scripted("noformat", "Add retries."),
18241 scripted("good", EN_REWRITE),
18242 ],
18243 );
18244 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18245 "bad".into(),
18246 "noformat".into(),
18247 "good".into(),
18248 ]));
18249 let winner = runner.state.candidates[0].clone();
18250 let msg = runner
18251 .guarded_pr_message(&winner, None, true)
18252 .await
18253 .unwrap()
18254 .expect("the third agent rewrites");
18255 assert!(msg.body.contains("`src/net.rs`"));
18256 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18257 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18258 }
18259
18260 #[tokio::test]
18261 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18262 crate::run::pin_test_home();
18263 for (agents, role, saved) in [
18265 (
18266 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18267 Some("beta"),
18268 true,
18269 ),
18270 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18271 (Vec::new(), None, false),
18272 ] {
18273 let dir = tempfile::tempdir().unwrap();
18274 let mut runner = rewrite_runner(dir.path(), true, agents);
18275 runner.state.config.roles.language_judge =
18276 role.map(|r| crate::config::AgentChoice::One(r.into()));
18277 let winner = runner.state.candidates[0].clone();
18278 let first = runner
18279 .guarded_pr_message(&winner, None, true)
18280 .await
18281 .unwrap();
18282 assert!(first.is_none(), "withheld text parks the run");
18283 assert!(runner.state.parked);
18284 assert_eq!(gate_questions(&runner), 1);
18285 let g = runner.state.github_text.as_ref().unwrap();
18286 assert_eq!(g.rewrite.is_some(), saved);
18287 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18288 }
18289 }
18290
18291 #[tokio::test]
18292 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18293 crate::run::pin_test_home();
18294 let dir = tempfile::tempdir().unwrap();
18295 let mut runner = runner_at(RunStatus::Gating);
18296 runner.state =
18297 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18298 runner.state.repo = dir.path().to_owned();
18299 runner.state.candidates[0].worktree = dir.path().to_owned();
18300 let mut author = spec("alpha");
18301 author.command = vec![
18302 "sh".into(),
18303 "-c".into(),
18304 "printf '%s' \"$REWRITE_REPLY\"".into(),
18305 ];
18306 author.env.insert(
18307 "REWRITE_REPLY".into(),
18308 format!(
18309 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18310 "English sentence. ".repeat(600)
18311 ),
18312 );
18313 runner.state.config.agents = vec![author];
18314 seed_public_summary(&mut runner);
18315 let winner = runner.state.candidates[0].clone();
18316 assert!(
18317 runner
18318 .guarded_pr_message(&winner, None, true)
18319 .await
18320 .unwrap()
18321 .is_none()
18322 );
18323 let record = runner.state.github_text.clone().unwrap();
18324 let rewrite = record.rewrite.clone().unwrap();
18325 assert_eq!(rewrite.failed, ["body-language"]);
18326 assert!(
18327 rewrite
18328 .body
18329 .ends_with(&format!("magi:run/{}", runner.state.id))
18330 );
18331 let store = ask::Questions::open();
18332 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18333 assert!(
18334 q.choices
18335 .contains(&crate::github_text::USE_REWRITE.to_owned())
18336 );
18337 for label in [
18338 "ORIGINAL",
18339 "FALLBACK",
18340 "REWRITE",
18341 "Rules still failed: body-language",
18342 "truncated",
18343 ] {
18344 assert!(q.detail.contains(label), "{label}");
18345 }
18346 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18347 &runner.state,
18348 &crate::github_text::artifact_name(&record.fingerprint),
18349 ))
18350 .unwrap();
18351 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18352 runner.state =
18354 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18355 store
18356 .update(&q.id, |q| {
18357 q.say("まだ日本語")?;
18358 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18359 })
18360 .unwrap();
18361 assert!(
18362 runner
18363 .guarded_pr_message(&winner, None, true)
18364 .await
18365 .unwrap()
18366 .is_none()
18367 );
18368 let retry = runner.state.github_text.as_ref().unwrap();
18369 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18370 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18371 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18372 assert!(
18373 q.posting_comparison
18374 .as_ref()
18375 .unwrap()
18376 .matches_detail(&q.detail)
18377 );
18378 store
18379 .update(&q.id, |q| {
18380 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18381 })
18382 .unwrap();
18383 let message = runner
18384 .guarded_pr_message(&winner, None, true)
18385 .await
18386 .unwrap()
18387 .unwrap();
18388 assert_eq!(message.title, rewrite.title);
18389 assert_eq!(message.body, rewrite.body);
18390 assert_eq!(
18391 runner.state.github_text.as_ref().unwrap().choice,
18392 Some(GithubTextChoice::Rewrite)
18393 );
18394 let again = runner
18395 .guarded_pr_message(&winner, None, true)
18396 .await
18397 .unwrap()
18398 .unwrap();
18399 assert_eq!(again.title, message.title);
18400 assert_eq!(again.body, message.body);
18401 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18402 assert!(
18403 !runner
18404 .state
18405 .events
18406 .iter()
18407 .any(|e| e.message.contains(&rewrite.title)
18408 || e.message.contains("English sentence."))
18409 );
18410 }
18411
18412 #[tokio::test]
18413 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18414 crate::run::pin_test_home();
18415 let dir = tempfile::tempdir().unwrap();
18416 let mut runner = runner_at(RunStatus::Gating);
18417 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18418 runner.state.repo = dir.path().to_owned();
18419 runner.state.candidates[0].worktree = dir.path().to_owned();
18420 seed_public_summary(&mut runner);
18421 let winner = runner.state.candidates[0].clone();
18422 assert!(
18423 runner
18424 .guarded_pr_message(&winner, None, true)
18425 .await
18426 .unwrap()
18427 .is_none()
18428 );
18429 let record = runner.state.github_text.as_ref().unwrap();
18430 assert!(record.rewrite.is_none());
18431 let mut old = serde_json::to_value(record).unwrap();
18433 old.as_object_mut().unwrap().remove("rewrite");
18434 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18435 assert!(old.rewrite.is_none());
18436 let id = record.question.clone().unwrap();
18437 ask::Questions::open()
18438 .update(&id, |q| {
18439 q.choices.push(crate::github_text::USE_REWRITE.into());
18440 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18441 })
18442 .unwrap();
18443 let message = runner
18444 .guarded_pr_message(&winner, None, true)
18445 .await
18446 .unwrap()
18447 .unwrap();
18448 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18449 assert_eq!(
18450 runner.state.github_text.as_ref().unwrap().choice,
18451 Some(GithubTextChoice::Fallback)
18452 );
18453 }
18454
18455 #[tokio::test]
18456 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18457 crate::run::pin_test_home();
18458 let dir = tempfile::tempdir().unwrap();
18459 let mut runner = runner_at(RunStatus::Gating);
18460 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18461 runner.state.repo = dir.path().to_owned();
18462 runner.state.candidates[0].worktree = dir.path().to_owned();
18463 seed_public_summary(&mut runner);
18464 let winner = runner.state.candidates[0].clone();
18465 assert!(
18466 runner
18467 .guarded_pr_message(&winner, None, true)
18468 .await
18469 .unwrap()
18470 .is_none()
18471 );
18472 let store = ask::Questions::open();
18473 let find = |run: &str| {
18474 store
18475 .list()
18476 .into_iter()
18477 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18478 .unwrap()
18479 };
18480 let run = runner.state.id.clone();
18481 let q = find(&run);
18482 let fp = runner
18483 .state
18484 .github_text
18485 .as_ref()
18486 .unwrap()
18487 .fingerprint
18488 .clone();
18489 let name = crate::github_text::artifact_name(&fp);
18490 let path = crate::run::artifact_path(&runner.state, &name);
18491 assert!(q.detail.contains(&path.display().to_string()));
18492 assert!(q.detail.contains("title-language: title"));
18493 let saved = std::fs::read_to_string(&path).expect("artifact written");
18494 assert!(saved.contains("日本語のタイトル"));
18495 assert!(q.cwd.is_none());
18496 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18497 let say = |id: &str, text: &str, choice: &str| {
18499 store
18500 .update(id, |q| {
18501 q.thread.push(ask::Turn {
18502 who: ask::Who::Operator,
18503 body: text.to_owned(),
18504 at: jiff::Timestamp::now(),
18505 note: None,
18506 });
18507 q.answer(ask::Answer::Choice(choice.to_owned()))
18508 })
18509 .unwrap();
18510 };
18511 say(&q.id, "まだ日本語", "use my text");
18512 assert!(
18513 runner
18514 .guarded_pr_message(&winner, None, true)
18515 .await
18516 .unwrap()
18517 .is_none()
18518 );
18519 let second = find(&run);
18520 assert_ne!(second.id, q.id);
18521 assert!(second.detail.contains("did not pass"));
18522 assert!(second.detail.contains("日本語のタイトル"));
18525 assert!(!second.detail.contains("まだ日本語"));
18526 assert!(second.detail.contains(&path.display().to_string()));
18527 say(&second.id, "fix: retry failed requests", "use my text");
18528 let message = runner
18529 .guarded_pr_message(&winner, None, true)
18530 .await
18531 .unwrap()
18532 .expect("a vetted replacement is posted");
18533 assert_eq!(message.title, "fix: retry failed requests");
18534 assert_eq!(
18535 runner
18536 .state
18537 .github_text
18538 .as_ref()
18539 .unwrap()
18540 .chosen_title
18541 .as_deref(),
18542 Some("fix: retry failed requests")
18543 );
18544 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18545 let again = runner
18547 .guarded_pr_message(&winner, None, true)
18548 .await
18549 .unwrap()
18550 .unwrap();
18551 assert_eq!(again.title, message.title);
18552 assert_eq!(
18553 store
18554 .list()
18555 .into_iter()
18556 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18557 .count(),
18558 2
18559 );
18560 }
18561
18562 #[test]
18563 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18564 let mut state = state_with_summary(
18568 "add retries",
18569 "TITLE: fix(web): batch reads\n- reads run.json once",
18570 );
18571 state.config.graph.language = "ja".to_owned();
18572 let m = pr_message(&state, 'A');
18573 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18574 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18575
18576 let task = "今回やってほしいこと: results projector を直す";
18579 let mut state = state_with_summary(task, "- no title line");
18580 state.config.graph.language = "ja".to_owned();
18581 let m = pr_message(&state, 'A');
18582 assert_eq!(
18583 m.title,
18584 format!("chore: land candidate A of run {}", state.id)
18585 );
18586 assert!(
18587 m.body.contains(&format!(
18588 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18589 )),
18590 "{}",
18591 m.body
18592 );
18593 }
18594
18595 #[test]
18596 fn pr_message_scrubs_home_paths_and_addresses() {
18597 let state = state_with_summary(
18598 "fix it in /Users/someone/src/x",
18599 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18600 );
18601 let m = pr_message(&state, 'A');
18602 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18603 assert!(!m.body.contains(leak), "{}", m.body);
18604 }
18605 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18606 }
18607
18608 #[test]
18609 fn pr_message_survives_a_task_that_closes_details() {
18610 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18611 let m = pr_message(&state, 'A');
18612 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18613 }
18614
18615 #[test]
18616 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18617 let cmd = manual_merge_command(
18618 MergeStyle::Squash,
18619 Path::new("/repo"),
18620 "b",
18621 "fix: \"quoted\" $(x) `y`\n\nbody",
18622 );
18623 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18624 }
18625
18626 #[test]
18627 fn manual_merge_command_matches_the_configured_style() {
18628 let repo = Path::new("/repo");
18629 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18630
18631 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18632 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18633
18634 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18635 assert_eq!(
18636 squash,
18637 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18638 \"Merge magi run 0832 (candidate A)\""
18639 );
18640
18641 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18642 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18643 }
18644
18645 #[test]
18646 fn a_nudge_gets_a_quarter_of_the_budget() {
18647 assert_eq!(retry_budget(secs(1200), true), secs(300));
18649 assert_eq!(retry_budget(secs(3600), true), secs(900));
18650 }
18651
18652 #[test]
18653 fn a_resent_prompt_keeps_the_whole_budget() {
18654 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18657 assert_eq!(retry_budget(secs(60), false), secs(60));
18658 }
18659
18660 #[test]
18661 fn the_floor_never_exceeds_the_original_budget() {
18662 assert_eq!(retry_budget(secs(60), true), secs(60));
18666 assert_eq!(retry_budget(secs(480), true), secs(120));
18667 assert_eq!(retry_budget(secs(0), true), secs(0));
18668 }
18669
18670 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18671 agent::CommandEvidence {
18672 id: "item1".to_owned(),
18673 description: "cargo test".to_owned(),
18674 exit_code,
18675 result_summary: String::new(),
18676 source: "codex".to_owned(),
18677 }
18678 }
18679
18680 #[test]
18681 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18682 assert!(!has_unconfirmed_command(&[]));
18686 }
18687
18688 #[test]
18689 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18690 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18694 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18695 assert!(!has_unconfirmed_command(&[
18696 evidence(Some(0)),
18697 evidence(Some(101))
18698 ]));
18699 }
18700
18701 #[test]
18702 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18703 assert!(has_unconfirmed_command(&[
18704 evidence(Some(0)),
18705 evidence(None)
18706 ]));
18707 }
18708
18709 #[test]
18710 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18711 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18712 assert_eq!(
18713 verified_noop_claim(true, &[], text).as_deref(),
18714 Some("already fixed by b32cfc4, on main.")
18715 );
18716 }
18717
18718 #[test]
18719 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18720 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18723 assert!(verified_noop_claim(false, &[], text).is_none());
18724 }
18725
18726 #[test]
18727 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18728 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18729 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18730 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18732 }
18733
18734 #[test]
18735 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18736 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18737 }
18738
18739 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18742 runner.state.candidates = shape
18743 .iter()
18744 .enumerate()
18745 .map(|(i, &(empty, verified))| Candidate {
18746 index: i,
18747 label: (b'A' + i as u8) as char,
18748 agent: "sonnet".to_owned(),
18749 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18750 worktree: PathBuf::from(format!("/wt/{i}")),
18751 summary: String::new(),
18752 stat: String::new(),
18753 files: 0,
18754 commits: 0,
18755 empty,
18756 failed: None,
18757 verified_noop: verified.map(str::to_owned),
18758 duration_ms: 0,
18759 folded: false,
18760 })
18761 .collect();
18762 }
18763
18764 #[test]
18765 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18766 ask_test_home();
18767 let mut runner = runner_at(RunStatus::Implementing);
18768 set_candidates(
18769 &mut runner,
18770 &[
18771 (true, Some("already on main at b32cfc4")),
18772 (true, Some("same fix, see the existing test")),
18773 ],
18774 );
18775
18776 runner
18777 .after_implement()
18778 .expect("a verified no-op is not an error");
18779
18780 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18781 }
18782
18783 #[test]
18784 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18785 ask_test_home();
18786 let mut runner = runner_at(RunStatus::Implementing);
18787 set_candidates(
18791 &mut runner,
18792 &[(true, Some("already on main at b32cfc4")), (true, None)],
18793 );
18794
18795 let err = runner
18796 .after_implement()
18797 .expect_err("an unverified empty candidate must still fail the run");
18798
18799 assert!(
18800 err.to_string().contains("no candidate produced a change"),
18801 "{err}"
18802 );
18803 assert_eq!(runner.state.status, RunStatus::Failed);
18804 }
18805
18806 #[test]
18807 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18808 ask_test_home();
18809 let mut runner = runner_at(RunStatus::Implementing);
18810 set_candidates(&mut runner, &[(true, None), (true, None)]);
18811
18812 let err = runner
18813 .after_implement()
18814 .expect_err("no candidate declared anything; this is an ordinary failure");
18815
18816 assert!(
18817 err.to_string().contains("no candidate produced a change"),
18818 "{err}"
18819 );
18820 assert_eq!(runner.state.status, RunStatus::Failed);
18821 }
18822}
18823
18824#[derive(Debug)]
18827struct GraftRestoreFailed(String);
18828
18829impl std::fmt::Display for GraftRestoreFailed {
18830 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18831 write!(
18832 f,
18833 "could not restore the winner to its pre-graft commit: {}",
18834 self.0
18835 )
18836 }
18837}
18838
18839impl std::error::Error for GraftRestoreFailed {}
18840
18841#[cfg(test)]
18845mod resource_wait_tests {
18846 use super::*;
18847 use std::collections::BTreeMap;
18848
18849 fn sh(id: &str, script: &str) -> AgentSpec {
18850 AgentSpec {
18851 id: id.to_owned(),
18852 kind: crate::config::AgentKind::Command,
18853 model: None,
18854 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18855 extra_args: Vec::new(),
18856 env: BTreeMap::new(),
18857 prompt_delivery: None,
18858 }
18859 }
18860
18861 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18862 SeatJob {
18863 seat: SeatState::new("impl-A", &spec.id, 1),
18864 spec,
18865 cwd: dir.to_path_buf(),
18866 prompt: "do it".to_owned(),
18867 timeout,
18868 allow_write: true,
18869 sessions: false,
18870 artifacts: dir.join("art"),
18871 stem: "impl-A".to_owned(),
18872 handover: None,
18873 }
18874 }
18875
18876 fn state(resource_wait_secs: u64) -> RunState {
18877 let mut config = Config::default();
18878 config.graph.timeout_resource_wait = resource_wait_secs;
18879 RunState::new(
18880 PathBuf::from("/repo"),
18881 "main".to_owned(),
18882 "abc1234".to_owned(),
18883 "task".to_owned(),
18884 config,
18885 )
18886 }
18887
18888 fn hold(cache: &Path) -> crate::cache::Guard {
18889 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18890 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18891 crate::cache::AcquireOutcome::Acquired(g) => g,
18892 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18893 }
18894 }
18895
18896 async fn run(
18897 jobs: Vec<SeatJob>,
18898 sem: Arc<Semaphore>,
18899 cache: Option<&Path>,
18900 st: &mut RunState,
18901 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18902 let prompts = Prompts::default();
18903 let ctx = WaveCtx {
18904 run: "r",
18905 node: "implement",
18906 prompts: &prompts,
18907 cache,
18908 round: None,
18909 carry_seats: false,
18910 pauses: None,
18911 };
18912 wave(jobs, sem, &ctx, st, 0).await
18913 }
18914
18915 #[tokio::test]
18916 async fn verification_cache_wait_preserves_the_command_budget() {
18917 crate::run::pin_test_home();
18918 let tmp = tempfile::tempdir().unwrap();
18919 let cache = tmp.path().join("target");
18920 let guard = hold(&cache);
18921 let release = tokio::spawn(async move {
18922 tokio::time::sleep(Duration::from_millis(100)).await;
18923 guard.release();
18924 });
18925 let mut st = state(10);
18926 let budget = Duration::from_millis(50);
18927 let wait = repository_wait(&st, None);
18928 let (outcomes, _) = with_cache_lease(
18929 &mut st,
18930 Some(&cache),
18931 "gate",
18932 "gate",
18933 tmp.path(),
18934 "abc1234",
18935 budget,
18936 "test gate",
18937 &wait,
18938 |_, available| async move {
18939 assert_eq!(
18940 available, budget,
18941 "contention must not shorten verification"
18942 );
18943 (Vec::new(), false, Vec::new())
18944 },
18945 )
18946 .await
18947 .unwrap();
18948 release.await.unwrap();
18949 assert!(outcomes.is_empty(), "{outcomes:?}");
18950 assert!(
18951 st.events
18952 .iter()
18953 .any(|event| event.message.contains("waiting for"))
18954 );
18955 assert!(st.resource_wait.is_none());
18956 hold(&cache).release();
18958 }
18959
18960 #[tokio::test]
18961 async fn verification_cache_wait_expires_without_starting_commands() {
18962 crate::run::pin_test_home();
18963 let tmp = tempfile::tempdir().unwrap();
18964 let cache = tmp.path().join("target");
18965 let _held = hold(&cache);
18966 let mut st = state(0);
18967 let wait = repository_wait(&st, None);
18968 let (outcomes, _) = with_cache_lease(
18969 &mut st,
18970 Some(&cache),
18971 "gate",
18972 "gate",
18973 tmp.path(),
18974 "abc1234",
18975 Duration::from_secs(60),
18976 "test gate",
18977 &wait,
18978 |_, _| async { panic!("commands must not start without the cache lease") },
18979 )
18980 .await
18981 .unwrap();
18982 assert_eq!(outcomes.len(), 1);
18983 assert!(outcomes[0].resource_blocked);
18984 assert!(st.resource_wait.is_none());
18985 }
18986
18987 #[tokio::test]
18988 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18989 crate::run::pin_test_home();
18990 let tmp = tempfile::tempdir().unwrap();
18991 let cache = tmp.path().join("target-a");
18992 let guard = hold(&cache);
18993 tokio::spawn(async move {
18995 tokio::time::sleep(Duration::from_millis(1500)).await;
18996 guard.release();
18997 });
18998 let mut st = state(60);
18999 let results = run(
19000 vec![job(
19001 sh("alpha", "echo ok"),
19002 tmp.path(),
19003 Duration::from_secs(1),
19004 )],
19005 Arc::new(Semaphore::new(1)),
19006 Some(&cache),
19007 &mut st,
19008 )
19009 .await;
19010 assert!(
19011 matches!(results[0].2, AgentOutcome::Ok(_)),
19012 "the wait must not be charged to the agent"
19013 );
19014 assert!(st.resource_block.is_none());
19015 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19016 assert!(st.resource_wait.is_none() && st.active.is_empty());
19017 }
19018
19019 #[tokio::test]
19020 #[cfg(unix)]
19021 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19022 use std::os::fd::AsRawFd as _;
19023 crate::run::pin_test_home();
19024 let tmp = tempfile::tempdir().unwrap();
19025 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19026 let common = tmp.path().join(".git");
19027 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19028 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19029 assert_eq!(
19030 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19031 0
19032 );
19033 let mut st = state(1);
19034 st.config.coordination = crate::config::test_coordination();
19035 let mut reader = job(
19036 sh("reader", "echo must-not-run"),
19037 tmp.path(),
19038 Duration::from_secs(30),
19039 );
19040 reader.allow_write = false;
19041 let results = run(
19042 vec![reader.clone()],
19043 Arc::new(Semaphore::new(1)),
19044 None,
19045 &mut st,
19046 )
19047 .await;
19048 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19049 assert_eq!(results[0].1.turns, 0);
19050 assert!(FailClass::of(&results[0].2).is_none());
19051 assert_eq!(
19052 st.resource_block.as_ref().unwrap().kind,
19053 "repository coordination"
19054 );
19055 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19056 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19057 drop(external);
19058 let editor = crate::repo_lock::SeatLease::for_checkout(
19059 tmp.path(),
19060 true,
19061 crate::config::test_coordination().as_ref(),
19062 )
19063 .await
19064 .unwrap();
19065 let pause = Pause::new();
19066 let interrupt = Pause::new();
19067 let prompts = Prompts::default();
19068 let ctx = WaveCtx {
19069 run: "r",
19070 node: "review",
19071 prompts: &prompts,
19072 cache: None,
19073 round: None,
19074 carry_seats: false,
19075 pauses: Some((&pause, &interrupt)),
19076 };
19077 let (results, ()) = tokio::join!(
19078 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19079 async {
19080 tokio::time::sleep(Duration::from_millis(50)).await;
19081 pause.park();
19082 }
19083 );
19084 assert!(
19085 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19086 );
19087 assert_eq!(results[0].1.turns, 0);
19088 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19089 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19090 drop(editor);
19091 }
19092
19093 #[tokio::test]
19094 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19095 crate::run::pin_test_home();
19096 let tmp = tempfile::tempdir().unwrap();
19097 let cache = tmp.path().join("target-b");
19098 let _held = hold(&cache);
19099 let mut st = state(1);
19100 let results = run(
19101 vec![job(
19102 sh("alpha", "echo ok"),
19103 tmp.path(),
19104 Duration::from_secs(1),
19105 )],
19106 Arc::new(Semaphore::new(1)),
19107 Some(&cache),
19108 &mut st,
19109 )
19110 .await;
19111 let (_, seat, out) = &results[0];
19112 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19113 assert!(wave_blocked(&results));
19114 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19115 assert_eq!(seat.turns, 0, "nothing was asked");
19116 assert!(
19117 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19118 "no agent was spawned"
19119 );
19120 assert!(FailClass::of(out).is_none(), "not a failure class");
19121 let block = st.resource_block.as_ref().expect("recorded for the resume");
19122 assert_eq!(block.kind, "shared build cache");
19123 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19124 assert!(st.active.is_empty());
19125 }
19126
19127 #[tokio::test]
19128 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19129 crate::run::pin_test_home();
19130 let tmp = tempfile::tempdir().unwrap();
19131 let mut st = state(1);
19132 let results = run(
19133 vec![job(
19134 sh("alpha", "echo ok"),
19135 tmp.path(),
19136 Duration::from_secs(1),
19137 )],
19138 Arc::new(Semaphore::new(0)),
19139 None,
19140 &mut st,
19141 )
19142 .await;
19143 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19144 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19145 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19146 }
19147
19148 #[tokio::test]
19149 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19150 crate::run::pin_test_home();
19151 let tmp = tempfile::tempdir().unwrap();
19152 let mut st = state(60);
19153 let results = run(
19154 vec![job(
19155 sh("alpha", "sleep 30"),
19156 tmp.path(),
19157 Duration::from_secs(1),
19158 )],
19159 Arc::new(Semaphore::new(1)),
19160 None,
19161 &mut st,
19162 )
19163 .await;
19164 let out = &results[0].2;
19165 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19166 let class = FailClass::of(out).expect("a failure");
19167 assert_eq!(class, FailClass::Timeout);
19168 assert!(should_hand_over(None, &class), "existing handover policy");
19169 assert!(st.resource_block.is_none());
19170 }
19171}