1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
188}
189
190#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220 #[must_use]
222 pub fn new() -> Self {
223 Self::default()
224 }
225
226 pub fn park(&self) {
228 self.0.store(true, Ordering::SeqCst);
229 }
230
231 pub fn park_because(&self, reason: impl Into<String>) {
237 self.park_for(ParkCause::Interrupt {
238 reason: reason.into(),
239 });
240 }
241
242 pub fn park_for(&self, cause: ParkCause) {
245 let mut reason_guard = self
246 .1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner);
249 if reason_guard.is_none() {
250 *reason_guard = Some(cause);
251 }
252 drop(reason_guard);
253 self.park();
254 }
255
256 #[must_use]
258 pub fn parked(&self) -> bool {
259 self.0.load(Ordering::SeqCst)
260 }
261
262 #[must_use]
264 pub fn reason(&self) -> Option<String> {
265 self.cause().map(|c| c.label())
266 }
267
268 #[must_use]
270 pub fn cause(&self) -> Option<ParkCause> {
271 self.1
272 .lock()
273 .unwrap_or_else(std::sync::PoisonError::into_inner)
274 .clone()
275 }
276}
277
278pub struct Runner {
280 pub state: RunState,
282 roles: ResolvedRoles,
283 sem: Arc<Semaphore>,
284 pause: Pause,
288 interrupt: Pause,
294}
295
296async fn review_base(
301 repo: &Path,
302 remote: &str,
303 base_branch: &str,
304 base_commit: &str,
305 branch: &str,
306) -> String {
307 review_base_checked(repo, remote, base_branch, base_commit, branch)
308 .await
309 .0
310}
311
312async fn review_base_checked(
317 repo: &Path,
318 remote: &str,
319 base_branch: &str,
320 base_commit: &str,
321 branch: &str,
322) -> (String, bool) {
323 let tracking = format!("{remote}/{base_branch}");
324 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325 if git::rev_exists(repo, &tracking).await
326 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327 && !mb.is_empty()
328 {
329 return (mb, fresh);
330 }
331 let mb = git::merge_base(repo, base_commit, branch)
332 .await
333 .ok()
334 .filter(|mb| !mb.is_empty())
335 .unwrap_or_else(|| base_commit.to_owned());
336 (mb, false)
337}
338
339pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343 if !is_review_run(state) {
344 return;
345 }
346 let base = review_base(
347 &state.repo,
348 &state.config.merge.remote,
349 &state.base_branch,
350 &state.base_commit,
351 branch,
352 )
353 .await;
354 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355 && !subjects.is_empty()
356 && state.reviewed_commits.as_ref() != Some(&subjects)
357 {
358 state.reviewed_commits = Some(subjects);
359 state.save().ok();
360 }
361}
362
363async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367 let (base, trusted) = review_base_checked(
368 &state.repo,
369 &state.config.merge.remote,
370 &state.base_branch,
371 &state.base_commit,
372 branch,
373 )
374 .await;
375 if !trusted {
376 return None;
377 }
378 git::subjects(&state.repo, &state.base_commit, &base)
379 .await
380 .ok()
381}
382
383pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388 if is_leak(computed) {
389 return false;
390 }
391 let cur = current.trim();
392 cur.is_empty()
393 || cur.starts_with(REVIEW_PROMPT_OPENING)
394 || cur.starts_with("chore: land candidate")
395 || cur.starts_with("magi: candidate")
396 || is_leak(cur)
397}
398
399async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428 let tracking = format!("{remote}/{branch}");
429 let fetched = git::fetch(repo, remote, branch).await;
430 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431 let local_exists = git::branch_exists(repo, branch).await?;
432 if !fresh {
433 if !local_exists {
434 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435 }
436 tracing::warn!(
437 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438 );
439 return Ok(());
440 }
441 let remote_sha = git::rev_parse(repo, &tracking).await?;
442 if !local_exists {
443 git::git(repo, &["branch", branch, &tracking]).await?;
444 return Ok(());
445 }
446 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448 return Ok(());
449 }
450 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457 .await?
458 {
459 crate::reconcile::Reconciliation::Pushed => {
460 tracing::warn!(
461 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462 short(&local_sha),
463 short(&remote_sha)
464 );
465 return Ok(());
466 }
467 crate::reconcile::Reconciliation::Placeholder => {}
468 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469 }
470 }
471 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472 if !out.ok() {
473 bail!(
474 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475 short(&local_sha),
476 short(&remote_sha),
477 out.stderr
478 );
479 }
480 tracing::warn!(
481 "local `{branch}` was stale: fast-forwarded {} -> {}",
482 short(&local_sha),
483 short(&remote_sha)
484 );
485 Ok(())
486}
487
488async fn resolve_base(
489 repo: &Path,
490 base_branch: &str,
491 remote: &str,
492 coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494 let _lease = crate::repo_lock::SeatLease::operation_with(
495 repo,
496 coordination,
497 &crate::repo_lock::WaitPolicy::current(),
498 )
499 .await?;
500 let tracking = format!("{remote}/{base_branch}");
501 let fetched = git::fetch(repo, remote, base_branch).await;
502 if let Ok(out) = &fetched
503 && out.ok()
504 && git::rev_exists(repo, &tracking).await
505 {
506 return Ok(crate::run::FetchedBase {
507 remote: remote.to_owned(),
508 branch: base_branch.to_owned(),
509 commit: git::rev_parse(repo, &tracking).await?,
510 fetched_at: jiff::Timestamp::now(),
511 });
512 }
513 let why = match &fetched {
514 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515 Ok(_) => format!("{remote} has no {base_branch}"),
516 Err(e) => e.to_string(),
517 };
518 bail!(
521 "cannot read {tracking} ({why}); refusing to branch off the local \
522 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523 base / remote in magi.toml. This is launcher preparation: check the \
524 launcher's Git administration access, network and repository coordination \
525 before retrying; an unattended seat cannot approve those permissions"
526 )
527}
528
529struct FixClaim {
546 path: PathBuf,
547}
548
549impl FixClaim {
550 fn acquire(dir: &Path) -> Result<Self> {
551 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552 let path = dir.join("fix.lock");
553 match Self::create(&path) {
554 Ok(claim) => Ok(claim),
555 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
556 if Self::reclaim_if_dead(&path) {
557 Self::create(&path).with_context(|| format!("lock {}", path.display()))
558 } else {
559 bail!(
560 "another repair or `magi fix` is already running for this run ({} exists)",
561 path.display()
562 )
563 }
564 }
565 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
566 }
567 }
568
569 fn create(path: &Path) -> std::io::Result<Self> {
570 let mut f = std::fs::OpenOptions::new()
571 .write(true)
572 .create_new(true)
573 .open(path)?;
574 use std::io::Write as _;
575 writeln!(f, "{}", std::process::id())?;
577 Ok(Self {
578 path: path.to_owned(),
579 })
580 }
581
582 fn reclaim_if_dead(path: &Path) -> bool {
586 let dead = std::fs::read_to_string(path)
587 .ok()
588 .and_then(|body| body.trim().parse::<u32>().ok())
589 .is_some_and(|pid| !crate::proc::pid_alive(pid));
590 dead && std::fs::remove_file(path).is_ok()
591 }
592}
593
594impl Drop for FixClaim {
595 fn drop(&mut self) {
596 let _ = std::fs::remove_file(&self.path);
597 }
598}
599
600struct DriverExecution<'a> {
603 runner: &'a mut Runner,
604 driver: crate::driver::DriverGuard,
605}
606
607impl Drop for DriverExecution<'_> {
608 fn drop(&mut self) {
609 self.driver.release_local(&mut self.runner.state);
610 }
611}
612
613impl Runner {
614 pub async fn start(
616 repo: &Path,
617 instruction: String,
618 config: Config,
619 origin: Origin,
620 ) -> Result<Self> {
621 Self::start_naming(repo, instruction, "", config, origin).await
622 }
623
624 pub async fn start_naming(
628 repo: &Path,
629 instruction: String,
630 also_scan: &str,
631 config: Config,
632 origin: Origin,
633 ) -> Result<Self> {
634 let repo = git::toplevel(repo).await?;
635 let missing = agent::missing_programs(&config.agents);
636 if !missing.is_empty() {
637 bail!(
638 "these agent programs are not on PATH: {}. Fix the roster in \
639 magi.toml or install them.",
640 missing.join(", ")
641 );
642 }
643 let base_branch =
644 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
645 .await?;
646 let fetched_base = resolve_base(
647 &repo,
648 &base_branch,
649 &config.merge.remote,
650 config.coordination.as_ref(),
651 )
652 .await?;
653 let base_commit = fetched_base.commit.clone();
654 let roles = config.resolve_roles()?;
655 let max_parallel = config.graph.max_parallel.max(1);
656 let seeds = refs::resolve(
659 &repo,
660 &base_commit,
661 &config.merge.remote,
662 &format!("{also_scan}\n{instruction}"),
663 )
664 .await;
665 refs::plan(&repo, &seeds).await?;
666 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
667 state.fetched_base = Some(fetched_base);
671 state.origin = Some(origin);
672 for seed in &seeds {
673 state.event(
674 "seed",
675 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
676 );
677 }
678 state.seeds = seeds;
679 state.event("start", format!("run {} created", state.id));
680 state.save()?;
681 Ok(Self {
682 state,
683 roles,
684 sem: Arc::new(Semaphore::new(max_parallel)),
685 pause: Pause::new(),
686 interrupt: Pause::new(),
687 })
688 }
689
690 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
704 Self::review_taking_over(repo, branch, config, None, origin).await
705 }
706
707 pub async fn review_taking_over(
714 repo: &Path,
715 branch: &str,
716 config: Config,
717 takeover: Option<crate::handover::Takeover>,
718 origin: Origin,
719 ) -> Result<Self> {
720 let repo = git::toplevel(repo).await?;
721 let missing = agent::missing_programs(&config.agents);
722 if !missing.is_empty() {
723 bail!(
724 "these agent programs are not on PATH: {}. Fix the roster in \
725 magi.toml or install them.",
726 missing.join(", ")
727 );
728 }
729 let base_branch =
730 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
731 .await?;
732 if base_branch == branch {
733 bail!("`{branch}` is the base branch; there is nothing to review against");
734 }
735 let fetched_base = resolve_base(
736 &repo,
737 &base_branch,
738 &config.merge.remote,
739 config.coordination.as_ref(),
740 )
741 .await?;
742 let base_commit = fetched_base.commit.clone();
743
744 let roles = config.resolve_roles()?;
745 let max_parallel = config.graph.max_parallel.max(1);
746 let mut state = RunState::new(
747 repo.clone(),
748 base_branch,
749 base_commit.clone(),
750 String::new(),
751 config,
752 );
753 state.fetched_base = Some(fetched_base);
754 state.origin = Some(origin);
755
756 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
763 earlier: Vec::new(),
764 home: crate::run::home(),
765 choice: None,
766 task_instruction: None,
767 });
768 let wait = repository_wait(&state, None);
769 let coordination_config = state.config.coordination.clone();
770 let coordination =
771 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
772 .await?;
773 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
774 if let Some(released) = &released {
775 state.event(
776 "release",
777 format!(
778 "took `{branch}` over from run {}: its worktree was released: {}",
779 crate::run::short_of(&released.old_id),
780 released.audit
781 ),
782 );
783 }
784 if let Some(choice) = takeover.choice.as_ref()
788 && let Err(e) =
789 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
790 .await
791 {
792 if let Some(released) = &released {
793 released.restore(&repo, branch).await;
794 }
795 return Err(e.context("applying the owner's answer about the diverged branch"));
796 }
797 for id in takeover.earlier.iter().rev() {
803 if let Ok(old) = RunState::load_under(id, &takeover.home)
804 && let Some(head) = old.pr_head.clone()
805 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
806 {
807 state.pr_head = Some(head);
808 break;
809 }
810 }
811 drop(coordination);
812 let opened = Self::open_review(
813 &repo,
814 branch,
815 state,
816 roles,
817 max_parallel,
818 base_commit,
819 &takeover,
820 )
821 .await;
822 if opened.is_err()
823 && let Some(released) = &released
824 {
825 let _coordination = crate::repo_lock::SeatLease::operation_with(
826 &repo,
827 coordination_config.as_ref(),
828 &wait,
829 )
830 .await?;
831 released.restore(&repo, branch).await;
832 }
833 opened
834 }
835
836 pub async fn recover_pr(
838 repo: &Path,
839 task: &str,
840 selection: crate::pr_recovery::Selection,
841 mut config: Config,
842 ) -> Result<Self> {
843 selection.configure(&mut config)?;
844 selection.check_task(task, &selection.instruction)?;
845 let base_branch =
846 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
847 .await?;
848 let fetched_base = resolve_base(
849 repo,
850 &base_branch,
851 &config.merge.remote,
852 config.coordination.as_ref(),
853 )
854 .await?;
855 let base_commit = fetched_base.commit.clone();
856 selection
857 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
858 .await?;
859 Box::pin(selection.verify_source(repo)).await?;
860 if let Some(proof) = &selection.source_proof {
861 anyhow::ensure!(
862 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
863 "source rebase base is outside the fetched landing base"
864 );
865 }
866 let roles = config.resolve_roles()?;
867 let max_parallel = config.graph.max_parallel.max(1);
868 let mut state = RunState::new(
869 repo.to_path_buf(),
870 base_branch,
871 base_commit.clone(),
872 selection.instruction.clone(),
873 config,
874 );
875 state.fetched_base = Some(fetched_base);
876 state.origin = Some(crate::run::Origin::queue(task));
877 state.pr_head = Some(selection.branch.clone());
878 state.pr_recovery = Some(selection.clone());
879 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
880 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
881 }
882
883 async fn open_selected(
884 repo: &Path,
885 state: RunState,
886 roles: ResolvedRoles,
887 max_parallel: usize,
888 base_commit: String,
889 selection: crate::pr_recovery::Selection,
890 ) -> Result<Self> {
891 let mut state = state;
892 let branch = state.branch_for('A');
893 let wait = repository_wait(&state, None);
894 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
895 git::git(repo, &["branch", &branch, selection.source()]).await?;
896 drop(coordination);
897 state.event(
898 "recovery",
899 format!(
900 "selected implementation {} for {} (public lease {})",
901 selection.source(),
902 selection.url,
903 selection.head
904 ),
905 );
906 let takeover = crate::handover::Takeover {
907 earlier: Vec::new(),
908 home: crate::run::home(),
909 choice: None,
910 task_instruction: Some(selection.instruction.clone()),
911 };
912 state.save()?;
913 let id = state.id.clone();
914 let opened = Self::open_review(
915 repo,
916 &branch,
917 state,
918 roles,
919 max_parallel,
920 base_commit,
921 &takeover,
922 )
923 .await;
924 match opened {
925 Ok(runner) => Ok(runner),
926 Err(e) => {
927 if let Ok(mut partial) = RunState::load(&id) {
928 partial.status = RunStatus::Blocked;
929 partial.driver_exited = true;
930 partial.event(
931 "recovery",
932 format!("startup failed; retained source and candidate {branch}: {e:#}"),
933 );
934 partial.save()?;
935 }
936 Err(crate::pr_recovery::StartupFailed {
937 run: id,
938 cause: format!("{e:#}"),
939 }
940 .into())
941 }
942 }
943 }
944
945 async fn inherit(
950 repo: &Path,
951 branch: &str,
952 base_commit: &str,
953 takeover: &crate::handover::Takeover,
954 ) -> Option<crate::recovery::RecoveryRecord> {
955 use crate::recovery::{Relation, assess};
956 let newest = takeover.earlier.last()?;
957 let mut chosen = None;
958 for id in takeover.earlier.iter().rev() {
959 if let Ok(old) = RunState::load_under(id, &takeover.home)
960 && !old.reviews.is_empty()
961 && (old.winner().is_some_and(|w| w.branch == branch)
962 || old.pr_head.as_deref() == Some(branch))
963 {
964 chosen = Some(old);
965 break;
966 }
967 }
968 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
969 .await
970 .unwrap_or_default();
971 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
972 Some(last) if last.head == tip => Relation::Same,
973 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
974 _ => Relation::Diverged,
975 };
976 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
977 Some(assess(from, chosen.as_ref(), base_commit, relation))
978 }
979
980 async fn open_review(
983 repo: &Path,
984 branch: &str,
985 mut state: RunState,
986 roles: ResolvedRoles,
987 max_parallel: usize,
988 base_commit: String,
989 takeover: &crate::handover::Takeover,
990 ) -> Result<Self> {
991 let wait = repository_wait(&state, None);
992 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
993 if state.pr_recovery.is_none() {
994 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
995 }
996 let start = review_base(
999 repo,
1000 &state.config.merge.remote,
1001 &state.base_branch,
1002 &base_commit,
1003 branch,
1004 )
1005 .await;
1006 let log = git::log_oneline(repo, &start, branch)
1007 .await
1008 .unwrap_or_default();
1009 let log_text = if log.trim().is_empty() {
1010 "(no commit messages)"
1011 } else {
1012 log.trim()
1013 };
1014 let mut instruction = format!(
1015 "Review the work already on branch `{branch}`. There is no task \
1016 statement: what the change claims to do is whatever its commits \
1017 say.\n\n{log_text}"
1018 );
1019 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1023 if let Some(task) = takeover
1024 .task_instruction
1025 .as_deref()
1026 .filter(|t| !t.trim().is_empty())
1027 && rec.reviewed_head.is_some()
1028 {
1029 instruction = format!(
1030 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1031 task.trim()
1032 );
1033 rec.criteria_inherited = true;
1034 }
1035 state.event("recovery", rec.summary());
1036 state.recovery = Some(rec);
1037 }
1038 if let Some(selected) = &state.pr_recovery {
1039 instruction = selected.instruction.clone();
1040 }
1041 state.task_instruction = takeover.task_instruction.clone();
1042 state.instruction = instruction;
1043 state.reviewed_commits = Some(
1044 git::subjects(repo, &start, branch)
1045 .await
1046 .unwrap_or_default(),
1047 );
1048
1049 let worktree = state.worktree_root().join("under-review");
1052 if let Some(parent) = worktree.parent() {
1053 tokio::fs::create_dir_all(parent).await.ok();
1054 }
1055 let path = worktree.to_string_lossy().to_string();
1056 git::git(repo, &["worktree", "add", &path, branch])
1057 .await
1058 .with_context(|| {
1059 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1060 })?;
1061
1062 let setup = match ensure_setup_config(&mut state, repo).await {
1063 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1064 Err(e) => Err(e),
1065 };
1066 if let Err(e) = setup {
1067 git::worktree_remove_clean(repo, &worktree).await.ok();
1068 return Err(e);
1069 }
1070 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1071 .await
1072 .unwrap_or(0);
1073 if commits == 0 {
1074 git::worktree_remove_clean(repo, &worktree).await.ok();
1075 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1076 }
1077 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1078 .await
1079 .map(|f| f.len())
1080 .unwrap_or(0);
1081 if files == 0
1082 && let (Ok(head_tree), Ok(base_tree)) = (
1083 git::tree_of(&worktree, "HEAD").await,
1084 git::tree_of(&worktree, &base_commit).await,
1085 )
1086 && head_tree == base_tree
1087 {
1088 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1089 git::worktree_remove_clean(repo, &worktree).await.ok();
1090 bail!(
1091 "`{branch}` at {} has a tree identical to base {}; this usually means \
1092 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1093 against `{}/{branch}`) rather than an empty change",
1094 short(&head),
1095 short(&base_commit),
1096 state.config.merge.remote
1097 );
1098 }
1099 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1100 .await
1101 .unwrap_or_default();
1102
1103 state.candidates.push(Candidate {
1104 index: 0,
1105 label: 'A',
1106 agent: EXISTING_BRANCH.to_owned(),
1109 branch: branch.to_owned(),
1110 worktree,
1111 summary: String::new(),
1112 stat,
1113 files,
1114 commits,
1115 empty: false,
1116 failed: None,
1117 verified_noop: None,
1118 duration_ms: 0,
1119 folded: false,
1120 });
1121 state.tally = Some(Tally {
1122 first_choice: BTreeMap::from([('A', 0)]),
1123 borda: BTreeMap::new(),
1124 winner: 'A',
1125 rankings: 0,
1126 unanimous_initial: false,
1127 deliberated: false,
1128 changed_votes: 0,
1129 unanimous_final: false,
1130 tie_break: None,
1131 judges: 0,
1135 present: 0,
1136 quorum: 0,
1137 met_quorum: true,
1138 uncontested: Some("review-only run: nothing competed".to_owned()),
1139 });
1140 state.status = RunStatus::Reviewing;
1141 state.prep_complete = Some(true);
1142 state.event(
1143 "start",
1144 format!(
1145 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1146 state.id
1147 ),
1148 );
1149 state.save()?;
1150 Ok(Self {
1151 state,
1152 roles,
1153 sem: Arc::new(Semaphore::new(max_parallel)),
1154 pause: Pause::new(),
1155 interrupt: Pause::new(),
1156 })
1157 }
1158
1159 pub fn resume(id: &str) -> Result<Self> {
1161 let state = RunState::load(id)?;
1162 crate::pr_head::ensure_owner(&state)?;
1163 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1164 bail!(
1165 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1166 );
1167 }
1168 if let Some(to) = &state.released_to {
1169 bail!(
1170 "run {} cannot be resumed: its worktree was released to run {}",
1171 state.short(),
1172 crate::run::short_of(to)
1173 );
1174 }
1175 let roles = state.config.resolve_roles()?;
1176 if let Some(id) = &state.config.graph.selected_implementer
1177 && state.candidates.iter().any(|c| c.agent != *id)
1178 {
1179 bail!(
1180 "replacement implementer pin does not match the saved candidates; task must stay held"
1181 );
1182 }
1183 let max_parallel = state.config.graph.max_parallel.max(1);
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub async fn execute(&mut self) -> Result<()> {
1200 if let Some(selection) = &self.state.pr_recovery
1201 && (selection.source_proof.is_some() || selection.source_run.is_some())
1202 {
1203 Box::pin(selection.verify_source(&self.state.repo)).await?;
1204 }
1205 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1208 || self.state.land_approval.is_some()
1209 || self.state.merge_decision.is_some()
1210 || !self.state.merge_repairs.is_empty()
1211 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1212 {
1213 let claim = FixClaim::acquire(&self.state.dir())?;
1214 let disk = RunState::load(&self.state.id)?;
1215 let checkpoint = |s: &RunState| {
1216 (
1217 s.merge_repairs.len(),
1218 s.merge_repairs
1219 .last()
1220 .map(|r| (r.question.clone(), r.phase)),
1221 s.land_approval
1222 .as_ref()
1223 .map(|a| (a.question.clone(), a.head.clone())),
1224 )
1225 };
1226 if checkpoint(&disk) != checkpoint(&self.state) {
1227 bail!(
1228 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1229 );
1230 }
1231 Some(claim)
1232 } else {
1233 None
1234 };
1235 let mut execution = self.claim_driver()?;
1236 let runner = &mut *execution.runner;
1237 let wait = runner.repository_wait();
1238 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1241 let result = match result {
1242 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1243 record_repository_block(&mut runner.state, &e);
1244 runner.park_here()?;
1245 Ok(())
1246 }
1247 other => other,
1248 };
1249 execution.driver.finish(&mut runner.state)?;
1250 let ended = if result.is_err() {
1251 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1252 } else {
1253 crate::notices::run_ended(&runner.state)
1254 };
1255 if let Some(notice) = ended {
1256 crate::notices::raise(notice);
1257 }
1258 result
1259 }
1260
1261 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1262 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1263 }
1264
1265 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1266 let wait = self.repository_wait();
1267 coordinate_operation(&mut self.state, cwd, &wait).await
1268 }
1269
1270 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1272 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1273 Ok(owner) => owner,
1274 Err(error) => {
1275 let detail = format!("worktree maintenance or ownership update: {error:#}");
1276 self.state.resource_block = Some(crate::run::ResourceBlock {
1280 node: "start".into(),
1281 kind: "worktree maintenance".into(),
1282 detail: detail.clone(),
1283 at: Timestamp::now(),
1284 });
1285 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1286 }
1287 };
1288 let disk = RunState::load(&self.state.id)?;
1289 crate::pr_head::ensure_owner(&disk)?;
1290 if disk.liveness(false) == crate::run::Liveness::Live {
1291 bail!("run already has a live driver; wait for its execution to end");
1292 }
1293 if let Some(to) = &disk.released_to {
1294 bail!(
1295 "run {} cannot continue: its worktree was released to run {}",
1296 self.state.short(),
1297 crate::run::short_of(to)
1298 );
1299 }
1300 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1301 let mut execution = DriverExecution {
1302 runner: self,
1303 driver,
1304 };
1305 let _owner = owner;
1308 let state = &mut execution.runner.state;
1309 state.parked = false;
1314 state.resource_wait = None;
1315 state.resource_block = None;
1316 state.clear_active();
1323 let pid = std::process::id();
1338 state.driver_pid = Some(pid);
1339 state.driver_started_at = crate::proc::process_started_at(pid);
1340 state.driver_exited = false;
1341 state.begin_pass();
1342 execution.driver.bind(state);
1343 state.save()?;
1344 execution.driver.record(state)?;
1345 Ok(execution)
1346 }
1347
1348 async fn execute_graph(&mut self) -> Result<()> {
1349 self.state.configure_recovery_resume()?;
1350 self.persist_candidates().await?;
1353 if self
1354 .state
1355 .merge_decision
1356 .as_ref()
1357 .is_some_and(|decision| !decision.closed)
1358 && self.state.status != RunStatus::Landing
1359 {
1360 if crate::merge_decision::resumable_hold(&self.state) {
1361 self.state.merge = Some(MergeOutcome {
1362 mode: MergeMode::Pr,
1363 ok: false,
1364 detail: self
1365 .state
1366 .merge_decision
1367 .as_ref()
1368 .unwrap()
1369 .target
1370 .pr
1371 .clone(),
1372 empty: false,
1373 });
1374 self.run_land().await?;
1375 self.settle_questions();
1376 return Ok(());
1377 }
1378 bail!(
1379 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1380 );
1381 }
1382 if self.state.config.graph.selected_implementer.is_some()
1383 && self.state.status == RunStatus::Blocked
1384 && self.state.tally.is_none()
1385 && self.state.candidate_preservation.is_empty()
1386 && self.state.interrupted_implementations.is_empty()
1387 {
1388 bail!(
1389 "selected implementation is held; explicitly release the owning task to retry it"
1390 );
1391 }
1392
1393 if self.state.merge_repairs.last().is_some_and(|r| {
1394 matches!(
1395 r.phase,
1396 crate::run::MergeRepairPhase::Requested
1397 | crate::run::MergeRepairPhase::Fixing
1398 | crate::run::MergeRepairPhase::Reviewing
1399 )
1400 }) {
1401 if self.repair_merge_review().await? {
1402 self.run_land().await?;
1403 }
1404 self.settle_questions();
1405 return Ok(());
1406 }
1407 if self.state.pr_publication_pending {
1410 self.merge().await?;
1411 if self.resource_halt()? {
1412 return Ok(());
1413 }
1414 self.state.save()?;
1415 self.settle_questions();
1416 return Ok(());
1417 }
1418 if self.state.status == RunStatus::Stalled {
1431 if self.recover_stall().await? {
1432 self.finish_after_tally().await?;
1433 } else {
1434 self.state.save()?;
1436 }
1437 return Ok(());
1438 }
1439 if self.state.status == RunStatus::Landing {
1449 self.run_land().await?;
1450 self.settle_questions();
1455 return Ok(());
1456 }
1457 if self.state.github_text.is_some()
1462 && self.state.merge.is_none()
1463 && self.state.status == RunStatus::Gating
1464 {
1465 self.merge().await?;
1466 if self.resource_halt()? {
1467 return Ok(());
1468 }
1469 self.state.save()?;
1470 return Ok(());
1471 }
1472 if let Err(e) = self.prep().await {
1473 if e.is::<crate::repo_lock::ResourceBlocked>() {
1474 return Err(e);
1475 }
1476 self.state.status = RunStatus::Blocked;
1477 self.state
1478 .prep_failure
1479 .get_or_insert_with(|| format!("{e:#}"));
1480 self.state
1481 .event("prep", format!("preparation stopped: {e:#}"));
1482 self.state.save()?;
1483 return Err(e);
1484 }
1485 if self.park_here()? {
1486 return Ok(());
1487 }
1488 self.advise().await?;
1489 if self.park_here()? {
1490 return Ok(());
1491 }
1492 self.implement().await?;
1493 if self.park_here()? {
1494 return Ok(());
1495 }
1496 if self.state.status == RunStatus::VerifiedNoop {
1500 return Ok(());
1501 }
1502 self.judge().await?;
1503 if self.park_here()? {
1504 return Ok(());
1505 }
1506 self.deliberate().await?;
1507 if self.park_here()? {
1508 return Ok(());
1509 }
1510 self.vote().await?;
1511 if self.park_here()? {
1512 return Ok(());
1513 }
1514 self.tally()?;
1515 if self.state.status == RunStatus::Stalled {
1520 self.state.save()?;
1524 return Ok(());
1525 }
1526 self.finish_after_tally().await?;
1527 Ok(())
1528 }
1529
1530 fn stop_for_resource(&mut self) -> Result<()> {
1535 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1537 {
1538 self.state.status = RunStatus::Blocked;
1539 }
1540 self.state.save()
1541 }
1542
1543 fn resource_halt(&mut self) -> Result<bool> {
1545 if self.state.resource_block.is_none() {
1546 return Ok(false);
1547 }
1548 self.stop_for_resource()?;
1549 Ok(true)
1550 }
1551
1552 fn park_here(&mut self) -> Result<bool> {
1559 let parking = self.pause.parked() || self.interrupt.parked();
1560 if !parking && self.state.resource_block.is_some() {
1561 self.stop_for_resource()?;
1562 return Ok(true);
1563 }
1564 if !self.pause.parked() && !self.interrupt.parked() {
1569 return Ok(false);
1570 }
1571 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1572 let why = match cause.as_ref().map(ParkCause::label) {
1573 Some(reason) => format!(
1574 "parked after `{}` ({reason}) — resume to carry on from here",
1575 self.state.status.as_str()
1576 ),
1577 None => format!(
1578 "parked after `{}` — resume to carry on from here",
1579 self.state.status.as_str()
1580 ),
1581 };
1582 self.state.event("park", why);
1583 self.state
1584 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1585 self.state.parked = true;
1586 self.state.save()?;
1587 Ok(true)
1588 }
1589
1590 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1592 if self.state.prep_failure.is_none() {
1593 return Ok(());
1594 }
1595 let failed_at = self
1596 .state
1597 .events
1598 .iter()
1599 .rev()
1600 .find(|event| event.node == "prep")
1601 .map_or(self.state.updated_at, |event| event.at);
1602 anyhow::ensure!(
1603 approval_at > failed_at,
1604 "preparation retry requires a new explicit release after the failure"
1605 );
1606 self.state.prep_failure = None;
1607 self.state.prep_complete = Some(false);
1608 self.state
1609 .event("prep-retry", "operator approved retry of preparation");
1610 Ok(())
1611 }
1612
1613 pub fn on_pause(&mut self, pause: Pause) {
1615 self.pause = pause;
1616 }
1617
1618 pub fn watch_interrupt(&mut self, pause: Pause) {
1624 self.interrupt = pause;
1625 }
1626
1627 fn settle_questions(&mut self) {
1647 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1648 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1649 }
1650 }
1651
1652 async fn finish_after_tally(&mut self) -> Result<()> {
1655 self.graft().await?;
1658 if self.resource_halt()? {
1659 return Ok(());
1660 }
1661 self.fold_losers().await?;
1662 self.sync_to_base().await?;
1667 if self.resource_halt()? {
1668 return Ok(());
1669 }
1670 if self.state.status == RunStatus::AlreadyInBase {
1671 return Ok(());
1672 }
1673 self.review_loop().await?;
1674 if self.resource_halt()? {
1675 return Ok(());
1676 }
1677 self.sync_to_base().await?;
1678 if self.resource_halt()? {
1679 return Ok(());
1680 }
1681 if self.state.status == RunStatus::AlreadyInBase {
1682 return Ok(());
1683 }
1684 self.gate().await?;
1685 self.merge().await?;
1686 if self.resource_halt()? {
1687 return Ok(());
1688 }
1689 self.state.save()?;
1690 Ok(())
1691 }
1692
1693 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1700 let result = match ensure_setup_config(&mut self.state, repo).await {
1701 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1702 Err(e) => Err(e),
1703 };
1704 if let Err(e) = result {
1705 git::worktree_remove_clean(repo, wt).await.ok();
1706 if in_prep {
1707 self.rollback_prep(repo).await;
1708 }
1709 self.state.status = RunStatus::Blocked;
1710 self.state.event(
1711 "setup",
1712 format!("worktree setup failed in {}: {e:#}", wt.display()),
1713 );
1714 self.state.save()?;
1715 return Err(e);
1716 }
1717 Ok(())
1718 }
1719
1720 async fn rollback_prep(&mut self, repo: &Path) {
1722 let root = self.state.worktree_root();
1723 for c in std::mem::take(&mut self.state.candidates) {
1724 if matches!(
1725 git::worktree_remove_clean(repo, &c.worktree).await,
1726 Ok(true)
1727 ) {
1728 git::branch_delete(repo, &c.branch).await.ok();
1729 } else {
1730 self.state.candidates.push(c);
1731 }
1732 }
1733 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1734 for j in 1..=seats {
1735 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1736 .await
1737 .ok();
1738 }
1739 for k in 1..=self.state.config.graph.advisors {
1740 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1741 .await
1742 .ok();
1743 }
1744 }
1745
1746 async fn prep(&mut self) -> Result<()> {
1747 if let Some(reason) = &self.state.prep_failure {
1748 bail!(
1749 "candidate preparation previously failed: {reason}; correct the source before retrying"
1750 );
1751 }
1752 if self
1753 .state
1754 .prep_complete
1755 .unwrap_or(!self.state.candidates.is_empty())
1756 {
1757 return Ok(());
1758 }
1759 let repo = self.state.repo.clone();
1760 let _coordination = self.coordinate(&repo).await?;
1763 self.rollback_prep(&repo).await;
1764 if !self.state.candidates.is_empty() {
1765 bail!(
1766 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1767 );
1768 }
1769 self.state.prep_complete = Some(false);
1770 self.state.status = RunStatus::Prep;
1771 self.state.save()?;
1772 let repo = self.state.repo.clone();
1773 let base = self.state.base_commit.clone();
1774 let plan = refs::plan(&repo, &self.state.seeds).await?;
1775 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1776 let root = self.state.worktree_root();
1777 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1778
1779 let hooks_dir = self.state.dir().join("hooks");
1782 if self.state.config.blind.commit_msg_hook {
1783 std::fs::create_dir_all(&hooks_dir)
1784 .with_context(|| format!("create {}", hooks_dir.display()))?;
1785 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1786 let path = hooks_dir.join("commit-msg");
1787 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1788 make_executable(&path)?;
1789 git::acquire_worktree_config(&repo).await?;
1797 self.state.enabled_worktree_config = true;
1798 }
1799
1800 ensure_setup_config(&mut self.state, &repo).await?;
1803
1804 for (index, (spec, label)) in self
1805 .roles
1806 .implementers
1807 .clone()
1808 .into_iter()
1809 .zip(labels)
1810 .enumerate()
1811 {
1812 let branch = self.state.branch_for(label);
1813 let worktree = root.join(format!("cand-{label}"));
1814 self.state.event(
1818 "prep-intent",
1819 serde_json::json!({
1820 "branch": branch,
1821 "worktree": worktree,
1822 "start": start,
1823 "picks": plan.picks,
1824 })
1825 .to_string(),
1826 );
1827 self.state.save()?;
1828 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1829 if self.state.config.blind.commit_msg_hook {
1830 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1831 }
1832 git::local_exclude(&worktree, "/.magi/").await?;
1833 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1834 git::worktree_remove_clean(&repo, &worktree).await.ok();
1837 git::branch_delete(&repo, &branch).await.ok();
1838 self.rollback_prep(&repo).await;
1839 self.state.status = RunStatus::Blocked;
1840 self.state
1841 .event("prep", format!("worktree setup failed: {e:#}"));
1842 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1843 self.state.save()?;
1844 return Err(e);
1845 }
1846 self.state.candidates.push(Candidate {
1847 index,
1848 label,
1849 agent: spec.id.clone(),
1850 branch,
1851 worktree,
1852 summary: String::new(),
1853 stat: String::new(),
1854 files: 0,
1855 commits: 0,
1856 empty: false,
1857 failed: None,
1858 verified_noop: None,
1859 duration_ms: 0,
1860 folded: false,
1861 });
1862 self.state.save()?;
1863 for pick in &plan.picks {
1864 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1865 {
1866 self.state.status = RunStatus::Blocked;
1867 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1868 self.state
1869 .event("prep", format!("cannot apply referenced commit: {e}"));
1870 self.state.save()?;
1871 return Err(e);
1872 }
1873 }
1874 }
1875
1876 for j in 1..=self.roles.judges.len() {
1877 let wt = root.join(format!("judge-{j}"));
1878 if !wt.exists() {
1879 git::worktree_add_detached(&repo, &wt, &base).await?;
1880 self.setup_seat_worktree(&repo, &wt, true).await?;
1881 }
1882 }
1883
1884 if self.state.config.graph.advise {
1892 for k in 1..=self.state.config.graph.advisors {
1893 let wt = root.join(format!("advisor-{k}"));
1894 if !wt.exists() {
1895 git::worktree_add_detached(&repo, &wt, &base).await?;
1896 self.setup_seat_worktree(&repo, &wt, true).await?;
1897 }
1898 }
1899 }
1900
1901 let authors: Vec<&str> = self
1906 .roles
1907 .implementers
1908 .iter()
1909 .map(|a| a.id.as_str())
1910 .collect();
1911 let overlap: Vec<String> = self
1912 .roles
1913 .judges
1914 .iter()
1915 .enumerate()
1916 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1917 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1918 .collect();
1919 if !overlap.is_empty() {
1920 let note = format!(
1921 "{} also authored a candidate; blind, but the panel is less \
1922 independent than {} distinct agents would be",
1923 overlap.join(", "),
1924 self.roles.judges.len()
1925 );
1926 self.state.event("prep", note);
1927 }
1928
1929 self.state.event(
1930 "prep",
1931 format!(
1932 "{} candidates, {} judges, base {} ({})",
1933 self.state.candidates.len(),
1934 self.roles.judges.len(),
1935 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1936 self.state.base_branch
1937 ),
1938 );
1939 self.state.prep_complete = Some(true);
1940 self.state.status = RunStatus::Implementing;
1941 self.state.save()?;
1942 Ok(())
1943 }
1944
1945 async fn advise(&mut self) -> Result<()> {
1978 let implement_untouched = self
1979 .state
1980 .candidates
1981 .iter()
1982 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1983 if !self.state.config.graph.advise || self.state.advise_attempted {
1984 return Ok(());
1985 }
1986 if !implement_untouched {
1987 self.state.event(
1988 "advise",
1989 "skipping the design-deliberation stage: at least one \
1990 candidate already shows implementation progress, so this \
1991 run is past the point the stage exists to run before"
1992 .to_owned(),
1993 );
1994 self.state.advise_attempted = true;
1995 self.state.save()?;
1996 return Ok(());
1997 }
1998 let run_id = self.state.id.clone();
1999 let prompts = self.state.config.prompts.clone();
2000 let instruction = self.state.instruction.clone();
2001 let language = self.state.config.graph.language.clone();
2002 let root = self.state.worktree_root();
2003 let n = self.state.config.graph.advisors;
2004 let where_recorded = self.state.dir().join("run.json");
2005
2006 let seats = match self.state.config.advisors() {
2007 Ok(seats) if !seats.is_empty() => seats,
2008 Ok(_) => {
2009 self.state.event(
2010 "advise",
2011 format!(
2012 "[graph] advisors is 0; skipping the design-deliberation \
2013 stage and continuing without a synthesis brief (see {})",
2014 where_recorded.display()
2015 ),
2016 );
2017 self.state.advise_attempted = true;
2018 self.state.save()?;
2019 return Ok(());
2020 }
2021 Err(e) => {
2022 self.state.event(
2023 "advise",
2024 format!(
2025 "could not resolve advisor seats ({e:#}); continuing \
2026 without a design-deliberation brief (see {})",
2027 where_recorded.display()
2028 ),
2029 );
2030 self.state.advise_attempted = true;
2031 self.state.save()?;
2032 return Ok(());
2033 }
2034 };
2035
2036 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2037 let artifacts = agent::artifacts_dir(&self.state.dir());
2038 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2039
2040 let mut jobs = Vec::new();
2041 for (i, spec) in seats.iter().cloned().enumerate() {
2042 let seat_key = format!("advisor-{}", i + 1);
2043 let seat = self.seat(&seat_key, &spec.id);
2044 jobs.push(SeatJob {
2045 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2046 spec,
2047 seat,
2048 cwd: worktrees[i % worktrees.len()].clone(),
2049 timeout,
2050 allow_write: false,
2051 sessions: false,
2052 artifacts: artifacts.clone(),
2053 stem: seat_key,
2054 handover: None,
2055 });
2056 }
2057
2058 self.state.event(
2059 "advise",
2060 format!(
2061 "{} advisor seat(s) sketching a design in parallel",
2062 jobs.len()
2063 ),
2064 );
2065 let mut quota_losses = Vec::new();
2066 let cache = self.state.config.cache_dir();
2067 let ctx = WaveCtx {
2068 carry_seats: false,
2069 pauses: Some((&self.pause, &self.interrupt)),
2070 run: &run_id,
2071 node: "advise",
2072 prompts: &prompts,
2073 cache: cache.as_deref(),
2074 round: None,
2075 };
2076 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2077 let results = ask_json_wave::<Proposal>(
2078 jobs,
2079 Arc::clone(&self.sem),
2080 self.state.config.graph.retries,
2081 &advisor_roster,
2082 &ctx,
2083 &mut quota_losses,
2084 &mut self.state,
2085 &|p: &Proposal| p.validate(),
2086 )
2087 .await;
2088 if self.state.resource_block.is_some() {
2089 return self.stop_for_resource();
2090 }
2091 self.state.quota.extend(quota_losses);
2092
2093 let mut records = Vec::with_capacity(results.len());
2094 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2095 let agent_id = seat.agent.clone();
2096 self.state.seats.insert(seat.key.clone(), seat);
2097 match res {
2098 Ok((proposal, out)) => {
2099 self.state
2100 .event("advise", format!("advisor-{} proposed a design", i + 1));
2101 records.push(advise::AdvisorRecord::proposed(
2102 i + 1,
2103 agent_id,
2104 proposal,
2105 out.duration_ms,
2106 ));
2107 }
2108 Err(e) => {
2109 self.state.event(
2110 "advise",
2111 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2112 );
2113 records.push(advise::AdvisorRecord::failed(
2114 i + 1,
2115 agent_id,
2116 e.to_string(),
2117 ));
2118 }
2119 }
2120 }
2121
2122 let mut advice = advise::Advice {
2123 records,
2124 synthesis: None,
2125 };
2126 if advice.proposals().is_empty() {
2127 self.state.event(
2128 "advise",
2129 "no advisor produced a usable proposal; continuing without a \
2130 synthesis brief"
2131 .to_owned(),
2132 );
2133 } else {
2134 match self
2135 .synthesize_brief(
2136 &advice,
2137 &instruction,
2138 &language,
2139 &worktrees[0],
2140 &artifacts,
2141 &run_id,
2142 &prompts,
2143 cache.as_deref(),
2144 )
2145 .await
2146 {
2147 Ok(Some(text)) => {
2148 self.state.event(
2149 "advise",
2150 "synthesized a design brief for the implementer".to_owned(),
2151 );
2152 advice.synthesis = Some(text);
2153 }
2154 Ok(None) => {
2155 self.state.event(
2156 "advise",
2157 "the synthesis seat produced nothing usable; continuing \
2158 without a design brief"
2159 .to_owned(),
2160 );
2161 }
2162 Err(e) => {
2163 if e.is::<crate::repo_lock::ResourceBlocked>() {
2164 return Err(e);
2165 }
2166 self.state.event(
2167 "advise",
2168 format!("could not synthesize a design brief: {e:#}"),
2169 );
2170 }
2171 }
2172 }
2173 advise::apply_reflection(&mut advice);
2174
2175 self.state.advice = Some(advice);
2176 self.state.advise_attempted = true;
2177 self.state.save()?;
2178 Ok(())
2179 }
2180
2181 #[allow(clippy::too_many_arguments)]
2193 async fn synthesize_brief(
2194 &mut self,
2195 advice: &advise::Advice,
2196 instruction: &str,
2197 language: &str,
2198 cwd: &Path,
2199 artifacts: &Path,
2200 run_id: &str,
2201 prompts: &Prompts,
2202 cache: Option<&Path>,
2203 ) -> Result<Option<String>> {
2204 let chain = agent::pick_chain(
2205 &self.state.config.agents,
2206 self.state.config.roles.synthesizer.as_ref(),
2207 &agent::installed,
2208 "synthesizer",
2209 )?;
2210 let proposals = advice.proposals();
2211 let mut prompt = prompt::with_overlay(
2212 prompt::synthesize_brief(instruction, &proposals, language),
2213 prompts.overlay("advise"),
2214 );
2215 if cache.is_some() {
2216 prompt.push('\n');
2221 prompt.push_str(&prompt::build_cache_note("advise", false));
2222 }
2223 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2224 let mut last = None;
2229 for (n, spec) in chain.iter().enumerate() {
2230 if n > 0 {
2231 self.state
2232 .event("advise", format!("synthesis falling back to {}", spec.id));
2233 }
2234 let mut seat = self.seat("advise-synthesis", &spec.id);
2235 let wait = self.repository_wait();
2236 let outcome = wait
2237 .scope(agent::invoke(
2238 spec,
2239 &mut seat,
2240 &Invocation {
2241 scope: crate::agent::InvocationScope::Repository,
2242 coordination: self.state.config.coordination.as_ref(),
2243 fetched_base: None,
2244 cwd,
2245 prompt: &prompt,
2246 timeout,
2247 allow_write: false,
2248 unsandboxed: false,
2249 sessions: false,
2250 artifacts,
2251 stem: &if n == 0 {
2252 "advise-synthesis".to_owned()
2253 } else {
2254 format!("advise-synthesis-{}", spec.id)
2255 },
2256 run: run_id,
2257 node: "advise",
2258 cache_dir: None,
2259 attachments: &[],
2260 writable: &[],
2261 },
2262 ))
2263 .await;
2264 if outcome.is_ok() {
2265 self.state.seats.insert(seat.key.clone(), seat);
2266 }
2267 let advance = agent::chain_advances(&outcome);
2268 last = Some(outcome);
2269 if !advance {
2270 break;
2271 }
2272 }
2273 let out = last.expect("a chain holds at least one agent")?;
2276 if !out.usable() {
2277 return Ok(None);
2278 }
2279 let text =
2280 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2281 Ok((!text.trim().is_empty()).then_some(text))
2282 }
2283
2284 async fn implement(&mut self) -> Result<()> {
2287 self.persist_candidates().await?;
2288 let run_id = self.state.id.clone();
2293 let prompts = self.state.config.prompts.clone();
2294 let todo: Vec<usize> = self
2295 .state
2296 .candidates
2297 .iter()
2298 .enumerate()
2299 .filter(|(_, c)| {
2300 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2301 && c.failed.is_none()
2302 && !c.empty
2303 })
2304 .map(|(i, _)| i)
2305 .collect();
2306 if todo.is_empty() {
2307 return self.after_implement();
2308 }
2309 self.state.status = RunStatus::Implementing;
2310
2311 let language = self.state.config.graph.language.clone();
2312 let github_language = self.state.config.graph.github_language.clone();
2313 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2314 let sessions = self.state.config.graph.sessions;
2315 let artifacts = agent::artifacts_dir(&self.state.dir());
2316 let brief = self
2320 .state
2321 .advice
2322 .as_ref()
2323 .and_then(|a| a.synthesis.as_deref())
2324 .map(str::to_owned);
2325 let attachments = self.state.attachments.clone();
2326
2327 let mut jobs = Vec::new();
2328 for &i in &todo {
2329 let (index, label, worktree) = {
2330 let c = &self.state.candidates[i];
2331 (c.index, c.label, c.worktree.clone())
2332 };
2333 let spec = self.roles.implementers[index].clone();
2334 let seat_key = format!("impl-{label}");
2335 let seat = self.seat(&seat_key, &spec.id);
2336 let instruction = seeded_instruction(&self.state);
2337 jobs.push(SeatJob {
2338 spec,
2339 seat,
2340 prompt: prompt::implement_in(
2341 &instruction,
2342 &worktree.to_string_lossy(),
2343 &language,
2344 &github_language,
2345 brief.as_deref(),
2346 &attachments,
2347 ),
2348 cwd: worktree,
2349 timeout,
2350 allow_write: true,
2351 sessions,
2352 artifacts: artifacts.clone(),
2353 stem: format!("impl-{label}"),
2354 handover: None,
2355 });
2356 }
2357
2358 self.state.event(
2359 "implement",
2360 format!("{} candidates in parallel", jobs.len()),
2361 );
2362 let mut sent = jobs.clone();
2368 let cache = self.state.config.cache_dir();
2369 let ctx = WaveCtx {
2370 carry_seats: false,
2371 pauses: Some((&self.pause, &self.interrupt)),
2372 run: &run_id,
2373 node: "implement",
2374 prompts: &prompts,
2375 cache: cache.as_deref(),
2376 round: None,
2377 };
2378 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2379 if !wave_blocked(&results) {
2380 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2381 .await;
2382 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2383 .await?;
2384 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2385 .await;
2386 }
2387 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2388
2389 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2390 && results
2391 .iter()
2392 .any(|(_, _, out)| FailClass::of(out).is_some());
2393 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2394 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2395 if self
2399 .state
2400 .candidate_preservation
2401 .contains_key(&self.state.candidates[i].label)
2402 {
2403 self.state.seats.insert(seat.key.clone(), seat);
2404 }
2405 continue;
2406 }
2407 let seat_key = seat.key.clone();
2408 let agent = seat.agent.clone();
2417 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2418 self.state.seats.insert(seat.key.clone(), seat);
2419 let label = self.state.candidates[i].label;
2420 self.state.interrupted_implementations.remove(&label);
2421
2422 let (summary, duration, failed, verified_claim) = match out {
2423 AgentOutcome::Ok(o) => {
2424 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2425 let failed = (!o.usable()).then(|| {
2426 if o.timed_out {
2427 "agent timed out".to_owned()
2428 } else {
2429 format!(
2430 "agent exited with {}",
2431 crate::notices::exit_status(o.exit_code)
2432 )
2433 }
2434 });
2435 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2436 (text, o.duration_ms, failed, verified_claim)
2437 }
2438 AgentOutcome::Dropped(o) => {
2444 let why = o
2445 .dropped
2446 .as_ref()
2447 .map(|d| d.why.as_str())
2448 .unwrap_or("the CLI ended the stream without delivering its answer");
2449 (
2450 String::new(),
2451 o.duration_ms,
2452 Some(format!("the CLI dropped the stream ({why})")),
2453 None,
2454 )
2455 }
2456 AgentOutcome::Quota(o) => {
2457 self.state.quota.push(QuotaLoss {
2458 seat: seat_key,
2459 node: "implement".to_owned(),
2460 at: Timestamp::now(),
2461 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2462 });
2463 (
2464 String::new(),
2465 o.duration_ms,
2466 Some("rate limited (quota); produced no change".to_owned()),
2467 None,
2468 )
2469 }
2470 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2471 (String::new(), 0, Some(e), None)
2472 }
2473 };
2474
2475 let c = &mut self.state.candidates[i];
2478 if !exhausted_the_fallback_chain {
2479 c.agent = agent;
2480 }
2481 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2482 c.duration_ms = duration;
2483 c.failed = failed;
2484 c.verified_noop = verified_claim;
2485 self.state
2486 .candidate_preservation
2487 .insert(label, "awaiting persistence".to_owned());
2488 }
2489 self.state.save()?;
2490 self.persist_candidates().await?;
2491
2492 if resource_blocked {
2493 return self.stop_for_resource();
2494 }
2495 if selected_failed {
2496 self.state.status = RunStatus::Blocked;
2497 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2498 self.state.save()?;
2499 bail!(
2500 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2501 );
2502 }
2503 self.after_implement()
2504 }
2505
2506 async fn persist_candidates(&mut self) -> Result<()> {
2508 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2509 for label in labels {
2510 let i = self
2511 .state
2512 .candidates
2513 .iter()
2514 .position(|c| c.label == label)
2515 .context("pending candidate is missing")?;
2516 let c = self.state.candidates[i].clone();
2517 let interrupted = self.state.interrupted_implementations.contains(&label);
2518 let result = async {
2519 let wait = crate::repo_lock::WaitPolicy::new(
2520 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2521 || false,
2522 );
2523 let _coordination =
2524 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2525 if let Err(e) = git::snapshot_uncommitted(
2528 &c.worktree,
2529 &self
2530 .state
2531 .dir()
2532 .join("artifacts")
2533 .join(format!("candidate-{label}-recovery")),
2534 )
2535 .await
2536 {
2537 self.state.event(
2538 "preserve",
2539 format!("candidate {label} snapshot incomplete: {e:#}"),
2540 );
2541 }
2542 let rescued = git::rescue_commit(
2543 &c.worktree,
2544 &format!("magi: candidate {label} (uncommitted work)"),
2545 )
2546 .await?;
2547 self.state.note_withheld("implement", &rescued.withheld);
2548 let base = &self.state.base_commit;
2549 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2550 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2551 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2552 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2553 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2554 if c.empty && commits > 0 && !patch.trim().is_empty()
2555 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2556 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2557 || self.state.tally.is_some())
2558 {
2559 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2563 }
2564 let c = &mut self.state.candidates[i];
2565 c.commits = commits;
2566 c.stat = stat;
2567 c.files = files;
2568 let no_changes = commits == 0 || patch.trim().is_empty();
2569 c.empty = no_changes && !interrupted;
2570 if interrupted && no_changes {
2571 c.failed = None;
2572 c.verified_noop = None;
2573 }
2574 if !c.empty {
2575 c.failed = None;
2576 c.verified_noop = None;
2577 }
2578 self.state.event(
2579 "preserve",
2580 format!("candidate {label}: {files} files, {commits} commits"),
2581 );
2582 Ok::<_, anyhow::Error>(())
2583 }
2584 .await;
2585 match result {
2586 Ok(()) => {
2587 self.state.candidate_preservation.remove(&label);
2588 if !interrupted || self.state.candidates[i].files > 0 {
2589 self.state.interrupted_implementations.remove(&label);
2590 }
2591 }
2592 Err(e) => {
2593 let reason = format!("candidate {label} preservation failed: {e:#}");
2594 self.state.event("preserve", &reason);
2595 self.state.candidate_preservation.insert(label, reason);
2596 }
2597 }
2598 if self.state.candidate_preservation.is_empty()
2599 && self.state.status == RunStatus::Blocked
2600 && self.state.candidates.iter().all(|c| c.failed.is_none())
2601 {
2602 self.state.status = RunStatus::Implementing;
2606 self.state.event(
2607 "preserve",
2608 "candidate persistence recovered; implementation can continue",
2609 );
2610 }
2611 self.state.save()?;
2612 }
2613 if !self.state.candidate_preservation.is_empty() {
2614 self.state.status = RunStatus::Blocked;
2615 self.state.save()?;
2616 bail!(
2617 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2618 );
2619 }
2620 Ok(())
2621 }
2622
2623 async fn resume_undelivered(
2651 &mut self,
2652 results: &mut [(usize, SeatState, AgentOutcome)],
2653 sent: &[SeatJob],
2654 prompts: &Prompts,
2655 run_id: &str,
2656 ) {
2657 for (wi, seat, out) in results.iter_mut() {
2658 let Some(dropped) = (match &*out {
2659 AgentOutcome::Dropped(o) => o.dropped.clone(),
2660 _ => None,
2661 }) else {
2662 continue;
2663 };
2664 let Some(job) = sent.get(*wi) else { continue };
2665 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2667 self.state.event(
2668 "implement",
2669 format!(
2670 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2671 work is in the tree",
2672 seat.key, dropped.output_tokens, dropped.why
2673 ),
2674 );
2675 continue;
2676 }
2677 if !has_context(&job.spec, seat, job.sessions) {
2685 self.state.event(
2686 "implement",
2687 format!(
2688 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2689 is no session left to resume",
2690 seat.key, dropped.output_tokens, dropped.why
2691 ),
2692 );
2693 continue;
2694 }
2695 self.state.event(
2696 "implement",
2697 format!(
2698 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2699 conversation",
2700 seat.key, dropped.output_tokens, dropped.why
2701 ),
2702 );
2703 let mut retry = job.clone();
2704 retry.seat = seat.clone();
2705 retry.prompt = prompt::resume_after_drop(&dropped.why);
2706 retry.timeout = retry_budget(job.timeout, true);
2707 retry.stem = format!("{}-resume", job.stem);
2708 let cache = self.state.config.cache_dir();
2709 let ctx = WaveCtx {
2710 carry_seats: false,
2711 pauses: Some((&self.pause, &self.interrupt)),
2712 run: run_id,
2713 node: "implement",
2714 prompts,
2715 cache: cache.as_deref(),
2716 round: None,
2717 };
2718 let (resumed_seat, resumed) =
2719 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2720 *seat = resumed_seat;
2721 *out = resumed;
2722 }
2723 }
2724
2725 async fn resume_seat_handovers(
2790 &mut self,
2791 results: &mut [(usize, SeatState, AgentOutcome)],
2792 sent: &mut [SeatJob],
2793 prompts: &Prompts,
2794 run_id: &str,
2795 ) -> Result<()> {
2796 let instruction = seeded_instruction(&self.state);
2797 let language = self.state.config.graph.language.clone();
2798 let github_language = self.state.config.graph.github_language.clone();
2799 let brief = self
2800 .state
2801 .advice
2802 .as_ref()
2803 .and_then(|a| a.synthesis.as_deref())
2804 .map(str::to_owned);
2805 let attachments = self.state.attachments.clone();
2806 for (wi, seat, out) in results.iter_mut() {
2807 let others: BTreeSet<String> = sent
2810 .iter()
2811 .enumerate()
2812 .filter(|(j, _)| j != wi)
2813 .map(|(_, j)| j.spec.id.clone())
2814 .collect();
2815 let Some(job) = sent.get_mut(*wi) else {
2816 continue;
2817 };
2818 let start = self
2823 .roles
2824 .implementer_roster
2825 .iter()
2826 .position(|s| s.id == job.spec.id)
2827 .unwrap_or(0);
2828 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2829 let mut fallback_attempt = 0usize;
2830 let mut prev: Option<FailClass> = None;
2831 while let Some(cur) = FailClass::of(&*out) {
2832 if !should_hand_over(prev.as_ref(), &cur) {
2833 break;
2834 }
2835 let Some(next) =
2836 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2837 .cloned()
2838 else {
2839 break;
2840 };
2841 tried.insert(next.id.clone());
2842 fallback_attempt += 1;
2843
2844 let coordination = self.coordinate(&job.cwd).await?;
2845 if let Ok(r) = git::rescue_commit(
2846 &job.cwd,
2847 &format!(
2848 "magi: candidate {} (uncommitted work before {} fallback)",
2849 seat.key,
2850 if cur == FailClass::Quota {
2851 "quota"
2852 } else {
2853 "handover"
2854 }
2855 ),
2856 )
2857 .await
2858 {
2859 self.state.note_withheld("implement", &r.withheld);
2860 }
2861 drop(coordination);
2862
2863 record_handover(
2864 &mut self.state,
2865 "implement",
2866 &seat.key,
2867 &seat.agent,
2868 &next.id,
2869 &cur,
2870 &fail_reason(&*out),
2871 );
2872 prev = Some(cur.clone());
2873
2874 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2875 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2876 job.spec = next.clone();
2884 let mut retry = job.clone();
2885 retry.seat = new_seat;
2886 retry.prompt = prompt::implement_in(
2887 &instruction,
2888 &job.cwd.to_string_lossy(),
2889 &language,
2890 &github_language,
2891 brief.as_deref(),
2892 &attachments,
2893 );
2894 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2895 let cache = self.state.config.cache_dir();
2896 let ctx = WaveCtx {
2897 carry_seats: false,
2898 pauses: Some((&self.pause, &self.interrupt)),
2899 run: run_id,
2900 node: "implement",
2901 prompts,
2902 cache: cache.as_deref(),
2903 round: None,
2904 };
2905 let (fallback_seat, fallback_out) = run_one(
2906 retry,
2907 Arc::clone(&self.sem),
2908 &ctx,
2909 &mut self.state,
2910 fallback_attempt,
2911 )
2912 .await;
2913 *seat = fallback_seat;
2914 *out = fallback_out;
2915 }
2916 }
2917 Ok(())
2918 }
2919
2920 async fn resume_unconfirmed_commands(
2944 &mut self,
2945 results: &mut [(usize, SeatState, AgentOutcome)],
2946 sent: &[SeatJob],
2947 prompts: &Prompts,
2948 run_id: &str,
2949 ) {
2950 for (wi, seat, out) in results.iter_mut() {
2951 let AgentOutcome::Ok(o) = &*out else {
2952 continue;
2953 };
2954 if !has_unconfirmed_command(&o.commands) {
2955 continue;
2956 }
2957 let Some(job) = sent.get(*wi) else { continue };
2958 if !has_context(&job.spec, seat, job.sessions) {
2959 self.state.event(
2960 "implement",
2961 format!(
2962 "{}: the reply named a command whose own CLI never confirmed the exit \
2963 status of, but there is no session left to resume",
2964 seat.key
2965 ),
2966 );
2967 continue;
2968 }
2969 self.state.event(
2970 "implement",
2971 format!(
2972 "{}: the reply named a command whose own CLI never confirmed the exit \
2973 status of; resuming the conversation",
2974 seat.key
2975 ),
2976 );
2977 let mut retry = job.clone();
2978 retry.seat = seat.clone();
2979 retry.prompt = prompt::resume_incomplete(
2980 "a command in your last reply had no confirmed exit status",
2981 );
2982 retry.timeout = retry_budget(job.timeout, true);
2983 retry.stem = format!("{}-confirm", job.stem);
2984 let cache = self.state.config.cache_dir();
2985 let ctx = WaveCtx {
2986 carry_seats: false,
2987 pauses: Some((&self.pause, &self.interrupt)),
2988 run: run_id,
2989 node: "implement",
2990 prompts,
2991 cache: cache.as_deref(),
2992 round: None,
2993 };
2994 let (resumed_seat, resumed) =
2995 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2996 *seat = resumed_seat;
2997 *out = resumed;
2998 }
2999 }
3000
3001 async fn continue_fix_report(
3022 &mut self,
3023 mut seat: SeatState,
3024 parse_err: String,
3025 job: &SeatJob,
3026 prompts: &Prompts,
3027 run_id: &str,
3028 round: usize,
3029 ) -> (
3030 SeatState,
3031 Option<FixReport>,
3032 Option<String>,
3033 ContinuationRecord,
3034 ) {
3035 let mut last_err = parse_err;
3036 let mut cumulative_wait_ms = 0u64;
3037 let mut attempts = 0usize;
3038 loop {
3039 if !has_context(&job.spec, &seat, job.sessions) {
3040 self.state.event(
3041 "fix",
3042 format!(
3043 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3044 session left to resume into"
3045 ),
3046 );
3047 let outcome = if attempts == 0 {
3048 ContinuationOutcome::NoSession
3049 } else {
3050 ContinuationOutcome::Exhausted
3051 };
3052 return (
3053 seat,
3054 None,
3055 Some(format!("unparsable fix report: {last_err}")),
3056 ContinuationRecord {
3057 attempts,
3058 cumulative_wait_ms,
3059 outcome,
3060 },
3061 );
3062 }
3063 if attempts >= MAX_FIX_CONTINUATIONS {
3064 self.state.event(
3065 "fix",
3066 format!(
3067 "round {round}: fixer's reply still had no adoption report after \
3068 {attempts} continuation(s) ({last_err}); giving up"
3069 ),
3070 );
3071 return (
3072 seat,
3073 None,
3074 Some(format!(
3075 "unparsable fix report after {attempts} continuation(s): {last_err}"
3076 )),
3077 ContinuationRecord {
3078 attempts,
3079 cumulative_wait_ms,
3080 outcome: ContinuationOutcome::Exhausted,
3081 },
3082 );
3083 }
3084 attempts += 1;
3085 self.state.event(
3086 "fix",
3087 format!(
3088 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3089 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3090 ),
3091 );
3092 let mut retry = job.clone();
3093 retry.seat = seat.clone();
3094 retry.prompt = prompt::resume_incomplete(&last_err);
3095 retry.timeout = retry_budget(job.timeout, true);
3096 retry.stem = format!("{}-continue{attempts}", job.stem);
3097 let cache = self.state.config.cache_dir();
3098 let ctx = WaveCtx {
3099 carry_seats: false,
3100 pauses: Some((&self.pause, &self.interrupt)),
3101 run: run_id,
3102 node: "fix",
3103 prompts,
3104 cache: cache.as_deref(),
3105 round: Some(round),
3106 };
3107 let (resumed_seat, resumed_out) = run_one(
3108 retry,
3109 Arc::clone(&self.sem),
3110 &ctx,
3111 &mut self.state,
3112 attempts,
3113 )
3114 .await;
3115 seat = resumed_seat;
3116 match resumed_out {
3117 AgentOutcome::Ok(o) => {
3118 cumulative_wait_ms += o.duration_ms;
3119 match verdict::extract_json::<FixReport>(&o.text) {
3120 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3121 self.state.event(
3122 "fix",
3123 format!(
3124 "round {round}: fixer's adoption report recovered after \
3125 {attempts} continuation(s)"
3126 ),
3127 );
3128 return (
3129 seat,
3130 Some(report),
3131 None,
3132 ContinuationRecord {
3133 attempts,
3134 cumulative_wait_ms,
3135 outcome: ContinuationOutcome::Resumed,
3136 },
3137 );
3138 }
3139 Ok(_) => {
3147 last_err = "the reply parsed, but it reported a command whose own CLI \
3148 never confirmed an exit status"
3149 .to_owned();
3150 }
3151 Err(e) => last_err = e.to_string(),
3152 }
3153 }
3154 AgentOutcome::Quota(o) => {
3155 cumulative_wait_ms += o.duration_ms;
3156 self.state.quota.push(QuotaLoss {
3157 seat: seat.key.clone(),
3158 node: "fix".to_owned(),
3159 at: Timestamp::now(),
3160 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3161 });
3162 self.state.event(
3163 "fix",
3164 format!(
3165 "round {round}: continuation rate limited (quota); not retrying now"
3166 ),
3167 );
3168 return (
3169 seat,
3170 None,
3171 Some("rate limited (quota) while recovering the fix report".to_owned()),
3172 ContinuationRecord {
3173 attempts,
3174 cumulative_wait_ms,
3175 outcome: ContinuationOutcome::QuotaLost,
3176 },
3177 );
3178 }
3179 AgentOutcome::Dropped(o) => {
3180 cumulative_wait_ms += o.duration_ms;
3181 let why = o
3182 .dropped
3183 .as_ref()
3184 .map(|d| d.why.as_str())
3185 .unwrap_or("the CLI ended the stream without delivering its answer");
3186 last_err = format!("the CLI dropped the stream ({why})");
3187 }
3188 AgentOutcome::Failed(e) => last_err = e,
3189 AgentOutcome::ResourceBlocked(e) => {
3192 return (
3193 seat,
3194 None,
3195 Some(format!("the continuation was not started: {e}")),
3196 ContinuationRecord {
3197 attempts: attempts.saturating_sub(1),
3198 cumulative_wait_ms,
3199 outcome: ContinuationOutcome::NoSession,
3200 },
3201 );
3202 }
3203 }
3204 }
3205 }
3206
3207 fn after_implement(&mut self) -> Result<()> {
3208 if !self.state.candidate_preservation.is_empty() {
3209 bail!("candidate preservation incomplete; judging is unsafe");
3210 }
3211 if self.state.leaks.is_empty() {
3213 let cfg = self.state.config.blind.clone();
3214 let mut leaks = Vec::new();
3215 for c in &self.state.candidates {
3216 let Some(patch) =
3217 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3218 else {
3219 continue;
3220 };
3221 leaks.extend(blind::scan(
3222 &format!("candidate {} patch", c.label),
3223 &patch,
3224 &cfg.vendor_tokens,
3225 ));
3226 }
3227 if !leaks.is_empty() {
3228 let summary = leaks
3229 .iter()
3230 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3231 .collect::<Vec<_>>()
3232 .join(", ");
3233 match cfg.on_leak {
3234 LeakPolicy::Fail => {
3235 self.state.status = RunStatus::Failed;
3236 self.state
3237 .event("blind", format!("vendor text in a patch: {summary}"));
3238 self.state.leaks = leaks;
3239 self.state.save()?;
3240 self.settle_questions();
3241 bail!(
3242 "blind.on_leak = \"fail\" and vendor text reached a \
3243 judged patch: {summary}"
3244 );
3245 }
3246 LeakPolicy::Redact => self.state.event(
3247 "blind",
3248 format!("redacting vendor text for judging: {summary}"),
3249 ),
3250 LeakPolicy::Warn => self.state.event(
3251 "blind",
3252 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3253 ),
3254 }
3255 self.state.leaks = leaks;
3256 }
3257 }
3258
3259 if self.state.viable().is_empty() {
3260 if self.state.all_candidates_verified_noop() {
3261 self.state.status = RunStatus::VerifiedNoop;
3272 self.state.save()?;
3273 self.settle_questions();
3274 return Ok(());
3275 }
3276 self.state.status = RunStatus::Failed;
3277 self.state.save()?;
3278 self.settle_questions();
3279 bail!("no candidate produced a change; nothing to judge");
3280 }
3281 self.state.status = RunStatus::Judging;
3282 self.state.save()?;
3283 Ok(())
3284 }
3285
3286 async fn judge(&mut self) -> Result<()> {
3289 let run_id = self.state.id.clone();
3294 let prompts = self.state.config.prompts.clone();
3295 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3296 return Ok(());
3297 }
3298 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3299 if viable.len() == 1 {
3300 self.state.judge_skipped = true;
3307 self.state.event(
3308 "judge",
3309 format!(
3310 "only candidate {} produced a change; judging skipped",
3311 viable[0].label
3312 ),
3313 );
3314 self.state.save()?;
3315 return Ok(());
3316 }
3317 self.state.status = RunStatus::Judging;
3318
3319 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3320 let language = self.state.config.graph.language.clone();
3321 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3322 let sessions = self.state.config.graph.sessions;
3323 let artifacts = agent::artifacts_dir(&self.state.dir());
3324 let root = self.state.worktree_root();
3325 let base_short = short(&self.state.base_commit);
3326
3327 let mut jobs = Vec::new();
3328 let mut orders = Vec::new();
3329 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3330 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3331 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3332 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3333 let seat_key = format!("judge-{}", j + 1);
3334 let seat = self.seat(&seat_key, &spec.id);
3335 jobs.push(SeatJob {
3336 prompt: prompt::judge(
3337 &self.state.instruction,
3338 &views,
3339 self.roles.judges.len(),
3340 &base_short,
3341 &language,
3342 ),
3343 spec,
3344 seat,
3345 cwd: root.join(format!("judge-{}", j + 1)),
3346 timeout,
3347 allow_write: false,
3348 sessions,
3349 artifacts: artifacts.clone(),
3350 stem: format!("judge-{}", j + 1),
3351 handover: None,
3352 });
3353 }
3354
3355 self.state.event(
3356 "judge",
3357 format!(
3358 "{} judges ranking {} candidates blind",
3359 jobs.len(),
3360 viable.len()
3361 ),
3362 );
3363 let labels_for_check = labels.clone();
3364 let mut quota_losses = Vec::new();
3365 let cache = self.state.config.cache_dir();
3366 let ctx = WaveCtx {
3367 carry_seats: false,
3368 pauses: Some((&self.pause, &self.interrupt)),
3369 run: &run_id,
3370 node: "judge",
3371 prompts: &prompts,
3372 cache: cache.as_deref(),
3373 round: None,
3374 };
3375 let results = ask_json_wave::<Ranking>(
3376 jobs,
3377 Arc::clone(&self.sem),
3378 self.state.config.graph.retries,
3379 &self.roles.judge_roster,
3380 &ctx,
3381 &mut quota_losses,
3382 &mut self.state,
3383 &move |r: &Ranking| r.validate(&labels_for_check),
3384 )
3385 .await;
3386 if self.state.resource_block.is_some() {
3387 return self.stop_for_resource();
3388 }
3389 self.state.quota.extend(quota_losses);
3390
3391 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3392 let agent_id = seat.agent.clone();
3393 self.state.seats.insert(seat.key.clone(), seat);
3394 let mut record = Judgement {
3395 judge: j + 1,
3396 seat: format!("judge-{}", j + 1),
3397 agent: agent_id,
3398 ranking: Vec::new(),
3399 reasons: BTreeMap::new(),
3400 confidence: None,
3401 strengths: BTreeMap::new(),
3402 order: orders[j].clone(),
3403 failed: None,
3404 duration_ms: 0,
3405 };
3406 match res {
3407 Ok((ranking, out)) => {
3408 record.ranking = ranking.normalized();
3409 record.reasons = ranking.reasons;
3410 record.strengths = ranking.strengths;
3411 record.confidence = ranking.confidence;
3412 record.duration_ms = out.duration_ms;
3413 self.state.event(
3414 "judge",
3415 format!(
3416 "judge {} ranked {}",
3417 j + 1,
3418 record.ranking.iter().collect::<String>()
3419 ),
3420 );
3421 }
3422 Err(e) => {
3423 record.failed = Some(e.to_string());
3424 self.state
3425 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3426 }
3427 }
3428 self.state.judgements.push(record);
3429 self.state.save()?;
3430 }
3431 Ok(())
3432 }
3433
3434 async fn deliberate(&mut self) -> Result<()> {
3437 let run_id = self.state.id.clone();
3442 let prompts = self.state.config.prompts.clone();
3443 if !self.state.deliberation.is_empty() {
3444 return Ok(());
3445 }
3446 let tops: Vec<char> = self
3447 .state
3448 .judgements
3449 .iter()
3450 .filter_map(|j| j.ranking.first().copied())
3451 .collect();
3452 let rounds = self.state.config.graph.deliberate_rounds;
3453 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3454 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3455 self.state.event(
3456 "deliberate",
3457 format!("judges agreed on {} outright; no deliberation", tops[0]),
3458 );
3459 }
3460 self.state.status = RunStatus::Voting;
3461 self.state.save()?;
3462 return Ok(());
3463 }
3464
3465 self.state.status = RunStatus::Deliberating;
3466 self.state.event(
3467 "deliberate",
3468 format!(
3469 "split: first choices were {} — opening {rounds} round(s)",
3470 tops.iter().collect::<String>()
3471 ),
3472 );
3473
3474 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3475 let language = self.state.config.graph.language.clone();
3476 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3477 let sessions = self.state.config.graph.sessions;
3478 let artifacts = agent::artifacts_dir(&self.state.dir());
3479 let root = self.state.worktree_root();
3480 let base_short = short(&self.state.base_commit);
3481
3482 for round in 1..=rounds {
3486 let mut turns: Vec<DeliberationTurn> = Vec::new();
3487 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3488 if self.state.judgements[j].failed.is_some() {
3489 continue;
3490 }
3491 let seat_key = format!("judge-{}", j + 1);
3492 let spec = self.occupant(&seat_key, spec);
3493 let mut seat = self.seat(&seat_key, &spec.id);
3494 let transcript = self.transcript(&turns, j);
3495 let build = |context: Option<&str>| {
3496 prompt::deliberate(
3497 &self.state.instruction,
3498 context,
3499 &transcript,
3500 round,
3501 rounds,
3502 &language,
3503 )
3504 };
3505 let block = self.candidate_block(&viable, &base_short);
3506 let full = build(Some(&block));
3507 let text = if has_context(&spec, &seat, sessions) {
3508 build(None)
3509 } else {
3510 full.clone()
3511 };
3512 let job = SeatJob {
3513 spec,
3514 seat: seat.clone(),
3515 prompt: text,
3516 cwd: root.join(format!("judge-{}", j + 1)),
3517 timeout,
3518 allow_write: false,
3519 sessions,
3520 artifacts: artifacts.clone(),
3521 stem: format!("delib-{round}-judge-{}", j + 1),
3522 handover: Some(full),
3523 };
3524 let cache = self.state.config.cache_dir();
3525 let ctx = WaveCtx {
3526 carry_seats: false,
3527 pauses: Some((&self.pause, &self.interrupt)),
3528 run: &run_id,
3529 node: "deliberate",
3530 prompts: &prompts,
3531 cache: cache.as_deref(),
3532 round: None,
3533 };
3534 let mut losses = Vec::new();
3538 let mut results = ask_wave_with::<String>(
3539 vec![job],
3540 Arc::clone(&self.sem),
3541 0,
3542 &self.roles.judge_roster,
3543 &ctx,
3544 &mut losses,
3545 &mut self.state,
3546 &|text: &str| {
3547 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3548 },
3549 )
3550 .await;
3551 self.state.quota.extend(losses);
3552 let (updated, res, _) = results.pop().expect("one job in, one result out");
3553 seat = updated;
3554 let agent_id = seat.agent.clone();
3555 self.state.seats.insert(seat.key.clone(), seat);
3556 let body = match res {
3557 Ok((body, _)) => body,
3558 Err(e) => {
3561 self.state
3562 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3563 continue;
3564 }
3565 };
3566 let tentative = verdict::extract_json::<Position>(&body)
3567 .ok()
3568 .and_then(|p| p.tentative)
3569 .and_then(|s| s.trim().chars().next())
3570 .map(|c| c.to_ascii_uppercase());
3571 self.state.event(
3572 "deliberate",
3573 format!(
3574 "round {round}: judge {} now favours {}",
3575 j + 1,
3576 tentative.map_or("—".to_owned(), |c| c.to_string())
3577 ),
3578 );
3579 turns.push(DeliberationTurn {
3580 judge: j + 1,
3581 agent: agent_id,
3582 body: blind::sanitize_prose(&body, &self.state.config.blind),
3583 tentative,
3584 });
3585 }
3586 self.state
3587 .deliberation
3588 .push(DeliberationRound { round, turns });
3589 self.state.save()?;
3590 }
3591
3592 self.state.status = RunStatus::Voting;
3593 self.state.save()?;
3594 Ok(())
3595 }
3596
3597 async fn vote(&mut self) -> Result<()> {
3600 let run_id = self.state.id.clone();
3605 let prompts = self.state.config.prompts.clone();
3606 if !self.state.votes.is_empty() {
3607 return Ok(());
3608 }
3609 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3610 if viable.len() == 1 {
3611 return Ok(());
3612 }
3613 self.state.status = RunStatus::Voting;
3614
3615 let language = self.state.config.graph.language.clone();
3616 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3617 let sessions = self.state.config.graph.sessions;
3618 let artifacts = agent::artifacts_dir(&self.state.dir());
3619 let root = self.state.worktree_root();
3620 let base_short = short(&self.state.base_commit);
3621 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3622
3623 let mut jobs = Vec::new();
3624 let mut seats_at = Vec::new();
3625 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3626 if self
3627 .state
3628 .judgements
3629 .get(j)
3630 .is_some_and(|r| r.failed.is_some())
3631 {
3632 continue;
3633 }
3634 let seat_key = format!("judge-{}", j + 1);
3635 let spec = self.occupant(&seat_key, spec);
3636 let seat = self.seat(&seat_key, &spec.id);
3637 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3638 let text = if has_context(&spec, &seat, sessions) {
3639 prompt::final_vote(&viable, &language)
3640 } else {
3641 full.clone()
3642 };
3643 jobs.push(SeatJob {
3644 spec,
3645 seat,
3646 prompt: text,
3647 cwd: root.join(format!("judge-{}", j + 1)),
3648 timeout,
3649 allow_write: false,
3650 sessions,
3651 artifacts: artifacts.clone(),
3652 stem: format!("vote-judge-{}", j + 1),
3653 handover: Some(full),
3654 });
3655 seats_at.push(j);
3656 }
3657
3658 self.state.event(
3659 "vote",
3660 format!(
3661 "collecting {} final votes one by one, privately",
3662 jobs.len()
3663 ),
3664 );
3665 let allowed = viable.clone();
3666 let mut quota_losses = Vec::new();
3667 let cache = self.state.config.cache_dir();
3668 let ctx = WaveCtx {
3669 carry_seats: false,
3670 pauses: Some((&self.pause, &self.interrupt)),
3671 run: &run_id,
3672 node: "vote",
3673 prompts: &prompts,
3674 cache: cache.as_deref(),
3675 round: None,
3676 };
3677 let results = ask_json_wave::<FinalVote>(
3678 jobs,
3679 Arc::clone(&self.sem),
3680 self.state.config.graph.retries,
3681 &self.roles.judge_roster,
3682 &ctx,
3683 &mut quota_losses,
3684 &mut self.state,
3685 &move |v: &FinalVote| match v.label() {
3686 Some(c) if allowed.contains(&c) => Ok(()),
3687 other => bail!("vote {other:?} is not one of {allowed:?}"),
3688 },
3689 )
3690 .await;
3691 if self.state.resource_block.is_some() {
3692 return self.stop_for_resource();
3693 }
3694 self.state.quota.extend(quota_losses);
3695
3696 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3697 let agent_id = seat.agent.clone();
3698 self.state.seats.insert(seat.key.clone(), seat);
3699 let initial = self
3700 .state
3701 .judgements
3702 .get(j)
3703 .and_then(|r| r.ranking.first().copied());
3704 let mut record = VoteRecord {
3705 judge: j + 1,
3706 agent: agent_id,
3707 vote: None,
3708 reason: String::new(),
3709 changed: false,
3710 };
3711 match res {
3712 Ok((v, _)) => {
3713 record.vote = v.label();
3714 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3715 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3716 self.state.event(
3717 "vote",
3718 format!(
3719 "judge {} voted {}{}",
3720 j + 1,
3721 record.vote.unwrap_or('?'),
3722 if record.changed { " (changed)" } else { "" }
3723 ),
3724 );
3725 }
3726 Err(e) => {
3727 self.state
3728 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3729 }
3730 }
3731 self.state.votes.push(record);
3732 self.state.save()?;
3733 }
3734 Ok(())
3735 }
3736
3737 fn tally(&mut self) -> Result<()> {
3740 if self.state.tally.is_some() {
3741 return Ok(());
3742 }
3743 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3744 let tops: Vec<char> = self
3745 .state
3746 .judgements
3747 .iter()
3748 .filter_map(|j| j.ranking.first().copied())
3749 .collect();
3750 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3751
3752 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3755 let mut cast: Vec<char> = Vec::new();
3756 for (i, j) in self.state.judgements.iter().enumerate() {
3757 let vote = self
3758 .state
3759 .votes
3760 .iter()
3761 .find(|v| v.judge == i + 1)
3762 .and_then(|v| v.vote)
3763 .or_else(|| j.ranking.first().copied());
3764 if let Some(v) = vote {
3765 *first_choice.entry(v).or_insert(0) += 1;
3766 cast.push(v);
3767 }
3768 }
3769
3770 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3771 for j in &self.state.judgements {
3772 let n = j.ranking.len();
3773 for (pos, label) in j.ranking.iter().enumerate() {
3774 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3775 }
3776 }
3777
3778 let best = first_choice.values().copied().max().unwrap_or(0);
3779 let mut leaders: Vec<char> = first_choice
3780 .iter()
3781 .filter(|(_, v)| **v == best)
3782 .map(|(k, _)| *k)
3783 .collect();
3784 let mut tie_break = None;
3785 if leaders.len() > 1 {
3786 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3787 let borda_leaders: Vec<char> = leaders
3788 .iter()
3789 .copied()
3790 .filter(|l| borda[l] == top_borda)
3791 .collect();
3792 tie_break = Some(if borda_leaders.len() == 1 {
3793 format!(
3794 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3795 leaders.len()
3796 )
3797 } else {
3798 format!(
3799 "{} way tie on both first-choice votes and Borda points, broken by label order",
3800 leaders.len()
3801 )
3802 });
3803 leaders = borda_leaders;
3804 leaders.sort_unstable();
3805 }
3806 let winner = *leaders
3807 .first()
3808 .or(viable.first())
3809 .context("no candidate to declare a winner from")?;
3810
3811 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3812 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3813 let deliberated = !self.state.deliberation.is_empty();
3814
3815 let quota_seats: std::collections::BTreeSet<&str> =
3819 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3820 let mut present = 0usize;
3821 for (i, j) in self.state.judgements.iter().enumerate() {
3822 if quota_seats.contains(j.seat.as_str()) {
3823 continue;
3824 }
3825 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3826 let voted = self
3827 .state
3828 .votes
3829 .iter()
3830 .any(|v| v.judge == i + 1 && v.vote.is_some());
3831 if ranked || voted {
3832 present += 1;
3833 }
3834 }
3835 let needs_quorum = viable.len() > 1;
3841 let judges_total = if needs_quorum {
3842 self.roles.judges.len()
3843 } else {
3844 0
3845 };
3846 let quorum = if needs_quorum {
3847 judges_total / 2 + 1
3848 } else {
3849 0
3850 };
3851 let met_quorum = !needs_quorum || present >= quorum;
3852 let uncontested = (!needs_quorum).then(|| {
3853 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3854 });
3855
3856 self.state.event(
3857 "tally",
3858 match &uncontested {
3859 Some(reason) => format!("winner {winner} — {reason}"),
3860 None => format!(
3861 "winner {winner} — votes {} | initial {} | {} changed | \
3862 {present}/{judges_total} judges{}",
3863 first_choice
3864 .iter()
3865 .map(|(k, v)| format!("{k}:{v}"))
3866 .collect::<Vec<_>>()
3867 .join(" "),
3868 if unanimous_initial {
3869 "unanimous"
3870 } else {
3871 "split"
3872 },
3873 changed_votes,
3874 if met_quorum {
3875 String::new()
3876 } else {
3877 format!(" — below quorum ({quorum} required)")
3878 },
3879 ),
3880 },
3881 );
3882 if !met_quorum {
3883 self.state.event(
3884 "stall",
3885 format!(
3886 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3887 the run stops here, resumable"
3888 ),
3889 );
3890 }
3891 self.state.tally = Some(Tally {
3892 first_choice,
3893 borda,
3894 winner,
3895 rankings: tops.len(),
3896 unanimous_initial,
3897 deliberated,
3898 changed_votes,
3899 unanimous_final,
3900 tie_break,
3901 judges: judges_total,
3902 present,
3903 quorum,
3904 met_quorum,
3905 uncontested,
3906 });
3907 self.state.status = if met_quorum {
3908 RunStatus::Reviewing
3909 } else {
3910 RunStatus::Stalled
3911 };
3912 self.state.save()?;
3913 Ok(())
3914 }
3915
3916 #[allow(clippy::too_many_lines)]
3937 async fn recover_stall(&mut self) -> Result<bool> {
3938 let run_id = self.state.id.clone();
3943 let prompts = self.state.config.prompts.clone();
3944 let quota_seats: BTreeSet<&str> =
3949 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3950 let absent: Vec<String> = self
3951 .state
3952 .judgements
3953 .iter()
3954 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3955 .map(|j| j.seat.clone())
3956 .collect();
3957 if absent.is_empty() {
3958 return Ok(false);
3959 }
3960 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3961 if viable.len() <= 1 {
3962 return Ok(false);
3963 }
3964 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3965 let language = self.state.config.graph.language.clone();
3966 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3967 let sessions = self.state.config.graph.sessions;
3968 let artifacts = agent::artifacts_dir(&self.state.dir());
3969 let root = self.state.worktree_root();
3970 let base_short = short(&self.state.base_commit);
3971 let candidates: Vec<Candidate> = viable.clone();
3972
3973 let mut positions: Vec<usize> = absent
3975 .iter()
3976 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3977 .collect();
3978 if positions.is_empty() {
3979 return Ok(false);
3980 }
3981 positions.sort_unstable();
3982 positions.dedup();
3983
3984 let mut judge_jobs = Vec::new();
3986 for &j in &positions {
3987 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3988 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3989 let seat_key = format!("judge-{}", j + 1);
3990 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3991 let seat = self.seat(&seat_key, &spec.id);
3992 judge_jobs.push(SeatJob {
3993 spec,
3994 seat,
3995 prompt: prompt::judge(
3996 &self.state.instruction,
3997 &views,
3998 self.roles.judges.len(),
3999 &base_short,
4000 &language,
4001 ),
4002 cwd: root.join(seat_key),
4003 timeout,
4004 allow_write: false,
4005 sessions,
4006 artifacts: artifacts.clone(),
4007 stem: format!("judge-{}-recover", j + 1),
4008 handover: None,
4009 });
4010 }
4011
4012 let labels_for_check = labels.clone();
4013 let mut judge_losses = Vec::new();
4014 let retries = self.state.config.graph.retries;
4015 let cache = self.state.config.cache_dir();
4016 let ctx = WaveCtx {
4017 carry_seats: false,
4018 pauses: Some((&self.pause, &self.interrupt)),
4019 run: &run_id,
4020 node: "judge",
4021 prompts: &prompts,
4022 cache: cache.as_deref(),
4023 round: None,
4024 };
4025 let results = ask_json_wave::<Ranking>(
4026 judge_jobs,
4027 Arc::clone(&self.sem),
4028 retries,
4029 &self.roles.judge_roster,
4030 &ctx,
4031 &mut judge_losses,
4032 &mut self.state,
4033 &move |r: &Ranking| r.validate(&labels_for_check),
4034 )
4035 .await;
4036 if self.state.resource_block.is_some() {
4037 self.stop_for_resource()?;
4038 return Ok(false);
4039 }
4040
4041 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4043 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4044 let agent_id = seat.agent.clone();
4045 self.state.seats.insert(seat.key.clone(), seat);
4046 let record = &mut self.state.judgements[j];
4047 match res {
4048 Ok((ranking, out)) => {
4049 record.agent = agent_id;
4050 record.ranking = ranking.normalized();
4051 record.reasons = ranking.reasons;
4052 record.strengths = ranking.strengths;
4053 record.confidence = ranking.confidence;
4054 record.failed = None;
4055 record.duration_ms = out.duration_ms;
4056 recovered.insert(j);
4057 self.state.event(
4058 "recover",
4059 format!("judge {} ranked again after the limit", j + 1),
4060 );
4061 }
4062 Err(e) => {
4063 self.state
4064 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4065 }
4066 }
4067 }
4068
4069 let mut vote_jobs = Vec::new();
4071 let mut vote_pos: Vec<usize> = Vec::new();
4072 for &j in &recovered {
4073 let seat_key = format!("judge-{}", j + 1);
4074 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4075 let seat = self.seat(&seat_key, &spec.id);
4076 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4077 let text = if has_context(&spec, &seat, sessions) {
4078 prompt::final_vote(&labels, &language)
4079 } else {
4080 full.clone()
4081 };
4082 vote_jobs.push(SeatJob {
4083 spec,
4084 seat,
4085 prompt: text,
4086 cwd: root.join(seat_key),
4087 timeout,
4088 allow_write: false,
4089 sessions,
4090 artifacts: artifacts.clone(),
4091 stem: format!("vote-judge-{}-recover", j + 1),
4092 handover: Some(full),
4093 });
4094 vote_pos.push(j);
4095 }
4096 let allowed = labels.clone();
4097 let mut vote_losses = Vec::new();
4098 let vote_retries = self.state.config.graph.retries;
4099 let vote_cache = self.state.config.cache_dir();
4100 let ctx = WaveCtx {
4101 carry_seats: false,
4102 pauses: Some((&self.pause, &self.interrupt)),
4103 run: &run_id,
4104 node: "vote",
4105 prompts: &prompts,
4106 cache: vote_cache.as_deref(),
4107 round: None,
4108 };
4109 let votes = ask_json_wave::<FinalVote>(
4110 vote_jobs,
4111 Arc::clone(&self.sem),
4112 vote_retries,
4113 &self.roles.judge_roster,
4114 &ctx,
4115 &mut vote_losses,
4116 &mut self.state,
4117 &move |v: &FinalVote| match v.label() {
4118 Some(c) if allowed.contains(&c) => Ok(()),
4119 other => bail!("vote {other:?} is not one of {allowed:?}"),
4120 },
4121 )
4122 .await;
4123 if self.state.resource_block.is_some() {
4124 self.stop_for_resource()?;
4125 return Ok(false);
4126 }
4127 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4128 let agent_id = seat.agent.clone();
4129 self.state.seats.insert(seat.key.clone(), seat);
4130 match res {
4131 Ok((v, _)) => {
4132 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4133 rec.vote = v.label();
4134 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4135 } else {
4136 self.state.votes.push(VoteRecord {
4137 judge: j + 1,
4138 agent: agent_id,
4139 vote: v.label(),
4140 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4141 changed: false,
4142 });
4143 }
4144 self.state.event(
4145 "recover",
4146 format!("judge {} voted again after the limit", j + 1),
4147 );
4148 }
4149 Err(e) => {
4150 self.state
4151 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4152 }
4153 }
4154 }
4155
4156 let recovered_keys: BTreeSet<String> = recovered
4160 .iter()
4161 .map(|&j| format!("judge-{}", j + 1))
4162 .collect();
4163 self.state
4164 .quota
4165 .retain(|q| !recovered_keys.contains(&q.seat));
4166 for loss in judge_losses.into_iter().chain(vote_losses) {
4170 if recovered_keys.contains(&loss.seat) {
4171 continue;
4172 }
4173 self.state.quota.retain(|q| q.seat != loss.seat);
4174 self.state.quota.push(loss);
4175 }
4176
4177 self.state.tally = None;
4179 self.tally()?;
4180 Ok(self
4181 .state
4182 .tally
4183 .as_ref()
4184 .map(|t| t.met_quorum)
4185 .unwrap_or(false))
4186 }
4187
4188 async fn graft(&mut self) -> Result<()> {
4204 let Err(e) = self.graft_inner().await else {
4205 return Ok(());
4206 };
4207 if e.is::<crate::repo_lock::ResourceBlocked>() {
4208 return Err(e);
4209 }
4210 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4211 self.state.status = RunStatus::Blocked;
4212 self.state.event("graft", format!("blocked: {e:#}"));
4213 self.state.save()?;
4214 return Err(e);
4215 }
4216 if let Some(rec) = self.state.graft.clone()
4221 && !rec.outcome.settled()
4222 {
4223 if let Some(winner) = self.state.winner().cloned() {
4224 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4225 if re.is::<crate::repo_lock::ResourceBlocked>() {
4226 return Err(re);
4227 }
4228 self.state.status = RunStatus::Blocked;
4229 self.state.event("graft", format!("blocked: {re:#}"));
4230 self.state.save()?;
4231 return Err(re);
4232 }
4233 }
4234 if let Some(g) = self.state.graft.as_mut() {
4235 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4236 }
4237 }
4238 self.state.event("graft", format!("skipped: {e:#}"));
4239 if let Err(e) = self.state.save() {
4240 tracing::warn!("save after graft: {e:#}");
4241 }
4242 Ok(())
4243 }
4244
4245 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4248 let _coordination = self.coordinate(wt).await?;
4249 crate::recovery::restore_repair(wt, pre)
4250 .await
4251 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4252 }
4253
4254 fn graft_skipped(&mut self, why: &str) {
4256 let message = format!("skipped: {why}");
4257 if !self
4258 .state
4259 .events
4260 .iter()
4261 .any(|e| e.node == "graft" && e.message == message)
4262 {
4263 self.state.event("graft", message);
4264 }
4265 }
4266
4267 async fn graft_inner(&mut self) -> Result<()> {
4268 if let Some(rec) = self.state.graft.clone()
4271 && !rec.outcome.settled()
4272 {
4273 if self.state.reviews.is_empty()
4277 && let Some(winner) = self.state.winner().cloned()
4278 {
4279 self.graft_restore(&winner.worktree, &rec.pre).await?;
4280 }
4281 if rec.outcome == GraftOutcome::WaitingForRestore {
4282 self.state.graft = None;
4283 self.state.save()?;
4284 } else {
4285 if let Some(g) = self.state.graft.as_mut() {
4286 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4287 }
4288 self.state.event(
4289 "graft",
4290 "rolled back: the previous round was interrupted; the winner is untouched",
4291 );
4292 self.state.save()?;
4293 return Ok(());
4294 }
4295 }
4296 let plan = match graft::plan(&self.state) {
4297 Ok(plan) => plan,
4298 Err(skip) => {
4299 if skip != graft::Skip::AlreadyDone {
4300 self.graft_skipped(skip.reason());
4301 self.state.save()?;
4302 }
4303 return Ok(());
4304 }
4305 };
4306 let Some(winner) = self.state.winner().cloned() else {
4307 return Ok(());
4308 };
4309 let wt = winner.worktree.clone();
4310 if !git::is_clean(&wt).await? {
4311 self.graft_skipped("the winner's worktree has uncommitted changes");
4312 self.state.save()?;
4313 return Ok(());
4314 }
4315 let pre = git::rev_parse(&wt, "HEAD").await?;
4316 self.state.graft = Some(GraftRecord {
4319 pre: pre.clone(),
4320 post: None,
4321 lines: None,
4322 limit: None,
4323 agent: None,
4324 outcome: GraftOutcome::Running,
4325 requested: graft::requested(&plan),
4326 adoption: None,
4327 adoption_error: None,
4328 });
4329 self.state.event(
4330 "graft",
4331 format!(
4332 "porting strengths of candidate(s) {} onto {}",
4333 plan.strengths
4334 .iter()
4335 .map(|(l, _, _)| l.to_string())
4336 .collect::<Vec<_>>()
4337 .join(", "),
4338 plan.winner
4339 ),
4340 );
4341 self.state.save()?;
4342
4343 let fix_prompt = prompt::graft(
4344 &self.state.instruction,
4345 plan.winner,
4346 &plan.strengths,
4347 &self.state.config.graph.language,
4348 );
4349 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4350 let sessions = self.state.config.graph.sessions;
4351 let artifacts = agent::artifacts_dir(&self.state.dir());
4352 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4353 let (_, seat, out) = self
4354 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4355 SeatJob {
4356 prompt: fix_prompt.clone(),
4357 spec,
4358 seat,
4359 cwd: wt.clone(),
4360 timeout,
4361 allow_write: true,
4362 sessions,
4363 artifacts: artifacts.clone(),
4364 stem: "graft".to_owned(),
4365 handover: None,
4366 }
4367 })
4368 .await;
4369 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4370 self.state.graft.as_mut().expect("graft record").outcome =
4373 GraftOutcome::WaitingForRestore;
4374 return self.stop_for_resource();
4375 }
4376 let mut why: Option<String> = None;
4377 match &out {
4378 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4379 why = Some("the fixer produced nothing usable".to_owned());
4380 }
4381 AgentOutcome::Ok(_) => {}
4382 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4383 AgentOutcome::Quota(o) => {
4384 self.state.quota.push(QuotaLoss {
4385 seat: seat.key.clone(),
4386 node: "graft".to_owned(),
4387 at: Timestamp::now(),
4388 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4389 });
4390 why = Some("rate limited (quota)".to_owned());
4391 }
4392 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4393 why = Some(format!(
4394 "the fixer failed: {}",
4395 e.lines()
4396 .next()
4397 .unwrap_or("")
4398 .chars()
4399 .take(160)
4400 .collect::<String>()
4401 ));
4402 }
4403 }
4404 let agent_id = seat.agent.clone();
4405 self.state.seats.insert(seat.key.clone(), seat);
4406
4407 let mut post = None;
4408 let mut lines = None;
4409 let _coordination = self.coordinate(&wt).await?;
4410 if why.is_none() {
4411 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4412 if let Ok(r) = &rescued {
4413 self.state.note_withheld("graft", &r.withheld);
4414 }
4415 let head = git::rev_parse(&wt, "HEAD").await?;
4416 let dirty = match &rescued {
4420 Ok(r) => git::status_porcelain(&wt)
4421 .await
4422 .map(|st| {
4423 st.lines().any(|l| {
4424 !r.withheld
4425 .iter()
4426 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4427 })
4428 })
4429 .unwrap_or(true),
4430 Err(_) => true,
4431 };
4432 if dirty {
4433 why = Some("the fixer left changes that could not be committed".to_owned());
4434 } else if head == pre {
4435 why = Some("the fixer changed nothing".to_owned());
4436 } else {
4437 lines = git::changed_lines(&wt, &pre, &head).await;
4438 post = Some(head);
4439 }
4440 }
4441 let (adoption, adoption_error) = match &out {
4442 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4443 _ => (None, None),
4444 };
4445 let requested = graft::requested(&plan);
4446 let outcome = match why {
4447 None => {
4448 self.state.event(
4449 "graft",
4450 format!(
4451 "applied: {}",
4452 lines.map_or_else(
4453 || "changed lines unavailable".to_owned(),
4454 |n| format!("{n} changed lines")
4455 )
4456 ),
4457 );
4458 GraftOutcome::Applied
4459 }
4460 Some(why) => {
4461 crate::recovery::restore_repair(&wt, &pre)
4464 .await
4465 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4466 self.state.event(
4467 "graft",
4468 format!("rolled back: {why}; the winner is untouched"),
4469 );
4470 GraftOutcome::RolledBack(why)
4471 }
4472 };
4473 self.state.graft = Some(GraftRecord {
4474 pre,
4475 post,
4476 lines,
4477 limit: None,
4478 agent: Some(agent_id),
4479 outcome,
4480 requested,
4481 adoption,
4482 adoption_error,
4483 });
4484 self.state.save()?;
4485 Ok(())
4486 }
4487
4488 async fn fold_losers(&mut self) -> Result<()> {
4491 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4492 return Ok(());
4493 };
4494 if !self.state.candidate_preservation.is_empty() {
4495 bail!("candidate preservation incomplete; refusing cleanup");
4496 }
4497 let repo = self.state.repo.clone();
4498 let _coordination = self.coordinate(&repo).await?;
4499 let mut folded = Vec::new();
4500 for i in 0..self.state.candidates.len() {
4501 let c = &self.state.candidates[i];
4502 if c.label == winner || c.folded {
4503 continue;
4504 }
4505 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4506 remove_preserved_worktree(&repo, &wt).await?;
4507 if git::branch_exists(&repo, &branch).await?
4508 && !git::branch_delete(&repo, &branch).await?
4509 {
4510 bail!("candidate branch removal refused");
4511 }
4512 self.state.candidates[i].folded = true;
4513 self.state.save()?;
4514 folded.push(label.to_string());
4515 }
4516 let root = self.state.worktree_root();
4518 for j in 1..=self.roles.judges.len() {
4519 let wt = root.join(format!("judge-{j}"));
4520 if wt.exists() {
4521 remove_preserved_worktree(&repo, &wt).await?;
4522 }
4523 }
4524 if self.state.config.graph.advise {
4527 for k in 1..=self.state.config.graph.advisors {
4528 let wt = root.join(format!("advisor-{k}"));
4529 if wt.exists() {
4530 remove_preserved_worktree(&repo, &wt).await?;
4531 }
4532 }
4533 }
4534 if !folded.is_empty() {
4535 self.state
4536 .event("fold", format!("folded candidates {}", folded.join(", ")));
4537 self.state.save()?;
4538 }
4539 Ok(())
4540 }
4541
4542 async fn sync_to_base(&mut self) -> Result<()> {
4582 if self.state.status == RunStatus::AlreadyInBase {
4583 return Ok(());
4584 }
4585 let conflicted = self
4586 .state
4587 .base_sync
4588 .as_ref()
4589 .is_some_and(|s| s.conflict.is_some());
4590 let Some(winner) = self.state.winner().cloned() else {
4591 return Ok(());
4592 };
4593
4594 let repo = self.state.repo.clone();
4595 let remote = self.state.config.merge.remote.clone();
4596 let base_branch = self.state.base_branch.clone();
4597 let tracking = format!("{remote}/{base_branch}");
4598
4599 git::fetch(&repo, &remote, &base_branch).await.ok();
4600 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4605 return Ok(());
4606 };
4607
4608 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4609 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4610 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4611
4612 if (behind > 0 || conflicted || head == tip)
4621 && self
4622 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4623 .await?
4624 {
4625 return Ok(());
4626 }
4627 if conflicted {
4628 return Ok(());
4629 }
4630
4631 if behind == 0 {
4632 if let Some(from) = self
4641 .state
4642 .rebase_fixes
4643 .iter()
4644 .rev()
4645 .find_map(|r| r.from.clone())
4646 && from != head
4647 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4648 .await
4649 .is_ok_and(|o| o.ok())
4650 {
4651 git::sync_to_head(&winner.worktree).await?;
4652 }
4653 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4659 if let Some(why) = &conflict {
4660 self.state.status = RunStatus::Blocked;
4661 self.state.event("land", why.clone());
4662 }
4663 self.state.base_sync = Some(BaseSync {
4664 tip,
4665 behind: 0,
4666 attempts,
4667 conflict,
4668 already_in: None,
4669 });
4670 self.state.save()?;
4671 return Ok(());
4672 }
4673
4674 if attempts >= BASE_SYNC_ROUNDS {
4675 let why = format!(
4676 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4677 rebase(s); rebasing again would only race it",
4678 winner.branch
4679 );
4680 self.state.status = RunStatus::Blocked;
4681 self.state.base_sync = Some(BaseSync {
4682 tip,
4683 behind,
4684 attempts,
4685 conflict: Some(why.clone()),
4686 already_in: None,
4687 });
4688 self.state.event("land", why);
4689 self.state.save()?;
4690 return Ok(());
4691 }
4692
4693 self.state.event(
4694 "land",
4695 format!(
4696 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4697 winner.branch
4698 ),
4699 );
4700 self.state.save()?;
4701
4702 let branch_tracking = format!("{remote}/{}", winner.branch);
4708 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4709 let remote_tip = if self.state.task_pr_branch().is_none()
4710 && matches!(&fetched_branch, Ok(o) if o.ok())
4711 {
4712 git::rev_parse(&repo, &branch_tracking).await.ok()
4713 } else {
4714 None
4715 };
4716 if let Some(theirs) = &remote_tip
4720 && !git::is_ancestor(&repo, theirs, &head).await
4721 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4722 .await
4723 .is_ok_and(|missing| missing.is_empty())
4724 {
4725 let why = format!(
4726 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4727 them",
4728 short(theirs),
4729 winner.branch
4730 );
4731 self.state.status = RunStatus::Blocked;
4732 self.state.base_sync = Some(BaseSync {
4733 tip,
4734 behind,
4735 attempts,
4736 conflict: Some(why.clone()),
4737 already_in: None,
4738 });
4739 self.state.event("land", why);
4740 self.state.save()?;
4741 return Ok(());
4742 }
4743
4744 let scratch = self.state.dir().join("base-sync");
4745 let rebased = match crate::rebase::rebase_with_fixer(
4746 &mut self.state,
4747 &scratch,
4748 &winner.branch,
4749 &tracking,
4750 )
4751 .await
4752 {
4753 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4754 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4755 Err(e) => Err(e),
4756 };
4757 let attempts = attempts + 1;
4758 match rebased {
4759 Ok(None) => {
4760 git::sync_to_head(&winner.worktree).await?;
4764 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4765 let mut conflict = None;
4766 if let Some(pinned) = &remote_tip {
4767 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4768 match pushed {
4769 Ok(o) if o.ok() => self.state.event(
4770 "land",
4771 format!("pushed rebased {} to {remote}", winner.branch),
4772 ),
4773 Ok(o) => {
4774 conflict = Some(format!(
4775 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4776 winner.branch,
4777 short(pinned),
4778 o.stderr.chars().take(600).collect::<String>()
4779 ));
4780 }
4781 Err(e) => {
4782 conflict = Some(format!(
4783 "rebased {} locally but could not push it: {e:#}",
4784 winner.branch
4785 ));
4786 }
4787 }
4788 }
4789 if let Some(why) = &conflict {
4790 self.state.status = RunStatus::Blocked;
4791 self.state.event("land", why.clone());
4792 }
4793 self.state.base_sync = Some(BaseSync {
4794 tip: tip.clone(),
4795 behind: 0,
4796 attempts,
4797 conflict,
4798 already_in: None,
4799 });
4800 self.state
4801 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4802 }
4803 Ok(Some(conflict)) => {
4804 let why = format!(
4805 "{} conflicts with {tracking} and did not rebase: {}",
4806 winner.branch,
4807 conflict.chars().take(600).collect::<String>()
4808 );
4809 self.state.status = RunStatus::Blocked;
4810 self.state.base_sync = Some(BaseSync {
4811 tip,
4812 behind,
4813 attempts,
4814 conflict: Some(why.clone()),
4815 already_in: None,
4816 });
4817 self.state.event("land", why);
4818 }
4819 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4823 self.state.save()?;
4824 return Ok(());
4825 }
4826 Err(e) => {
4827 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4828 self.state.status = RunStatus::Blocked;
4829 self.state.base_sync = Some(BaseSync {
4830 tip,
4831 behind,
4832 attempts,
4833 conflict: Some(why.clone()),
4834 already_in: None,
4835 });
4836 self.state.event("land", why);
4837 }
4838 }
4839 self.state.save()?;
4840 Ok(())
4841 }
4842
4843 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4849 if self.state.task_pr_branch().is_some() {
4852 return None;
4853 }
4854 let repo = self.state.repo.clone();
4855 let remote = self.state.config.merge.remote.clone();
4856 let fetched = git::fetch(&repo, &remote, branch).await;
4857 if !matches!(&fetched, Ok(o) if o.ok()) {
4858 return None;
4859 }
4860 let branch_tracking = format!("{remote}/{branch}");
4861 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4862 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4863 return None;
4864 }
4865 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4866 .await
4867 .is_ok_and(|missing| missing.is_empty())
4868 {
4869 return Some(format!(
4870 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4871 them",
4872 short(&theirs)
4873 ));
4874 }
4875 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4876 Ok(o) if o.ok() => {
4877 self.state
4878 .event("land", format!("pushed rebased {branch} to {remote}"));
4879 None
4880 }
4881 Ok(o) => Some(format!(
4882 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4883 someone may have pushed): {}",
4884 short(&theirs),
4885 o.stderr.chars().take(600).collect::<String>()
4886 )),
4887 Err(e) => Some(format!(
4888 "{branch} is rebased locally but could not be pushed: {e:#}"
4889 )),
4890 }
4891 }
4892
4893 async fn settle_already_in(
4910 &mut self,
4911 branch: &str,
4912 tip: &str,
4913 head: &str,
4914 attempts: usize,
4915 behind: usize,
4916 ) -> Result<bool> {
4917 let repo = self.state.repo.clone();
4918 let remote = self.state.config.merge.remote.clone();
4919 let start = self.state.base_commit.clone();
4920 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4921 Ok(Some(e)) => e,
4922 Ok(None) => return Ok(false),
4923 Err(e) => {
4924 tracing::warn!("already-in-base check for {branch}: {e:#}");
4925 return Ok(false);
4926 }
4927 };
4928 let mut verified = vec![head.to_owned()];
4929 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4930 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4931 if matches!(&fetched, Ok(o) if o.ok())
4932 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4933 && theirs != head
4934 {
4935 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4936 Ok(Some(_)) => verified.push(theirs),
4937 _ => return Ok(false),
4938 }
4939 }
4940 let base_branch = self.state.base_branch.clone();
4941 let message = format!(
4942 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4943 land",
4944 evidence.names(),
4945 evidence.proof.as_str()
4946 );
4947 let closed =
4948 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4949 .await;
4950 match closed {
4951 Ok(Ok(url)) => self
4952 .state
4953 .event("land", format!("closed {url}: superseded on {base_branch}")),
4954 Ok(Err(why)) => self
4955 .state
4956 .event("land", format!("did not close a pull request: {why}")),
4957 Err(e) => {
4958 let why = format!(
4959 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4960 not be closed ({e:#}); resume to retry"
4961 );
4962 self.state.status = RunStatus::Blocked;
4963 self.state.base_sync = Some(BaseSync {
4964 tip: tip.to_owned(),
4965 behind,
4966 attempts,
4967 conflict: Some(why.clone()),
4968 already_in: None,
4969 });
4970 self.state.event("land", why);
4971 self.state.save()?;
4972 return Ok(true);
4973 }
4974 }
4975 self.state.status = RunStatus::AlreadyInBase;
4976 self.state.base_sync = Some(BaseSync {
4977 tip: tip.to_owned(),
4978 behind,
4979 attempts,
4980 conflict: None,
4981 already_in: Some(evidence),
4982 });
4983 self.state.event("land", message);
4984 self.state.save()?;
4985 self.settle_questions();
4986 Ok(true)
4987 }
4988
4989 fn landing_base(&self) -> String {
4999 self.state
5000 .base_sync
5001 .as_ref()
5002 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5003 }
5004
5005 pub async fn fix_selected(
5038 &mut self,
5039 ids: &[String],
5040 reason: &str,
5041 allow_stale: bool,
5042 ) -> Result<()> {
5043 let reason = reason.trim();
5044 if reason.is_empty() {
5045 bail!("a fix request needs a reason — that is the operator's own record of why");
5046 }
5047 if ids.is_empty() {
5048 bail!("no finding id given");
5049 }
5050 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5051 bail!(
5052 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5053 has already concluded — can be given a targeted fix. A run still \
5054 in progress should simply be resumed; a `merged` run's branch has \
5055 already landed, so its answer is a fresh `magi review <branch>`, \
5056 not reopening this run's own record",
5057 self.state.id,
5058 self.state.status.as_str()
5059 );
5060 }
5061 let Some(winner) = self.state.winner().cloned() else {
5062 bail!("run {} has no winning candidate to fix", self.state.id);
5063 };
5064 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5065 bail!(
5066 "branch `{}` no longer exists; this run cannot be extended",
5067 winner.branch
5068 );
5069 }
5070 let home = crate::run::home();
5071 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5072 bail!(
5073 "run {} is currently being worked on by another magi process",
5074 self.state.id
5075 );
5076 }
5077 let _claim = FixClaim::acquire(&self.state.dir())?;
5083
5084 let mut seen = BTreeSet::new();
5088 let mut findings = Vec::new();
5089 let mut missing = Vec::new();
5090 for id in ids {
5091 if !seen.insert(id.clone()) {
5092 continue;
5093 }
5094 match self.state.finding(id) {
5095 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5096 id: f.id.clone(),
5097 severity: f.severity,
5098 reviewer_vote: rec.vote,
5099 round: round.round,
5100 round_head: round.head.clone(),
5101 reviewer: rec.reviewer,
5102 agent: rec.agent.clone(),
5103 file: f.file.clone(),
5104 line: f.line,
5105 title: f.title.clone(),
5106 detail: f.detail.clone(),
5107 outcome: OperatorFixOutcome::Pending,
5108 }),
5109 None => missing.push(id.clone()),
5110 }
5111 }
5112 if !missing.is_empty() {
5113 bail!(
5114 "unknown finding id(s): {}; nothing was changed",
5115 missing.join(", ")
5116 );
5117 }
5118
5119 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5120 let stale_details: Vec<(String, String)> = findings
5121 .iter()
5122 .filter(|f| f.round_head != head_at_request)
5123 .map(|f| (f.id.clone(), f.round_head.clone()))
5124 .collect();
5125 let stale = !stale_details.is_empty();
5126 if stale && !allow_stale {
5127 bail!(
5128 "the branch has moved since some finding(s) were raised — {} — now \
5129 at {}; pass --allow-stale to fix anyway, or re-run review first",
5130 stale_details
5131 .iter()
5132 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5133 .collect::<Vec<_>>()
5134 .join(", "),
5135 short(&head_at_request)
5136 );
5137 }
5138
5139 let request = OperatorFixRequest {
5140 requested_at: Timestamp::now(),
5141 reason: reason.to_owned(),
5142 findings,
5143 head_at_request: head_at_request.clone(),
5144 allow_stale,
5145 stale,
5146 fix: None,
5147 result_head: None,
5148 follow_up_review_run: None,
5149 };
5150 self.state.event(
5151 "fix",
5152 format!(
5153 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5154 request.findings.len(),
5155 request
5156 .findings
5157 .iter()
5158 .map(|f| f.id.as_str())
5159 .collect::<Vec<_>>()
5160 .join(", "),
5161 ),
5162 );
5163 self.state.operator_fixes.push(request);
5170 self.state.save()?;
5171 let request_index = self.state.operator_fixes.len() - 1;
5172
5173 let repo = self.state.repo.clone();
5182 let coordination = self.coordinate(&repo).await?;
5183 if winner.worktree.exists() {
5184 let dirty = git::git(
5187 &winner.worktree,
5188 &["status", "--porcelain", "--untracked-files=all"],
5189 )
5190 .await?;
5191 let only_withheld = dirty.lines().all(|l| {
5192 l.strip_prefix("?? ")
5193 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5194 });
5195 if !only_withheld {
5196 bail!(
5197 "`{}` has uncommitted changes; refusing to touch it — commit or \
5198 discard them first",
5199 winner.worktree.display()
5200 );
5201 }
5202 git::worktree_remove(&self.state.repo, &winner.worktree)
5203 .await
5204 .ok();
5205 }
5206 let fix_worktree = self.state.worktree_root().join("operator-fix");
5207 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5208 git::git(
5209 &self.state.repo,
5210 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5211 )
5212 .await
5213 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5214 if !git::is_clean(&fix_worktree).await? {
5215 git::worktree_remove(&self.state.repo, &fix_worktree)
5216 .await
5217 .ok();
5218 bail!(
5219 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5220 winner.branch
5221 );
5222 }
5223 let repo_for_setup = self.state.repo.clone();
5224 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5225 Ok(()) => {
5226 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5227 }
5228 Err(e) => Err(e),
5229 };
5230 if let Err(e) = setup {
5231 git::worktree_remove(&self.state.repo, &fix_worktree)
5232 .await
5233 .ok();
5234 return Err(e);
5235 }
5236
5237 let run_id = self.state.id.clone();
5238 let prompts = self.state.config.prompts.clone();
5239 let language = self.state.config.graph.language.clone();
5240 let sessions = self.state.config.graph.sessions;
5241 let artifacts = agent::artifacts_dir(&self.state.dir());
5242 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5243 .findings
5244 .iter()
5245 .map(|f| Finding {
5246 id: f.id.clone(),
5247 severity: f.severity,
5248 file: f.file.clone(),
5249 line: f.line,
5250 title: f.title.clone(),
5251 detail: f.detail.clone(),
5252 })
5253 .collect();
5254 let fix_prompt = prompt::operator_fix_in(
5255 &self.state.instruction,
5256 &finding_list,
5257 reason,
5258 &stale_details,
5259 &head_at_request,
5260 &language,
5261 &self.state.config.graph.github_language,
5262 );
5263 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5264 drop(coordination);
5265 let (job, seat, out) = self
5266 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5267 prompt: fix_prompt.clone(),
5268 spec,
5269 seat,
5270 cwd: fix_worktree.clone(),
5271 timeout,
5272 allow_write: true,
5273 sessions,
5274 artifacts: artifacts.clone(),
5275 stem: "operator-fix".to_owned(),
5276 handover: None,
5277 })
5278 .await;
5279 let agent_id = seat.agent.clone();
5280
5281 let mut fix = FixRecord {
5282 agent: agent_id,
5283 addressed: Vec::new(),
5284 rejected: Vec::new(),
5285 notes: String::new(),
5286 committed: false,
5287 failed: None,
5288 duration_ms: 0,
5289 continuation: None,
5290 };
5291 let mut final_seat = seat.clone();
5292 match out {
5293 AgentOutcome::Ok(o) => {
5294 fix.duration_ms = o.duration_ms;
5295 let parsed = verdict::extract_json::<FixReport>(&o.text);
5296 let incomplete_reason = match &parsed {
5297 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5298 "the reply parsed, but it reported a command whose own CLI \
5299 never confirmed an exit status"
5300 .to_owned(),
5301 ),
5302 Ok(_) => None,
5303 Err(e) => Some(e.to_string()),
5304 };
5305 match incomplete_reason {
5306 None => {
5307 let report = parsed.expect("checked Ok above");
5308 fix.addressed = report.addressed;
5309 fix.rejected = report.rejected;
5310 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5311 }
5312 Some(reason) => {
5313 let (resumed_seat, resolved, failure, cont) = self
5314 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5315 .await;
5316 fix.duration_ms += cont.cumulative_wait_ms;
5317 fix.continuation = Some(cont);
5318 final_seat = resumed_seat;
5319 match resolved {
5320 Some(report) => {
5321 fix.addressed = report.addressed;
5322 fix.rejected = report.rejected;
5323 fix.notes =
5324 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5325 }
5326 None => fix.failed = failure,
5327 }
5328 }
5329 }
5330 }
5331 AgentOutcome::Dropped(o) => {
5332 fix.duration_ms = o.duration_ms;
5333 let why = o
5334 .dropped
5335 .as_ref()
5336 .map(|d| d.why.as_str())
5337 .unwrap_or("the CLI ended the stream without delivering its answer");
5338 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5339 }
5340 AgentOutcome::Quota(o) => {
5341 self.state.quota.push(QuotaLoss {
5342 seat: final_seat.key.clone(),
5343 node: "fix".to_owned(),
5344 at: Timestamp::now(),
5345 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5346 });
5347 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5348 }
5349 AgentOutcome::Failed(e) => fix.failed = Some(e),
5350 AgentOutcome::ResourceBlocked(e) => {
5351 fix.failed = Some(format!("the fixer was not started: {e}"));
5352 }
5353 }
5354 if fix.continuation.is_none() {
5355 fix.continuation = Some(ContinuationRecord::not_needed());
5356 }
5357 self.state.seats.insert(final_seat.key.clone(), final_seat);
5358
5359 let rescue_message = format!(
5360 "magi: operator-selected fix ({}) (uncommitted work)",
5361 self.state.operator_fixes[request_index]
5362 .findings
5363 .iter()
5364 .map(|f| f.id.as_str())
5365 .collect::<Vec<_>>()
5366 .join(", ")
5367 );
5368 let repo = self.state.repo.clone();
5369 let coordination = self.coordinate(&repo).await?;
5370 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5371 self.state.note_withheld("fix", &r.withheld);
5372 }
5373 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5374 fix.committed = after != head_at_request;
5375 git::worktree_remove(&self.state.repo, &fix_worktree)
5376 .await
5377 .ok();
5378 drop(coordination);
5379
5380 self.state.event(
5381 "fix",
5382 match &fix.failed {
5383 Some(reason) => format!(
5384 "operator fix: adoption report was lost ({reason}); {}",
5385 if fix.committed {
5386 "committed"
5387 } else {
5388 "NO new commit"
5389 }
5390 ),
5391 None => format!(
5392 "operator fix: {} addressed, {} rejected, {}",
5393 fix.addressed.len(),
5394 fix.rejected.len(),
5395 if fix.committed {
5396 "committed"
5397 } else {
5398 "NO new commit"
5399 }
5400 ),
5401 },
5402 );
5403
5404 for f in &mut self.state.operator_fixes[request_index].findings {
5411 f.outcome = if fix.failed.is_some() {
5412 OperatorFixOutcome::Unreported
5413 } else if fix.addressed.contains(&f.id) {
5414 OperatorFixOutcome::Addressed
5415 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5416 OperatorFixOutcome::Rejected { why: r.why.clone() }
5417 } else {
5418 OperatorFixOutcome::Unreported
5419 };
5420 }
5421
5422 let committed = fix.committed;
5423 if committed {
5424 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5425 }
5426 self.state.operator_fixes[request_index].fix = Some(fix);
5427 self.state.save()?;
5430
5431 if committed {
5432 self.state.event(
5433 "fix",
5434 format!(
5435 "operator fix committed {}; opening a follow-up review-only run",
5436 short(&after)
5437 ),
5438 );
5439 let origin =
5442 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5443 match Self::review(
5444 &self.state.repo,
5445 &winner.branch,
5446 self.state.config.clone(),
5447 origin,
5448 )
5449 .await
5450 {
5451 Ok(mut follow_up) => {
5452 follow_up.state.event(
5453 "start",
5454 format!(
5455 "requested by an operator fix on run {} for finding(s) {}",
5456 self.state.id,
5457 self.state.operator_fixes[request_index]
5458 .findings
5459 .iter()
5460 .map(|f| f.id.as_str())
5461 .collect::<Vec<_>>()
5462 .join(", "),
5463 ),
5464 );
5465 follow_up.state.save()?;
5466 let follow_up_id = follow_up.state.id.clone();
5467 let adopted = match crate::direct::adopt(
5470 &follow_up.state,
5471 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5472 ) {
5473 Ok(a) => a,
5474 Err(e) => {
5475 self.state.event(
5478 "fix",
5479 format!(
5480 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5481 ),
5482 );
5483 self.state.save()?;
5484 return Ok(());
5485 }
5486 };
5487 let executed = follow_up.execute().await;
5488 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5489 if let Some(a) = adopted {
5490 a.finish(&follow_up.state, executed);
5491 }
5492 if let Some(e) = failure {
5493 self.state.event(
5494 "fix",
5495 format!(
5496 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5497 ),
5498 );
5499 }
5500 self.state.operator_fixes[request_index].follow_up_review_run =
5501 Some(follow_up_id);
5502 }
5503 Err(e) => {
5504 self.state.event(
5505 "fix",
5506 format!("committed the fix but could not open a follow-up review: {e:#}"),
5507 );
5508 }
5509 }
5510 self.state.save()?;
5511 }
5512
5513 Ok(())
5514 }
5515
5516 async fn ask_fixer(
5536 &mut self,
5537 winner: &Candidate,
5538 node: &'static str,
5539 round: Option<usize>,
5540 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5541 ) -> (SeatJob, SeatState, AgentOutcome) {
5542 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5543 self.ask_fixer_among(attempts, node, round, None, build)
5544 .await
5545 }
5546
5547 async fn ask_fixer_among(
5552 &mut self,
5553 attempts: Vec<(AgentSpec, String)>,
5554 node: &'static str,
5555 round: Option<usize>,
5556 restore: Option<(&Path, &str)>,
5557 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5558 ) -> (SeatJob, SeatState, AgentOutcome) {
5559 let run_id = self.state.id.clone();
5560 let prompts = self.state.config.prompts.clone();
5561 let cache = self.state.config.cache_dir();
5562 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5563 let mut last = None;
5564 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5565 let seat = self.seat(&key, &spec.id);
5566 let mut job = build(spec.clone(), seat);
5567 if i > 0 {
5568 job.stem = format!("{}-{}", job.stem, spec.id);
5569 }
5570 let ctx = WaveCtx {
5571 carry_seats: false,
5572 pauses: Some((&self.pause, &self.interrupt)),
5573 run: &run_id,
5574 node,
5575 prompts: &prompts,
5576 cache: cache.as_deref(),
5577 round,
5578 };
5579 let (seat, out) =
5580 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5581 let advances = match &out {
5584 AgentOutcome::Ok(o) => agent::output_advances(o),
5585 AgentOutcome::ResourceBlocked(_) => false,
5586 _ => true,
5587 };
5588 if let Some(next) = ids.get(i + 1)
5589 && advances
5590 {
5591 self.state.seats.insert(seat.key.clone(), seat.clone());
5592 let class =
5593 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5594 let reason = match &out {
5595 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5596 other => fail_reason(other),
5597 };
5598 if let Some((wt, rev)) = restore
5602 && let Err(e) = self.graft_restore(wt, rev).await
5603 {
5604 let why =
5605 format!("could not restore the worktree before the next fixer: {e:#}");
5606 tracing::warn!("{why}");
5607 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5608 AgentOutcome::ResourceBlocked(why)
5609 } else {
5610 AgentOutcome::Failed(why)
5611 };
5612 last = Some((job, seat, out));
5613 break;
5614 }
5615 record_handover(
5616 &mut self.state,
5617 node,
5618 &seat.key,
5619 &spec.id,
5620 next,
5621 &class,
5622 &reason,
5623 );
5624 continue;
5625 }
5626 last = Some((job, seat, out));
5627 break;
5628 }
5629 last.expect("the fixer chain always has an entry")
5630 }
5631
5632 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5643 use crate::recovery::{Mode, Repair};
5644 if !self.state.reviews.is_empty() {
5645 return Ok(());
5646 }
5647 let Some(mut rec) = self.state.recovery.clone() else {
5648 return Ok(());
5649 };
5650 if rec.mode != Mode::VerifyRepair {
5651 return Ok(());
5652 }
5653 let wt = winner.worktree.clone();
5654 let mut head = git::rev_parse(&wt, "HEAD").await?;
5655 match rec.repair.clone() {
5656 Repair::Done { .. } => return Ok(()),
5657 Repair::WaitingForRestore { before } => {
5658 self.graft_restore(&wt, &before).await?;
5659 head = before;
5660 rec.repair = Repair::NotStarted;
5661 self.state.recovery = Some(rec.clone());
5662 self.state.save()?;
5663 }
5664 Repair::Started { before } => {
5665 if let Err(e) = self.graft_restore(&wt, &before).await {
5669 if e.is::<crate::repo_lock::ResourceBlocked>() {
5670 return Err(e);
5671 }
5672 return self
5673 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5674 .await;
5675 }
5676 rec.repair = Repair::Done {
5677 before: before.clone(),
5678 after: before,
5679 committed: false,
5680 failed: Some("interrupted; rolled back".to_owned()),
5681 };
5682 self.state
5683 .event("recovery", "the repair was interrupted and rolled back");
5684 self.state.recovery = Some(rec);
5685 self.state.save()?;
5686 return Ok(());
5687 }
5688 Repair::NotStarted => {}
5689 }
5690 if crate::recovery::recheck(&mut rec, &head) {
5691 self.state.event("recovery", rec.summary());
5692 self.state.recovery = Some(rec);
5693 self.state.save()?;
5694 return Ok(());
5695 }
5696 rec.repair = Repair::Started {
5697 before: head.clone(),
5698 };
5699 self.state.recovery = Some(rec.clone());
5700 self.state.save()?;
5701
5702 let language = self.state.config.graph.language.clone();
5703 let sessions = self.state.config.graph.sessions;
5704 let artifacts = agent::artifacts_dir(&self.state.dir());
5705 let fix_prompt = prompt::recovery_fix(
5706 &self.state.instruction,
5707 &rec,
5708 self.state.config.graph.review_rounds,
5709 &language,
5710 );
5711 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5712 self.state.event(
5713 "recovery",
5714 format!(
5715 "repair: the fixer starts from the failed verification on {}",
5716 short(&head)
5717 ),
5718 );
5719 let (_job, seat, out) = self
5720 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5721 prompt: fix_prompt.clone(),
5722 spec,
5723 seat,
5724 cwd: wt.clone(),
5725 timeout,
5726 allow_write: true,
5727 sessions,
5728 artifacts: artifacts.clone(),
5729 stem: "fix-recovery".to_owned(),
5730 handover: None,
5731 })
5732 .await;
5733 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5734 rec.repair = Repair::WaitingForRestore { before: head };
5735 self.state.recovery = Some(rec);
5736 return self.stop_for_resource();
5737 }
5738 let failed = match &out {
5739 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5740 AgentOutcome::Quota(o) => {
5741 self.state.quota.push(QuotaLoss {
5742 seat: seat.key.clone(),
5743 node: "fix".to_owned(),
5744 at: Timestamp::now(),
5745 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5746 });
5747 Some("rate limited (quota); the fixer could not run".to_owned())
5748 }
5749 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5750 AgentOutcome::Failed(e) => Some(e.clone()),
5751 };
5752 self.state.seats.insert(seat.key.clone(), seat);
5753 let coordination = self.coordinate(&wt).await?;
5754 let failed = match failed {
5755 Some(why) => {
5757 if let Err(e) = git::restore_to(&wt, &head).await {
5758 return self
5759 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5760 .await;
5761 }
5762 Some(why)
5763 }
5764 None => {
5765 if let Ok(r) = git::rescue_commit(
5766 &wt,
5767 "magi: recovery repair of the failed verification (uncommitted work)",
5768 )
5769 .await
5770 {
5771 self.state.note_withheld("fix", &r.withheld);
5772 }
5773 None
5774 }
5775 };
5776 let after = git::rev_parse(&wt, "HEAD").await?;
5777 drop(coordination);
5778 self.state.event(
5779 "recovery",
5780 match &failed {
5781 Some(why) => format!("repair: nothing kept ({why})"),
5782 None if after != head => {
5783 format!("repair: committed {}", short(&after))
5784 }
5785 None => "repair: the fixer made no commit".to_owned(),
5786 },
5787 );
5788 rec.repair = Repair::Done {
5789 before: head.clone(),
5790 committed: after != head,
5791 after,
5792 failed,
5793 };
5794 self.state.recovery = Some(rec);
5795 self.state.save()?;
5796 Ok(())
5797 }
5798
5799 async fn stop_recovery(
5802 &mut self,
5803 mut rec: crate::recovery::RecoveryRecord,
5804 head: &str,
5805 why: String,
5806 ) -> Result<()> {
5807 rec.repair = crate::recovery::Repair::Started {
5809 before: head.to_owned(),
5810 };
5811 self.state.recovery = Some(rec);
5812 self.state
5813 .event("recovery", format!("repair could not be undone: {why}"));
5814 self.state.status = RunStatus::Blocked;
5815 self.state.save()?;
5816 bail!("recovery repair could not be rolled back: {why}")
5817 }
5818
5819 async fn review_loop(&mut self) -> Result<()> {
5820 if self
5825 .state
5826 .base_sync
5827 .as_ref()
5828 .is_some_and(|s| s.conflict.is_some())
5829 {
5830 return Ok(());
5831 }
5832 let run_id = self.state.id.clone();
5837 let prompts = self.state.config.prompts.clone();
5838 let Some(winner) = self.state.winner().cloned() else {
5839 return Ok(());
5840 };
5841 let max_rounds = self.state.config.graph.review_rounds;
5842 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5852 if status == RunStatus::Gating {
5855 self.record_contested_handoff();
5856 }
5857 self.state.status = status;
5858 self.state.save()?;
5859 return Ok(());
5860 }
5861 self.state.status = RunStatus::Reviewing;
5862 if self
5872 .state
5873 .reviews
5874 .last()
5875 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5876 {
5877 let shell = self.state.config.shell();
5878 return self
5879 .stop_reviewing(
5880 "the last round's own verification never resolved",
5881 &shell,
5882 &winner.worktree,
5883 )
5884 .await;
5885 }
5886
5887 self.recovery_repair(&winner).await?;
5888 if self.resource_halt()? {
5889 return Ok(());
5890 }
5891
5892 let repo = self.state.repo.clone();
5893 let root = self.state.worktree_root();
5894 let language = self.state.config.graph.language.clone();
5895 let sessions = self.state.config.graph.sessions;
5896 let artifacts = self.merge_repair_artifacts();
5897 let base = self.landing_base();
5898 let base_short = short(&base);
5899 let reviewers = self.roles.reviewers.clone();
5900 let shell = self.state.config.shell();
5901
5902 for round in (self.state.reviews.len() + 1)..=max_rounds {
5903 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5904 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5905 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5906 let prev_verification = self
5915 .state
5916 .reviews
5917 .last()
5918 .and_then(|r| r.verification_summary(&head));
5919 let prior_section = match (&self.state.recovery, round) {
5922 (Some(rec), 1) if self.state.reviews.is_empty() => {
5923 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5924 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5925 Some(crate::recovery::head_of(
5926 &git::diff(&winner.worktree, reviewed, "HEAD")
5927 .await
5928 .unwrap_or_default(),
5929 crate::recovery::SINCE_DIFF_MAX,
5930 ))
5931 }
5932 _ => None,
5933 };
5934 Some(prompt::recovery_review(rec, since.as_deref()))
5935 }
5936 _ => None,
5937 };
5938
5939 let mut jobs = Vec::new();
5943 let coordination = self.coordinate(&repo).await?;
5944 for (r, spec) in reviewers.iter().cloned().enumerate() {
5945 let wt = root.join(format!("review-{}", r + 1));
5946 if wt.exists() {
5949 git::reset_detached(&wt, &head).await?;
5950 } else {
5951 git::worktree_add_detached(&repo, &wt, &head).await?;
5952 }
5953 self.setup_seat_worktree(&repo, &wt, false).await?;
5954 let seat_key = format!("review-{}", r + 1);
5955 let spec = pick_start_spec(
5958 &self.roles.reviewer_roster,
5959 spec,
5960 self.state.seat_history.get(&seat_key),
5961 );
5962 let seat = self.seat(&seat_key, &spec.id);
5963 jobs.push(SeatJob {
5964 prompt: prompt::review(&prompt::ReviewCtx {
5965 instruction: &self.state.instruction,
5966 branch: &winner.branch,
5967 base_short: &base_short,
5968 stat: &stat,
5969 patch: &patch,
5970 verification: prev_verification.as_ref(),
5971 prior: prior_section.as_deref(),
5972 reviewers: reviewers.len(),
5973 round,
5974 rounds: max_rounds,
5975 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5978 lens: Lens::for_seat(r),
5979 language: &language,
5980 github_language: &self.state.config.graph.github_language,
5981 }),
5982 spec,
5983 seat,
5984 cwd: wt,
5985 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5986 allow_write: false,
5987 sessions,
5988 artifacts: artifacts.clone(),
5989 stem: format!("review-{round}-{}", r + 1),
5990 handover: None,
5991 });
5992 }
5993
5994 drop(coordination);
5995 self.state.event(
5996 "review",
5997 format!(
5998 "round {round}: {} reviewers on {}",
5999 jobs.len(),
6000 short(&head)
6001 ),
6002 );
6003 let mut quota_losses = Vec::new();
6004 let review_retries = self.state.config.graph.retries;
6005 let review_cache = self.state.config.cache_dir();
6006 let ctx = WaveCtx {
6007 carry_seats: true,
6008 pauses: Some((&self.pause, &self.interrupt)),
6009 run: &run_id,
6010 node: "review",
6011 prompts: &prompts,
6012 cache: review_cache.as_deref(),
6013 round: Some(round),
6014 };
6015 let results = ask_json_wave::<Review>(
6016 jobs,
6017 Arc::clone(&self.sem),
6018 review_retries,
6019 &self.roles.reviewer_roster,
6020 &ctx,
6021 &mut quota_losses,
6022 &mut self.state,
6023 &|_: &Review| Ok(()),
6024 )
6025 .await;
6026 if self.state.resource_block.is_some() {
6027 return self.stop_for_resource();
6028 }
6029 let round_quota_missing = quota_losses.len();
6033 self.state.quota.extend(quota_losses);
6034
6035 let mut records = Vec::new();
6036 let mut all_findings = Vec::new();
6037 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6038 let agent_id = seat.agent.clone();
6039 self.state.seats.insert(seat.key.clone(), seat);
6040 let mut record = ReviewRecord {
6041 reviewer: r + 1,
6042 agent: agent_id,
6043 summary: String::new(),
6044 findings: Vec::new(),
6045 vote: None,
6046 failed: None,
6047 duration_ms: 0,
6048 attempts,
6054 };
6055 match res {
6056 Ok((review, out)) => {
6057 record.summary =
6065 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6066 record.vote = Some(review.vote);
6067 record.duration_ms = out.duration_ms;
6068 for (n, mut f) in review.findings.into_iter().enumerate() {
6069 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6072 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6073 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6074 f.file = f
6080 .file
6081 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6082 all_findings.push(f.clone());
6083 record.findings.push(f);
6084 }
6085 self.state.event(
6086 "review",
6087 format!(
6088 "round {round}: reviewer {} voted {} with {} finding(s)",
6089 r + 1,
6090 review.vote.label(),
6091 record.findings.len()
6092 ),
6093 );
6094 }
6095 Err(e) => {
6096 record.failed = Some(e.to_string());
6097 self.state.event(
6098 "review",
6099 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6100 );
6101 }
6102 }
6103 records.push(record);
6104 }
6105
6106 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6113 let vote_split =
6114 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6115 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6116 if vote_split {
6117 self.state.event(
6118 "review",
6119 format!(
6120 "round {round}: votes split ({}) — one round of reconsideration",
6121 initial_votes
6122 .iter()
6123 .map(|v| v.label())
6124 .collect::<Vec<_>>()
6125 .join(", ")
6126 ),
6127 );
6128 let panel: Vec<ReviewSeatReport<'_>> = records
6131 .iter()
6132 .filter_map(|r| {
6133 r.vote.map(|vote| ReviewSeatReport {
6134 reviewer: r.reviewer,
6135 vote,
6136 summary: &r.summary,
6137 findings: &r.findings,
6138 })
6139 })
6140 .collect();
6141
6142 let mut jobs = Vec::new();
6143 let mut seats_at = Vec::new();
6144 for (r, spec) in reviewers.iter().cloned().enumerate() {
6145 if records[r].vote.is_none() {
6149 continue;
6150 }
6151 let wt = root.join(format!("review-{}", r + 1));
6152 let seat_key = format!("review-{}", r + 1);
6153 let spec = self.occupant(&seat_key, spec);
6154 let seat = self.seat(&seat_key, &spec.id);
6155 let build = |with_patch: bool| {
6163 prompt::review_reconsider(&ReviewReconsiderCtx {
6164 instruction: &self.state.instruction,
6165 reviewer: r + 1,
6166 lens: Lens::for_seat(r),
6167 panel: &panel,
6168 patch: with_patch.then_some(ReviewPatch {
6169 branch: &winner.branch,
6170 base_short: &base_short,
6171 stat: &stat,
6172 patch: &patch,
6173 }),
6174 round,
6175 rounds: max_rounds,
6176 language: &language,
6177 })
6178 };
6179 let full = build(true);
6180 let prompt = if has_context(&spec, &seat, sessions) {
6181 build(false)
6182 } else {
6183 full.clone()
6184 };
6185 jobs.push(SeatJob {
6186 prompt,
6187 spec,
6188 seat,
6189 cwd: wt,
6190 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6191 allow_write: false,
6192 sessions,
6193 artifacts: artifacts.clone(),
6194 stem: format!("review-{round}-reconsider-{}", r + 1),
6195 handover: Some(full),
6196 });
6197 seats_at.push(r);
6198 }
6199
6200 let mut recon_quota_losses = Vec::new();
6201 let recon_cache = self.state.config.cache_dir();
6202 let recon_ctx = WaveCtx {
6203 carry_seats: true,
6204 pauses: Some((&self.pause, &self.interrupt)),
6205 run: &run_id,
6206 node: "review",
6207 prompts: &prompts,
6208 cache: recon_cache.as_deref(),
6209 round: Some(round),
6210 };
6211 let recon_results = ask_json_wave::<ReviewRevote>(
6212 jobs,
6213 Arc::clone(&self.sem),
6214 review_retries,
6215 &self.roles.reviewer_roster,
6216 &recon_ctx,
6217 &mut recon_quota_losses,
6218 &mut self.state,
6219 &|_: &ReviewRevote| Ok(()),
6220 )
6221 .await;
6222 if self.state.resource_block.is_some() {
6223 return self.stop_for_resource();
6224 }
6225 self.state.quota.extend(recon_quota_losses);
6226
6227 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6228 let agent_id = seat.agent.clone();
6229 self.state.seats.insert(seat.key.clone(), seat);
6230 let mut rec = ReviewRevoteRecord {
6231 reviewer: r + 1,
6232 agent: agent_id,
6233 vote: None,
6234 reason: String::new(),
6235 failed: None,
6236 };
6237 match res {
6238 Ok((rv, _)) => {
6239 rec.vote = Some(rv.vote);
6240 rec.reason =
6241 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6242 self.state.event(
6243 "review",
6244 format!(
6245 "round {round}: reviewer {} revoted {}",
6246 r + 1,
6247 rv.vote.label()
6248 ),
6249 );
6250 }
6251 Err(e) => {
6252 rec.failed = Some(e.to_string());
6253 self.state.event(
6254 "review",
6255 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6256 );
6257 }
6258 }
6259 reconsideration.push(rec);
6260 }
6261 } else if initial_votes.len() > 1 {
6262 self.state.event(
6263 "review",
6264 format!(
6265 "round {round}: votes agreed ({}) — no reconsideration",
6266 initial_votes[0].label()
6267 ),
6268 );
6269 }
6270
6271 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6272 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6273 let defer_e2e =
6284 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6285 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6286 let reason =
6287 format!("{blocking} blocking finding(s) already required a fix this round");
6288 self.state.event(
6289 "verify",
6290 format!(
6291 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6292 {}); it will run once a round has none left",
6293 short(&head)
6294 ),
6295 );
6296 (Vec::new(), false, true, Some(reason))
6297 } else {
6298 let e2e_commands = self.state.config.verify.e2e.clone();
6299 let cache_dir = self.state.config.cache_dir();
6300 let context = format!("round {round}");
6301 let wait = self.repository_wait();
6302 let (e2e, verify_retried) = with_cache_lease(
6303 &mut self.state,
6304 cache_dir.as_deref(),
6305 "e2e",
6306 "e2e",
6307 &winner.worktree,
6308 &head,
6309 verify_timeout,
6310 &context,
6311 &wait,
6312 |state, budget| {
6313 let shell = shell.clone();
6314 let e2e_commands = e2e_commands.clone();
6315 let worktree = winner.worktree.clone();
6316 let context = context.clone();
6317 async move {
6318 run_e2e_with_retry(
6319 state,
6320 &shell,
6321 &e2e_commands,
6322 &worktree,
6323 budget,
6324 &context,
6325 )
6326 .await
6327 }
6328 },
6329 )
6330 .await?;
6331 (e2e, verify_retried, false, None)
6332 };
6333
6334 let expected = records.len();
6335 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6336 let incomplete = answered < expected;
6337 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6338 let policy = self.state.config.graph.incomplete_review;
6339 let clean = round_is_clean(
6340 blocking,
6341 e2e_ok,
6342 answered,
6343 expected,
6344 round_quota_missing,
6345 policy,
6346 );
6347
6348 let mut round_record = ReviewRound {
6349 round,
6350 head: head.clone(),
6351 verified_head: None,
6352 verified_at: None,
6353 reviews: records,
6354 e2e,
6355 verify_retried,
6356 e2e_deferred,
6357 e2e_defer_reason,
6358 fix: None,
6359 blocking,
6360 answered,
6361 expected,
6362 clean,
6363 progressed: false,
6364 vote_split,
6365 reconsideration,
6366 verdict: None,
6367 };
6368 round_record.verdict = ReviewVote::worst(
6372 round_record
6373 .final_votes()
6374 .into_iter()
6375 .map(|(_, _, vote)| vote),
6376 );
6377 if !matches!(
6388 round_record.e2e_status(),
6389 E2eStatus::Deferred | E2eStatus::NotConfigured
6390 ) {
6391 round_record.verified_head = Some(head.clone());
6392 round_record.verified_at = Some(Timestamp::now());
6393 }
6394 let this_round_verification = round_record.verification_summary(&head);
6395
6396 if incomplete {
6397 let missing: Vec<String> = round_record
6398 .reviews
6399 .iter()
6400 .filter(|r| r.failed.is_some())
6401 .map(|r| format!("review-{}", r.reviewer))
6402 .collect();
6403 self.state.event(
6404 "review",
6405 format!(
6406 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6407 missing.join(", ")
6408 ),
6409 );
6410 }
6411
6412 if clean {
6413 self.state.event(
6414 "review",
6415 if incomplete && policy == IncompleteReviewPolicy::Warn {
6416 format!(
6417 "round {round}: clean (warn policy, incomplete panel) — no \
6418 blocking findings from the seats that answered, verification green"
6419 )
6420 } else if incomplete {
6421 format!(
6422 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6423 quorum) — no blocking findings from the seats that answered, \
6424 verification green",
6425 expected - answered
6426 )
6427 } else {
6428 format!("round {round}: clean — no blocking findings, verification green")
6429 },
6430 );
6431 self.state.reviews.push(round_record);
6432 self.state.status = RunStatus::Gating;
6433 self.state.save()?;
6434 return Ok(());
6435 }
6436
6437 if incomplete && blocking == 0 && e2e_ok {
6445 self.state.reviews.push(round_record);
6446 self.state.save()?;
6447 if round == max_rounds {
6448 self.state.status = RunStatus::Blocked;
6449 self.state.event(
6450 "review",
6451 format!(
6452 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6453 refusing to call it clean",
6454 expected - answered
6455 ),
6456 );
6457 return Ok(());
6458 }
6459 continue;
6460 }
6461
6462 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6474 self.state.reviews.push(round_record);
6475 return self
6476 .stop_reviewing(
6477 "the round's own verification could not run",
6478 &shell,
6479 &winner.worktree,
6480 )
6481 .await;
6482 }
6483
6484 if round == max_rounds {
6485 self.state.reviews.push(round_record);
6486 return self
6487 .stop_reviewing(
6488 &format!(
6489 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6490 ),
6491 &shell,
6492 &winner.worktree,
6493 )
6494 .await;
6495 }
6496
6497 let blocking_findings: Vec<_> = all_findings
6500 .iter()
6501 .filter(|f| f.severity.blocks())
6502 .cloned()
6503 .collect();
6504 let fix_prompt = prompt::fix_in(
6505 &self.state.instruction,
6506 &blocking_findings,
6507 this_round_verification.as_ref(),
6508 round,
6509 max_rounds,
6510 &language,
6511 &self.state.config.graph.github_language,
6512 );
6513 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6514 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6515 let (job, seat, out) = self
6516 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6517 prompt: fix_prompt.clone(),
6518 spec,
6519 seat,
6520 cwd: winner.worktree.clone(),
6521 timeout,
6522 allow_write: true,
6523 sessions,
6524 artifacts: artifacts.clone(),
6525 stem: format!("fix-{round}"),
6526 handover: None,
6527 })
6528 .await;
6529 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6532 return self.stop_for_resource();
6533 }
6534 let agent_id = seat.agent.clone();
6535
6536 let mut fix = FixRecord {
6537 agent: agent_id,
6538 addressed: Vec::new(),
6539 rejected: Vec::new(),
6540 notes: String::new(),
6541 committed: false,
6542 failed: None,
6543 duration_ms: 0,
6544 continuation: None,
6545 };
6546 let mut continuation = ContinuationRecord::not_needed();
6547 let mut final_seat = seat.clone();
6548 match out {
6549 AgentOutcome::Ok(o) => {
6550 fix.duration_ms = o.duration_ms;
6551 let parsed = verdict::extract_json::<FixReport>(&o.text);
6552 let incomplete_reason = match &parsed {
6559 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6560 "the reply parsed, but it reported a command whose own CLI never \
6561 confirmed an exit status"
6562 .to_owned(),
6563 ),
6564 Ok(_) => None,
6565 Err(e) => Some(e.to_string()),
6566 };
6567 match incomplete_reason {
6568 None => {
6569 let report = parsed.expect("checked Ok above");
6570 fix.addressed = report.addressed;
6571 fix.rejected = report.rejected;
6572 fix.notes =
6573 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6574 }
6575 Some(reason) => {
6576 let (resumed_seat, resolved, failure, cont) = self
6577 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6578 .await;
6579 fix.duration_ms += cont.cumulative_wait_ms;
6580 continuation = cont;
6581 final_seat = resumed_seat;
6582 match resolved {
6583 Some(report) => {
6584 fix.addressed = report.addressed;
6585 fix.rejected = report.rejected;
6586 fix.notes = blind::sanitize_prose(
6587 &report.notes,
6588 &self.state.config.blind,
6589 );
6590 }
6591 None => fix.failed = failure,
6592 }
6593 }
6594 }
6595 }
6596 AgentOutcome::Dropped(o) => {
6598 fix.duration_ms = o.duration_ms;
6599 let why = o
6600 .dropped
6601 .as_ref()
6602 .map(|d| d.why.as_str())
6603 .unwrap_or("the CLI ended the stream without delivering its answer");
6604 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6605 }
6606 AgentOutcome::Quota(o) => {
6607 self.state.quota.push(QuotaLoss {
6608 seat: final_seat.key.clone(),
6609 node: "fix".to_owned(),
6610 at: Timestamp::now(),
6611 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6612 });
6613 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6614 }
6615 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6616 }
6617 fix.continuation = Some(continuation);
6618 self.state.seats.insert(final_seat.key.clone(), final_seat);
6619 let coordination = self.coordinate(&winner.worktree).await?;
6620 if let Ok(r) = git::rescue_commit(
6621 &winner.worktree,
6622 &format!("magi: review round {round} fixes (uncommitted work)"),
6623 )
6624 .await
6625 {
6626 self.state.note_withheld("fix", &r.withheld);
6627 }
6628 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6629 fix.committed = after != before;
6630 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6638 drop(coordination);
6639 let progressed = diff_after != patch;
6640 let commit_note = if fix.committed {
6641 "committed"
6642 } else {
6643 "NO new commit"
6644 };
6645 let tree_note = if progressed {
6646 "changed vs base"
6647 } else {
6648 "unchanged vs base"
6649 };
6650 self.state.event(
6651 "fix",
6652 match &fix.failed {
6653 Some(reason) => {
6659 format!(
6660 "round {round}: fixer's adoption report was lost ({reason}); \
6661 {commit_note}, tree {tree_note}"
6662 )
6663 }
6664 None => format!(
6665 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6666 {tree_note}{}",
6667 fix.addressed.len(),
6668 fix.rejected.len(),
6669 if continuation.outcome == ContinuationOutcome::Resumed {
6670 format!(
6671 " (adoption report recovered after {} continuation(s))",
6672 continuation.attempts
6673 )
6674 } else {
6675 String::new()
6676 },
6677 ),
6678 },
6679 );
6680 round_record.fix = Some(fix);
6681 round_record.progressed = progressed;
6682 self.state.reviews.push(round_record);
6683 self.state.save()?;
6684
6685 if matches!(
6698 continuation.outcome,
6699 ContinuationOutcome::Exhausted
6700 | ContinuationOutcome::QuotaLost
6701 | ContinuationOutcome::NoSession
6702 ) {
6703 return self
6704 .stop_reviewing(
6705 "the fixer's adoption report never came back, even after resuming its \
6706 own seat; refusing to start another round against the same worktree \
6707 while that is unresolved",
6708 &shell,
6709 &winner.worktree,
6710 )
6711 .await;
6712 }
6713
6714 let streak = self
6715 .state
6716 .reviews
6717 .iter()
6718 .rev()
6719 .take_while(|r| !r.progressed)
6720 .count();
6721 if streak >= STAGNANT_LIMIT {
6722 return self
6723 .stop_reviewing(
6724 &format!(
6725 "the tree has not moved against base for {streak} round(s) in a row"
6726 ),
6727 &shell,
6728 &winner.worktree,
6729 )
6730 .await;
6731 }
6732 }
6733 Ok(())
6734 }
6735
6736 fn record_contested_handoff(&mut self) {
6770 if self.state.contested_handoff.is_some() {
6771 return;
6772 }
6773 let Some(contested) = self
6774 .state
6775 .reviews
6776 .last()
6777 .and_then(ReviewRound::contested_handoff)
6778 else {
6779 return;
6780 };
6781 self.state.event(
6782 "review",
6783 format!(
6784 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6785 wait for the owner's approval",
6786 contested.findings.len(),
6787 contested.rejecters.len()
6788 ),
6789 );
6790 self.state.contested_handoff = Some(contested);
6791 }
6792
6793 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6794 let round_idx = self.state.reviews.len() - 1;
6795 let needs_catchup_run = matches!(
6803 self.state.reviews[round_idx].e2e_status(),
6804 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6805 );
6806 if needs_catchup_run {
6807 let round = self.state.reviews[round_idx].round;
6808 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6809 let commands = self.state.config.verify.e2e.clone();
6810 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6811 let cache_dir = self.state.config.cache_dir();
6812 let context = format!(
6813 "round {round}: verification unresolved, catching up before the final decision"
6814 );
6815 let wait = self.repository_wait();
6816 let (outcomes, verify_retried) = with_cache_lease(
6817 &mut self.state,
6818 cache_dir.as_deref(),
6819 "e2e",
6820 "e2e",
6821 worktree,
6822 &attempted_head,
6823 timeout,
6824 &context,
6825 &wait,
6826 |state, budget| {
6827 let shell = shell.to_vec();
6828 let commands = commands.clone();
6829 let context = context.clone();
6830 async move {
6831 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6832 .await
6833 }
6834 },
6835 )
6836 .await?;
6837 let last = &mut self.state.reviews[round_idx];
6838 last.e2e = outcomes;
6839 last.verify_retried = verify_retried;
6840 last.verified_head = Some(attempted_head);
6847 last.verified_at = Some(Timestamp::now());
6848 if verify_inconclusive(&last.e2e) {
6849 self.state.save()?;
6856 return Ok(());
6857 }
6858 last.e2e_deferred = false;
6859 }
6860 let last = &self.state.reviews[round_idx];
6861 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6862
6863 match last.e2e_status() {
6864 E2eStatus::Failed => {
6865 let red: Vec<String> = last
6866 .e2e
6867 .iter()
6868 .filter(|o| !o.ok())
6869 .map(|o| {
6870 format!(
6871 "`{}` -> {:?}\n{}",
6872 o.command,
6873 o.code,
6874 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6875 )
6876 })
6877 .collect();
6878 self.state
6879 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6880 self.state.status = RunStatus::Blocked;
6881 }
6882 E2eStatus::ResourceBlocked => {
6887 self.state.event(
6888 "review",
6889 format!(
6890 "{why}; e2e could not run (shared build cache unavailable); not \
6891 deciding yet"
6892 ),
6893 );
6894 }
6895 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6896 self.state.event(
6897 "review",
6898 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6899 );
6900 self.record_contested_handoff();
6901 self.state.status = RunStatus::Gating;
6902 }
6903 }
6904 self.state.save()?;
6905 Ok(())
6906 }
6907
6908 async fn gate(&mut self) -> Result<()> {
6911 if self.state.status == RunStatus::Failed
6923 || self
6924 .state
6925 .base_sync
6926 .as_ref()
6927 .is_some_and(|s| s.conflict.is_some())
6928 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6929 != Some(RunStatus::Gating)
6930 {
6931 return Ok(());
6932 }
6933 if self.state.gate_ran {
6934 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6945 self.state.status = RunStatus::Blocked;
6946 self.state.save()?;
6947 }
6948 return Ok(());
6949 }
6950 let Some(winner) = self.state.winner().cloned() else {
6951 return Ok(());
6952 };
6953 self.state.status = RunStatus::Gating;
6954 let mut outcomes = self.run_gate(&winner).await?;
6955 loop {
6956 if verify_inconclusive(&outcomes) {
6967 self.state.save()?;
6968 return Ok(());
6969 }
6970 if outcomes.iter().all(CommandOutcome::ok) {
6971 break;
6972 }
6973 match self.gate_fix_round(&winner, &outcomes).await? {
6974 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6975 GateFix::Stop => break,
6976 GateFix::Defer => {
6977 self.state.save()?;
6978 return Ok(());
6979 }
6980 }
6981 }
6982 let passed = outcomes.iter().all(CommandOutcome::ok);
6983 self.state.gate = outcomes;
6984 self.state.gate_ran = true;
6985 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6986 if !passed {
6987 self.state.status = RunStatus::Blocked;
6988 let spent = self.state.gate_fixes.len();
6989 self.state.event(
6990 "gate",
6991 if spent == 0 {
6992 "gate failed; not merging".to_owned()
6993 } else {
6994 format!("gate failed after {spent} gate-fix round(s); not merging")
6995 },
6996 );
6997 }
6998 self.state.save()?;
6999 Ok(())
7000 }
7001
7002 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7012 let commands = state.config.verify.pre_gate.clone();
7013 if commands.is_empty() {
7014 return;
7015 }
7016 let shell = state.config.shell();
7017 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7018 let (outcomes, _) = run_commands(
7019 state,
7020 "pre_gate",
7021 "pre_gate",
7022 0,
7023 &shell,
7024 &commands,
7025 &winner.worktree,
7026 timeout,
7027 )
7028 .await;
7029 for o in &outcomes {
7030 if !o.ok() {
7031 tracing::warn!(
7032 "pre_gate `{}` failed ({:?}); the gate decides",
7033 o.command,
7034 o.code
7035 );
7036 }
7037 state.event(
7038 "pre_gate",
7039 format!(
7040 "`{}` -> {}",
7041 o.command,
7042 if o.ok() {
7043 "pass".to_owned()
7044 } else {
7045 format!(
7046 "FAIL ({:?})\n{}",
7047 o.code,
7048 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7049 )
7050 }
7051 ),
7052 );
7053 }
7054 state.pre_gate = outcomes;
7055 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7056 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7057 Ok(head) => {
7058 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7059 state.pre_gate_commit = Some(head);
7060 }
7061 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7062 },
7063 Ok(false) => {}
7064 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7065 }
7066 if let Err(e) = state.save() {
7067 tracing::warn!("could not persist the pre_gate record: {e:#}");
7068 }
7069 }
7070
7071 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7074 let shell = self.state.config.shell();
7075 let gate_commands = self.state.config.verify.gate.clone();
7076 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7077 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7078 Vec::new()
7079 } else {
7080 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7081 let cache_dir = self.state.config.cache_dir();
7082 let wait = self.repository_wait();
7083 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7084 let (outcomes, _) = with_cache_lease(
7085 &mut self.state,
7086 cache_dir.as_deref(),
7087 "gate",
7088 "gate",
7089 &winner.worktree,
7090 &head,
7091 timeout,
7092 "final gate",
7093 &wait,
7094 |state, budget| {
7095 let shell = shell.clone();
7096 let gate_commands = gate_commands.clone();
7097 let winner = winner.clone();
7098 async move {
7099 Self::run_pre_gate(state, &winner).await;
7100 let (outcomes, timed_out_pids) = run_commands(
7101 state,
7102 "gate",
7103 "gate",
7104 0,
7105 &shell,
7106 &gate_commands,
7107 &winner.worktree,
7108 budget,
7109 )
7110 .await;
7111 (outcomes, false, timed_out_pids)
7112 }
7113 },
7114 )
7115 .await?;
7116 outcomes
7117 };
7118 if outcomes.is_empty() {
7119 self.state
7124 .event("gate", "no gate commands configured; not tested");
7125 }
7126 for o in &outcomes {
7127 self.state.event(
7128 "gate",
7129 format!(
7130 "`{}` -> {}",
7131 o.command,
7132 if o.ok() {
7133 "pass".to_owned()
7134 } else {
7135 format!(
7136 "FAIL ({:?})\n{}",
7137 o.code,
7138 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7139 )
7140 }
7141 ),
7142 );
7143 }
7144 Ok(outcomes)
7145 }
7146
7147 async fn gate_fix_round(
7160 &mut self,
7161 winner: &Candidate,
7162 outcomes: &[CommandOutcome],
7163 ) -> Result<GateFix> {
7164 let cap = self.state.config.graph.gate_fix_rounds;
7165 let spent = self.state.gate_fixes.len();
7166 if spent >= cap {
7167 if cap > 0 {
7168 self.state.event(
7169 "gate",
7170 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7171 );
7172 }
7173 return Ok(GateFix::Stop);
7174 }
7175 if !gate_fixable(outcomes) {
7176 self.state.event(
7177 "gate",
7178 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7179 command or similar); not spending a fix round on it",
7180 );
7181 return Ok(GateFix::Stop);
7182 }
7183 let min_free = self.state.config.disk.min_free_bytes;
7184 if min_free > 0 {
7185 match crate::disk::free_bytes(&winner.worktree) {
7186 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7187 Ok(free) => {
7188 self.state.event(
7189 "gate",
7190 format!(
7191 "only {free} bytes free ({min_free} required by `[disk] \
7192 min_free_bytes`); not spending a fix round on a failure the disk \
7193 may explain"
7194 ),
7195 );
7196 return Ok(GateFix::Stop);
7197 }
7198 Err(e) => {
7199 self.state.event(
7200 "gate",
7201 format!("free disk space could not be measured ({e:#}); no fix round"),
7202 );
7203 return Ok(GateFix::Stop);
7204 }
7205 }
7206 }
7207
7208 let attempt = spent + 1;
7209 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7210 let base = self.landing_base();
7211 let fix_prompt = prompt::gate_fix_in(
7212 &self.state.instruction,
7213 &failed,
7214 attempt,
7215 cap,
7216 &self.state.config.graph.language,
7217 &self.state.config.graph.github_language,
7218 );
7219 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7220 let sessions = self.state.config.graph.sessions;
7221 let artifacts = self.merge_repair_artifacts();
7222 self.state.event(
7223 "gate",
7224 format!("gate failed; gate-fix round {attempt} of {cap}"),
7225 );
7226 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7227 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7228 let (_, seat, out) = self
7229 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7230 prompt: fix_prompt.clone(),
7231 spec,
7232 seat,
7233 cwd: winner.worktree.clone(),
7234 timeout,
7235 allow_write: true,
7236 sessions,
7237 artifacts: artifacts.clone(),
7238 stem: format!("gate-fix-{attempt}"),
7239 handover: None,
7240 })
7241 .await;
7242 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7245 return Ok(GateFix::Defer);
7246 }
7247 let mut record = GateFixRecord {
7248 agent: seat.agent.clone(),
7249 failed,
7250 notes: String::new(),
7251 committed: false,
7252 error: None,
7253 };
7254 match out {
7255 AgentOutcome::Ok(o) => {
7256 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7259 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7260 }
7261 }
7262 AgentOutcome::Dropped(_) => {
7263 record.error = Some("the CLI dropped the stream".to_owned());
7264 }
7265 AgentOutcome::Quota(o) => {
7266 self.state.quota.push(QuotaLoss {
7267 seat: seat.key.clone(),
7268 node: "gate-fix".to_owned(),
7269 at: Timestamp::now(),
7270 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7271 });
7272 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7273 }
7274 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7275 }
7276 self.state.seats.insert(seat.key.clone(), seat);
7277 let coordination = self.coordinate(&winner.worktree).await?;
7278 if let Ok(r) = git::rescue_commit(
7279 &winner.worktree,
7280 &format!("magi: gate fix {attempt} (uncommitted work)"),
7281 )
7282 .await
7283 {
7284 self.state.note_withheld("gate-fix", &r.withheld);
7285 }
7286 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7287 record.committed = after != before;
7288 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7289 let note = record.error.clone();
7290 self.state.gate_fixes.push(record);
7291 self.state.save()?;
7292 if !changed {
7293 self.state.event(
7294 "gate",
7295 match note {
7296 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7297 None => format!("gate-fix round {attempt}: the tree did not change"),
7298 },
7299 );
7300 return Ok(GateFix::Stop);
7301 }
7302 self.state.event(
7303 "gate",
7304 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7305 );
7306
7307 drop(coordination);
7308 let commands = self.state.config.verify.e2e.clone();
7309 if !commands.is_empty() {
7310 let shell = self.state.config.shell();
7311 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7312 let cache_dir = self.state.config.cache_dir();
7313 let context = format!("gate-fix round {attempt}");
7314 let wait = self.repository_wait();
7315 let (e2e, _) = with_cache_lease(
7316 &mut self.state,
7317 cache_dir.as_deref(),
7318 "e2e",
7319 "e2e",
7320 &winner.worktree,
7321 &after,
7322 timeout,
7323 &context,
7324 &wait,
7325 |state, budget| {
7326 let shell = shell.clone();
7327 let commands = commands.clone();
7328 let context = context.clone();
7329 let worktree = winner.worktree.clone();
7330 async move {
7331 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7332 .await
7333 }
7334 },
7335 )
7336 .await?;
7337 if verify_inconclusive(&e2e) {
7338 return Ok(GateFix::Defer);
7339 }
7340 if e2e.iter().any(|o| !o.ok()) {
7341 self.state.event(
7342 "gate",
7343 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7344 );
7345 return Ok(GateFix::Stop);
7346 }
7347 }
7348 Ok(GateFix::Retry)
7349 }
7350
7351 async fn guarded_pr_message(
7358 &mut self,
7359 winner: &Candidate,
7360 facts: Option<&BranchFacts>,
7361 posting: bool,
7362 ) -> Result<Option<PrMessage>> {
7363 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7364 let head = match final_head.as_deref() {
7365 Some(h) => crate::pr_outcome::FinalHead::At(h),
7366 None => crate::pr_outcome::FinalHead::Unknown,
7367 };
7368 let summarized = if posting {
7370 Some(self.summarize_change(winner, final_head.as_deref()).await)
7371 } else {
7372 None
7373 };
7374 if self.state.resource_block.is_some() {
7379 self.state.event(
7380 "pr-summary",
7381 "a shared resource was busy; nothing was published, resume to summarize again",
7382 );
7383 return Ok(None);
7384 }
7385 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7386 anyhow::bail!(
7387 "the final commit moved during public description generation; retry before publishing"
7388 );
7389 }
7390 let note = match (&summarized, &self.state.change_summary) {
7391 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7392 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7393 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7394 (None, _) => ChangeNote::NotAsked,
7395 };
7396 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7397 if !posting {
7398 let identity = crate::scrub::Identity::current();
7399 return Ok(Some(PrMessage {
7400 title: crate::scrub::scrub(&pr.title, &identity),
7401 body: crate::scrub::scrub(&pr.body, &identity),
7402 }));
7403 }
7404 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7407 if let Some(known) = self
7408 .state
7409 .github_text
7410 .clone()
7411 .filter(|g| g.fingerprint == raw_fp)
7412 {
7413 return self.settle_withheld(winner, &pr, known).await;
7414 }
7415 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7416 let mut violations =
7417 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7418 let identity = crate::scrub::Identity::current();
7419 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7420 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7421 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7422 {
7423 violations.push(crate::github_text::Violation::SensitiveData);
7424 }
7425 let has_language = violations
7429 .iter()
7430 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7431 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7432 self.state.event(
7433 "github-text",
7434 if has_language {
7435 format!("description rejected: {violations:?}; requesting one rewrite")
7436 } else {
7437 format!("description rejected: {violations:?}; asking the owner")
7438 },
7439 );
7440 let mut rewritten = false;
7441 let mut rejected_rewrite = None;
7442 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7443 {
7444 let output = text;
7445 if let Some(title) = summary_title(&output) {
7446 let raw_body = summary_without_title(&output);
7449 let clean_title = crate::scrub::scrub(&title, &identity);
7450 let mut body = crate::scrub::scrub(&raw_body, &identity);
7451 let redacted = clean_title != title || body != raw_body;
7452 let title = clean_title;
7453 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7454 if !complete {
7455 self.state.event(
7456 "github-text",
7457 "description rewrite omitted required findings",
7458 );
7459 }
7460 if !body.trim().is_empty() {
7461 body.push_str("\n\n");
7462 body.push_str(&public_pr_evidence(&self.state, head));
7463 }
7464 let marker = format!("magi:run/{}", self.state.id);
7465 if !body.trim().is_empty() && !body.contains(&marker) {
7466 body.push_str(&format!("\n\n{marker}"));
7467 }
7468 let rewrite = PrMessage {
7469 title: title.clone(),
7470 body: body.clone(),
7471 };
7472 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7473 let mut failed = crate::github_text::check_for(
7474 &self.state.config.graph,
7475 &title,
7476 &body,
7477 redecision,
7478 );
7479 if (!crate::github_text::shareable(&title)
7480 || !crate::github_text::shareable(&body))
7481 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7482 {
7483 failed.push(crate::github_text::Violation::SensitiveData);
7484 }
7485 if complete
7486 && !title.trim().is_empty()
7487 && !body.trim().is_empty()
7488 && !failed.is_empty()
7489 {
7490 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7491 title: title.clone(),
7492 body: body.clone(),
7493 failed: failed
7494 .iter()
7495 .map(|v| crate::github_text::category(*v).to_owned())
7496 .collect(),
7497 });
7498 }
7499 if complete
7500 && !title.trim().is_empty()
7501 && !raw_body.trim().is_empty()
7502 && failed.is_empty()
7503 {
7504 decision = redecision;
7505 pr = PrMessage { title, body };
7506 rewritten = true;
7507 self.state
7508 .event("github-text", "description rewrite passed");
7509 if redacted {
7510 self.state.event(
7511 "github-text",
7512 "sensitive data removed from description rewrite",
7513 );
7514 }
7515 }
7516 }
7517 }
7518 if !rewritten {
7519 self.state.event(
7520 "github-text",
7521 "description rewrite unavailable or rejected; asking the owner",
7522 );
7523 if let Some(w) =
7524 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7525 {
7526 self.retire_github_text_question("the withheld text changed");
7529 let record = crate::run::GithubTextAsk {
7530 rewrite: rejected_rewrite,
7531 fingerprint: raw_fp,
7532 title: w.title,
7533 body: w.body,
7534 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7535 question: None,
7536 asks: 0,
7537 resolved: false,
7538 chosen_title: None,
7539 choice: None,
7540 };
7541 return self.settle_withheld(winner, &pr, record).await;
7542 }
7543 }
7544 }
7545 let (title, body) =
7546 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7547 Ok(Some(PrMessage { title, body }))
7548 }
7549
7550 async fn summarize_change(
7561 &mut self,
7562 winner: &Candidate,
7563 head: Option<&str>,
7564 ) -> std::result::Result<(), String> {
7565 let Some(head) = head else {
7566 return Err("the final commit could not be read".to_owned());
7567 };
7568 let remote = self.state.config.merge.remote.clone();
7569 let tracking = format!("{remote}/{}", self.state.base_branch);
7570 if !git::rev_exists(&winner.worktree, &tracking).await {
7571 return Err(format!("the remote base `{tracking}` could not be read"));
7572 }
7573 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7574 Ok(b) if !b.is_empty() => b,
7575 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7576 };
7577 let evidence = crate::pr_outcome::summary_evidence(
7578 &self.state,
7579 crate::pr_outcome::FinalHead::At(head),
7580 );
7581 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7582 if self
7583 .state
7584 .change_summary
7585 .as_ref()
7586 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7587 {
7588 return Ok(());
7589 }
7590 if self.state.change_summary.take().is_some() {
7591 self.state.event(
7592 "pr-summary",
7593 "the final diff or evidence changed; the earlier change summary was discarded",
7594 );
7595 }
7596 if let Err(e) = write_artifact(
7598 &self.state,
7599 &format!("public-description-{fingerprint}.md"),
7600 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7601 ) {
7602 tracing::warn!("could not save public description evidence: {e:#}");
7603 }
7604 let chain = match agent::pick_chain(
7605 &self.state.config.agents,
7606 self.state.config.roles.synthesizer.as_ref(),
7607 &agent::installed,
7608 "synthesizer",
7609 ) {
7610 Ok(chain) => chain,
7611 Err(e) => {
7612 tracing::warn!("no agent for the change summary: {e:#}");
7613 self.state.event("pr-summary", "no summary agent available");
7614 return Err("no agent was available to summarize it".to_owned());
7615 }
7616 };
7617 let subjects = git::subjects(&winner.worktree, &base, head)
7618 .await
7619 .unwrap_or_default();
7620 let stat = git::diff_stat(&winner.worktree, &base, head)
7621 .await
7622 .unwrap_or_default();
7623 let stat: String = stat.chars().take(6_000).collect();
7624 let mut prompt = crate::prompt::change_summary_in(
7625 &base,
7626 head,
7627 &subjects,
7628 stat.trim(),
7629 &self.state.config.graph.github_language,
7630 );
7631 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7632 let sessions = self.state.config.graph.sessions;
7633 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7634 let prompts = self.state.config.prompts.clone();
7635 let cache = self.state.config.cache_dir();
7636 let run = self.state.id.clone();
7637 for (i, spec) in chain.into_iter().enumerate() {
7638 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7639 let job = SeatJob {
7640 spec: spec.clone(),
7641 seat,
7642 cwd: winner.worktree.clone(),
7643 prompt: prompt.clone(),
7644 timeout,
7645 allow_write: false,
7646 sessions,
7647 artifacts: agent::artifacts_dir(&self.state.dir()),
7648 stem: if i == 0 {
7649 "change-summary".to_owned()
7650 } else {
7651 format!("change-summary-{}", spec.id)
7652 },
7653 handover: None,
7654 };
7655 let ctx = WaveCtx {
7656 run: &run,
7657 node: "pr-summary",
7658 prompts: &prompts,
7659 cache: cache.as_deref(),
7660 round: None,
7661 carry_seats: false,
7662 pauses: Some((&self.pause, &self.interrupt)),
7663 };
7664 let (seat, outcome) =
7665 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7666 self.state.seats.insert(seat.key.clone(), seat);
7667 let text = match outcome {
7668 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7669 AgentOutcome::ResourceBlocked(why) => {
7670 if self.state.resource_block.is_none() {
7673 self.state.resource_block = Some(crate::run::ResourceBlock {
7674 node: "pr-summary".to_owned(),
7675 kind: "shared resource".to_owned(),
7676 detail: why,
7677 at: Timestamp::now(),
7678 });
7679 }
7680 return Err("a shared resource was busy".to_owned());
7681 }
7682 _ => continue,
7683 };
7684 let blind = &self.state.config.blind;
7685 let Some(title) = summary_title(&text) else {
7686 continue;
7687 };
7688 let body: String = summary_without_title(&text)
7689 .lines()
7690 .filter(|l| !l.contains("magi:run/"))
7691 .collect::<Vec<_>>()
7692 .join("\n");
7693 let body = blind::sanitize_prose(body.trim(), blind);
7694 let title = blind::sanitize_prose(&title, blind);
7695 if body.trim().is_empty() || title.trim().is_empty() {
7696 continue;
7697 }
7698 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7699 continue;
7700 }
7701 if git::rev_parse(&winner.worktree, "HEAD")
7703 .await
7704 .ok()
7705 .as_deref()
7706 != Some(head)
7707 {
7708 return Err("the branch moved while the summary was written".to_owned());
7709 }
7710 if git::merge_base(&winner.worktree, &tracking, head)
7711 .await
7712 .ok()
7713 .as_deref()
7714 != Some(base.as_str())
7715 || crate::pr_outcome::summary_evidence(
7716 &self.state,
7717 crate::pr_outcome::FinalHead::At(head),
7718 ) != evidence
7719 {
7720 return Err("the summary evidence changed while it was written".to_owned());
7721 }
7722 self.state.change_summary = Some(crate::run::ChangeSummary {
7723 head: head.to_owned(),
7724 base,
7725 evidence: fingerprint,
7726 title,
7727 body,
7728 });
7729 self.state.event(
7730 "pr-summary",
7731 format!("change summary written by {} for the final diff", spec.id),
7732 );
7733 if let Err(e) = self.state.save() {
7734 tracing::warn!("could not save the change summary: {e:#}");
7735 }
7736 return Ok(());
7737 }
7738 self.state
7739 .event("pr-summary", "no agent produced a change summary");
7740 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7741 }
7742
7743 async fn rewrite_pr_text(
7751 &mut self,
7752 winner: &Candidate,
7753 pr: &PrMessage,
7754 violations: &[crate::github_text::Violation],
7755 ) -> Option<String> {
7756 let prompt = format!(
7757 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7758 pr.title, pr.body
7759 );
7760 let sessions = self.state.config.graph.sessions;
7761 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7762 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7764 let key = format!("impl-{}", winner.label);
7765 if let Some(spec) = self
7766 .state
7767 .config
7768 .agents
7769 .iter()
7770 .find(|a| a.id == winner.agent)
7771 .cloned()
7772 && let Some(existing) = self.state.seats.get(&key)
7773 && existing.agent == spec.id
7774 && agent::has_session(spec.kind, existing, sessions)
7775 {
7776 attempts.push((spec, existing.clone(), true));
7777 } else {
7778 let chain = agent::pick_chain(
7779 &self.state.config.agents,
7780 self.state.config.roles.language_judge.as_ref(),
7781 &agent::installed,
7782 "language_judge",
7783 );
7784 match chain {
7785 Ok(chain) => {
7786 for spec in chain {
7787 let seat = crate::fixer::seat_for(
7788 &mut self.state,
7789 "github-text-rewrite",
7790 &spec.id,
7791 );
7792 attempts.push((spec, seat, false));
7793 }
7794 }
7795 Err(e) => {
7796 tracing::warn!("no agent for the description rewrite: {e:#}");
7797 self.state
7798 .event("github-text", "no rewrite agent available");
7799 return None;
7800 }
7801 }
7802 }
7803 let prompts = self.state.config.prompts.clone();
7804 let cache = self.state.config.cache_dir();
7805 let run = self.state.id.clone();
7806 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7807 let job = SeatJob {
7808 spec: spec.clone(),
7809 seat,
7810 cwd: winner.worktree.clone(),
7811 prompt: prompt.clone(),
7812 timeout: retry_budget(timeout_full, resumed),
7813 allow_write: false,
7814 sessions,
7815 artifacts: agent::artifacts_dir(&self.state.dir()),
7816 stem: if i == 0 {
7817 "github-text-rewrite".to_owned()
7818 } else {
7819 format!("github-text-rewrite-{}", spec.id)
7820 },
7821 handover: None,
7822 };
7823 let ctx = WaveCtx {
7824 run: &run,
7825 node: "github-text",
7826 prompts: &prompts,
7827 cache: cache.as_deref(),
7828 round: None,
7829 carry_seats: false,
7830 pauses: Some((&self.pause, &self.interrupt)),
7831 };
7832 let (seat, outcome) =
7833 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7834 self.state.seats.insert(seat.key.clone(), seat);
7835 match outcome {
7836 AgentOutcome::Ok(o)
7837 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7838 {
7839 return Some(o.text);
7840 }
7841 AgentOutcome::ResourceBlocked(_) => return None,
7842 _ => {}
7843 }
7844 }
7845 None
7846 }
7847
7848 fn retire_github_text_question(&mut self, why: &str) {
7850 let Some(id) = self
7851 .state
7852 .github_text
7853 .as_ref()
7854 .and_then(|g| g.question.clone())
7855 else {
7856 return;
7857 };
7858 let _ = ask::Questions::open().update(&id, |q| {
7859 q.abandon(why);
7860 Ok(())
7861 });
7862 }
7863
7864 async fn settle_withheld(
7868 &mut self,
7869 winner: &Candidate,
7870 pr: &PrMessage,
7871 mut g: crate::run::GithubTextAsk,
7872 ) -> Result<Option<PrMessage>> {
7873 use crate::github_text as gt;
7874 let store = ask::Questions::open();
7875 let timeout = self.state.config.graph.answer_timeout;
7876 if !g.resolved {
7877 let mut standing = g
7878 .question
7879 .as_ref()
7880 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7881 if let Some(q) = standing.as_ref()
7882 && q.status.open()
7883 && gt::expired(q, timeout)
7884 {
7885 let why = format!("no answer within {}s of asking", timeout.max(1));
7886 standing = store
7887 .update(&q.id, |q| {
7888 q.abandon(&why);
7889 Ok(())
7890 })
7891 .ok()
7892 .map(|(q, ())| q);
7893 }
7894 let mut ask_again: Option<Option<Vec<String>>> = None;
7895 match standing {
7896 None if g.asks <= 1 => {
7899 g.asks = 0;
7900 ask_again = Some(None);
7901 }
7902 None => {
7903 g.resolved = true;
7904 g.chosen_title = None;
7905 g.choice = Some(GithubTextChoice::Fallback);
7906 }
7907 Some(q) if q.status.open() => {
7908 self.park_on_github_text(&g)?;
7909 return Ok(None);
7910 }
7911 Some(q) => match gt::read_reply(&q) {
7912 gt::Reply::Fallback => {
7913 g.resolved = true;
7914 g.chosen_title = None;
7915 g.choice = Some(GithubTextChoice::Fallback);
7916 }
7917 gt::Reply::Original => {
7918 g.resolved = true;
7919 g.chosen_title = None;
7920 g.choice = Some(GithubTextChoice::Original);
7921 }
7922 gt::Reply::Rewrite => {
7923 g.resolved = true;
7924 g.chosen_title = None;
7925 g.choice = Some(if g.rewrite.is_some() {
7926 GithubTextChoice::Rewrite
7927 } else {
7928 GithubTextChoice::Fallback
7929 });
7930 }
7931 gt::Reply::Title(text) => {
7932 let first = text
7933 .lines()
7934 .map(str::trim)
7935 .find(|l| !l.is_empty())
7936 .unwrap_or("");
7937 let decision = if first.is_empty() {
7938 None
7939 } else {
7940 let candidate = PrMessage {
7941 title: first.to_owned(),
7942 body: String::new(),
7943 };
7944 self.judge_pr_language(&winner.worktree, &candidate).await
7945 };
7946 match gt::vet_title(
7947 &text,
7948 decision,
7949 self.state.config.graph.github_english(),
7950 ) {
7951 Ok(title) => {
7952 g.resolved = true;
7953 g.chosen_title = Some(title);
7954 g.choice = Some(GithubTextChoice::MyTitle);
7955 }
7956 Err(cats) if g.asks < 2 => {
7957 ask_again = Some(Some(
7958 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7959 ))
7960 }
7961 Err(_) => {
7962 g.resolved = true;
7963 g.chosen_title = None;
7964 g.choice = Some(GithubTextChoice::Fallback);
7965 }
7966 }
7967 }
7968 },
7969 }
7970 if let Some(rejected) = ask_again {
7971 let w = gt::Withheld {
7974 title: g.title,
7975 body: g.body,
7976 categories: g.categories.clone(),
7977 };
7978 let artifact = match crate::run::write_artifact(
7981 &self.state,
7982 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7983 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7984 ) {
7985 Ok(p) => Some(p),
7986 Err(e) => {
7987 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7988 None
7989 }
7990 };
7991 let (detail, comparison) = gt::question_content(
7992 &w,
7993 &pr.title,
7994 &pr.body,
7995 &self.state.id,
7996 rejected.as_deref(),
7997 artifact.as_deref(),
7998 g.rewrite.as_ref(),
7999 );
8000 let mut q = ask::Question::new(
8001 self.state.id.clone(),
8002 gt::ASK_NODE.to_owned(),
8003 gt::ASK_SEAT.to_owned(),
8004 gt::question_summary(&self.state.config.graph.language, &w),
8005 detail,
8006 gt::question_choices(&w, g.rewrite.as_ref()),
8007 );
8008 q.posting_comparison = Some(comparison);
8009 q.answer_timeout = timeout;
8010 g.question = Some(q.id.clone());
8013 g.asks += 1;
8014 self.state.github_text = Some(g.clone());
8015 self.state.event(
8016 "github-text",
8017 format!("asking the owner about the withheld text ({})", q.short()),
8018 );
8019 self.state.save()?;
8020 store
8021 .put_executable(&mut q)
8022 .context("file the posting-gate question")?;
8023 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8024 tracing::warn!(
8025 "could not notify about posting-gate question {}: {e:#}",
8026 q.short()
8027 );
8028 }
8029 self.park_on_github_text(&g)?;
8030 return Ok(None);
8031 }
8032 }
8033 let resolved = g.chosen_title.clone();
8034 let original = g.choice == Some(GithubTextChoice::Original);
8035 self.state.event(
8037 "github-text",
8038 if original {
8039 format!(
8040 "posting the original text for: {}",
8041 match (g.title, g.body) {
8042 (true, true) => "title, description",
8043 (true, false) => "title",
8044 _ => "description",
8045 }
8046 )
8047 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8048 "posting the saved rewrite title and description".to_owned()
8049 } else if resolved.is_some() {
8050 "posting the owner's replacement title".to_owned()
8051 } else {
8052 "posting the neutral text for the withheld fields".to_owned()
8053 },
8054 );
8055 self.state.github_text = Some(g.clone());
8056 self.state.save()?;
8057 if g.choice == Some(GithubTextChoice::Rewrite)
8058 && let Some(r) = g.rewrite.as_ref()
8059 {
8060 return Ok(Some(PrMessage {
8061 title: r.title.clone(),
8062 body: r.body.clone(),
8063 }));
8064 }
8065 let w = gt::Withheld {
8066 title: g.title,
8067 body: g.body,
8068 categories: g.categories.clone(),
8069 };
8070 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8071 if let Some(t) = resolved {
8072 title = t;
8073 }
8074 let english = gt::LanguageDecision {
8076 title_english: true,
8077 body_english: true,
8078 body_complete: true,
8079 };
8080 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8081 if original {
8082 if g.title {
8085 title = pr.title.clone();
8086 }
8087 if g.body {
8088 body = pr.body.clone();
8089 }
8090 }
8091 Ok(Some(PrMessage { title, body }))
8092 }
8093
8094 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8095 self.state.github_text = Some(g.clone());
8096 self.state.end_pass_parked(ParkCause::Question);
8097 self.state.parked = true;
8098 self.state.event(
8099 "github-text",
8100 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8101 );
8102 self.state.save()
8103 }
8104
8105 async fn judge_pr_language(
8108 &mut self,
8109 cwd: &Path,
8110 pr: &PrMessage,
8111 ) -> Option<crate::github_text::LanguageDecision> {
8112 self.state.config.roles.language_judge.as_ref()?;
8113 if !self.state.config.graph.language_gate() {
8114 return None;
8115 }
8116 let decision =
8117 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8118 self.state.event(
8119 "github-text",
8120 if decision.is_some() {
8121 "language decided by the language judge"
8122 } else {
8123 "language judge unavailable; using built-in heuristics"
8124 },
8125 );
8126 decision
8127 }
8128
8129 async fn merge(&mut self) -> Result<()> {
8130 if self
8145 .state
8146 .base_sync
8147 .as_ref()
8148 .is_some_and(|s| s.conflict.is_some())
8149 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8150 != Some(RunStatus::Gating)
8151 || !self.state.gate_allows_progress()
8159 {
8160 return Ok(());
8161 }
8162 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8171 return Ok(());
8172 }
8173 let Some(winner) = self.state.winner().cloned() else {
8174 return Ok(());
8175 };
8176 let repo = self.state.repo.clone();
8177 let base = self.state.base_branch.clone();
8178 let mode = self.state.config.merge.mode;
8179 let style = self.state.config.merge.style;
8180 let facts = if is_review_run(&self.state) {
8181 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8182 let start = review_base(
8183 &repo,
8184 &self.state.config.merge.remote,
8185 &base,
8186 &self.state.base_commit,
8187 &winner.branch,
8188 )
8189 .await;
8190 branch_facts(&repo, &start, &winner.branch).await
8191 } else {
8192 None
8193 };
8194 let leaked = if is_review_run(&self.state) {
8197 leaked_subjects(&self.state, &winner.branch).await
8198 } else {
8199 Some(Vec::new())
8200 };
8201 let Some(pr) = self
8205 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8206 .await?
8207 else {
8208 return Ok(());
8209 };
8210 let message = pr.commit_message();
8211
8212 let outcome = match mode {
8213 MergeMode::None => MergeOutcome {
8214 mode,
8215 ok: true,
8216 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8217 empty: false,
8218 },
8219 MergeMode::Pr | MergeMode::Local
8220 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8221 {
8222 MergeOutcome {
8223 mode,
8224 ok: false,
8225 detail: empty_candidate_detail(&self.state, &base),
8226 empty: true,
8227 }
8228 }
8229 MergeMode::Local => {
8230 let on = git::current_branch(&repo).await?;
8231 if on.as_deref() != Some(base.as_str()) {
8232 MergeOutcome {
8233 mode,
8234 ok: false,
8235 detail: format!(
8236 "{} has {} checked out, not the base branch {base}",
8237 repo.display(),
8238 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8239 ),
8240 empty: false,
8241 }
8242 } else if !git::is_clean(&repo).await? {
8243 MergeOutcome {
8244 mode,
8245 ok: false,
8246 detail: format!("{} is dirty; refusing to merge", repo.display()),
8247 empty: false,
8248 }
8249 } else {
8250 let out = match style {
8251 MergeStyle::Merge => {
8252 git::merge_no_ff(&repo, &winner.branch, &message).await?
8253 }
8254 MergeStyle::Squash => {
8255 git::merge_squash(&repo, &winner.branch, &message).await?
8256 }
8257 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8258 };
8259 MergeOutcome {
8260 mode,
8261 ok: out.ok(),
8262 detail: if out.ok() { out.stdout } else { out.stderr },
8263 empty: false,
8264 }
8265 }
8266 }
8267 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8268 self.state.pr_publication_pending = true;
8269 self.state.save()?;
8270 let out =
8271 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8272 self.state.pr_publication_pending = out.is_err();
8273 MergeOutcome {
8274 mode,
8275 ok: out.is_ok(),
8276 detail: match out {
8277 Ok(url) => url,
8278 Err(e) => format!("{e:#}"),
8279 },
8280 empty: false,
8281 }
8282 }
8283 MergeMode::Pr => {
8284 let remote = self.state.config.merge.remote.clone();
8285 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8286 if !pushed.ok() {
8287 MergeOutcome {
8288 mode,
8289 ok: false,
8290 detail: pushed.stderr,
8291 empty: false,
8292 }
8293 } else {
8294 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8300 let out = match pr_merge_plan(found) {
8301 PrPlan::Create => {
8302 gh_pr_create(
8303 &winner.worktree,
8304 &base,
8305 &winner.branch,
8306 &pr.title,
8307 &pr.body,
8308 )
8309 .await
8310 }
8311 PrPlan::Adopt { url, title } => {
8312 self.state
8313 .event("merge", format!("Pr: adopted open pull request {url}"));
8314 if self.state.github_text.as_ref().is_some_and(|g| {
8315 g.resolved
8316 && g.choice == Some(GithubTextChoice::Rewrite)
8317 && g.rewrite.is_some()
8318 }) {
8319 land::set_pr_text_verbatim(
8322 &winner.worktree,
8323 &url,
8324 &pr.title,
8325 &pr.body,
8326 )
8327 .await
8328 .map(|()| url)
8329 } else {
8330 if title != pr.title
8331 && (!is_review_run(&self.state)
8332 || leaked
8333 .as_deref()
8334 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8335 && let Err(e) =
8336 if self.state.github_text.as_ref().is_some_and(|g| {
8337 g.resolved
8338 && g.title
8339 && g.choice == Some(GithubTextChoice::Original)
8340 }) {
8341 land::set_pr_title_verbatim(
8343 &winner.worktree,
8344 &url,
8345 &pr.title,
8346 )
8347 .await
8348 } else {
8349 land::set_pr_title(
8350 &mut self.state,
8351 &winner.worktree,
8352 &url,
8353 &pr.title,
8354 )
8355 .await
8356 }
8357 {
8358 tracing::warn!("could not refresh title of {url}: {e:#}");
8359 self.state
8360 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8361 }
8362 Ok(url)
8363 }
8364 }
8365 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8366 };
8367 match out {
8368 Ok(url) => MergeOutcome {
8369 mode,
8370 ok: true,
8371 detail: url,
8372 empty: false,
8373 },
8374 Err(e) => MergeOutcome {
8375 mode,
8376 ok: false,
8377 detail: e.to_string(),
8378 empty: false,
8379 },
8380 }
8381 }
8382 }
8383 };
8384
8385 self.state.status = match (mode, outcome.ok) {
8386 (MergeMode::None, _) => RunStatus::Ready,
8387 (_, true) if outcome.empty => RunStatus::Ready,
8388 (MergeMode::Pr, true) => RunStatus::Ready,
8389 (_, true) => RunStatus::Merged,
8390 (_, false) => RunStatus::Blocked,
8391 };
8392 self.state.event(
8393 "merge",
8394 format!(
8395 "{:?}: {}",
8396 mode,
8397 outcome.detail.lines().next().unwrap_or("")
8398 ),
8399 );
8400 self.state.merge = Some(outcome);
8401 self.state.save()?;
8402
8403 if self.state.config.graph.land
8409 && mode == MergeMode::Pr
8410 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8411 {
8412 self.run_land().await?;
8413 }
8414 self.settle_questions();
8419 Ok(())
8420 }
8421
8422 async fn run_land(&mut self) -> Result<()> {
8433 crate::pr_head::ensure_owner(&self.state)?;
8434 let url = self
8435 .state
8436 .merge
8437 .as_ref()
8438 .map(|m| m.detail.clone())
8439 .unwrap_or_default();
8440 let url = url.lines().next().unwrap_or("").trim().to_owned();
8441 if !url.starts_with("http") {
8442 return Ok(());
8443 }
8444 loop {
8447 match land::land(&mut self.state, &url).await {
8448 Ok(_)
8449 if self
8450 .state
8451 .merge_repairs
8452 .last()
8453 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8454 {
8455 if self.repair_merge_review().await? {
8456 continue;
8457 }
8458 }
8459 Ok(pr) if self.state.parked => {
8460 let _ = pr;
8464 }
8465 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8466 self.state.status = RunStatus::Superseded;
8467 self.state.save()?;
8468 }
8469 Ok(pr) => {
8470 self.state.status = match pr.state {
8471 land::PrLifecycle::Merged => RunStatus::Merged,
8472 _ => RunStatus::Blocked,
8473 };
8474 if bump::should_release_bump(self.state.status)
8481 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8482 {
8483 self.state
8489 .event("bump", format!("release bump skipped: {e:#}"));
8490 }
8491 if self.state.status == RunStatus::Merged {
8494 crate::followup::after_merge(&mut self.state, &pr.url).await;
8495 }
8496 self.state.save()?;
8497 }
8498 Err(e) => {
8499 if e.is::<crate::repo_lock::ResourceBlocked>() {
8500 return Err(e);
8501 }
8502 self.state.status = RunStatus::Blocked;
8503 self.state.event("land", format!("gave up: {e}"));
8504 self.state.save()?;
8505 }
8506 }
8507 break;
8508 }
8509 Ok(())
8510 }
8511
8512 async fn repair_merge_review(&mut self) -> Result<bool> {
8515 use crate::run::MergeRepairPhase as Phase;
8516 crate::pr_head::ensure_owner(&self.state)?;
8517 let i = self.state.merge_repairs.len() - 1;
8518 let request = self.state.merge_repairs[i].clone();
8519 if request.phase == Phase::Fixing {
8520 let restored = match self.state.winner().cloned() {
8521 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8522 None => Err(anyhow::anyhow!("repair has no winning branch")),
8523 };
8524 let why = match restored {
8525 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8526 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8527 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8528 };
8529 self.hold_merge_repair(i, why)?;
8530 return Ok(false);
8531 }
8532 let failure = if i >= self.state.config.graph.land_rounds
8533 || self.state.config.graph.review_rounds == 0
8534 {
8535 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8536 } else {
8537 None
8538 };
8539 if let Some(why) = failure {
8540 self.hold_merge_repair(i, why)?;
8541 return Ok(false);
8542 }
8543 if request.phase == Phase::Requested {
8544 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8545 Ok(pr) => pr,
8546 Err(e) => {
8547 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8548 return Ok(false);
8549 }
8550 };
8551 self.state.merge_repairs[i].phase = Phase::Fixing;
8552 self.state.land_approval = None;
8553 self.state.land_armed_head = None;
8554 self.state.save()?;
8555 let evidence = serde_json::to_string_pretty(&request)?;
8558 let reason = format!(
8559 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8560 );
8561 let budget = self.state.config.graph.land_rounds;
8562 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8563 match result {
8564 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8565 Ok(land::Fixed::Failed(why)) => {
8566 self.hold_merge_repair(
8567 i,
8568 format!(
8569 "repair failed: {why}; inspect the branch and run an explicit review"
8570 ),
8571 )?;
8572 return Ok(false);
8573 }
8574 Err(e) => {
8575 self.hold_merge_repair(
8576 i,
8577 format!(
8578 "repair failed: {e:#}; inspect the branch and run an explicit review"
8579 ),
8580 )?;
8581 return Ok(false);
8582 }
8583 }
8584 self.begin_merge_repair_review(i)?;
8588 }
8589 self.review_loop().await?;
8590 self.gate().await?;
8591 if self.resource_halt()? || self.state.parked {
8592 return Ok(false);
8593 }
8594 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8595 != Some(RunStatus::Gating)
8596 || !self.state.gate_allows_progress()
8597 {
8598 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8599 return Ok(false);
8600 }
8601 let winner = self
8604 .state
8605 .winner()
8606 .cloned()
8607 .context("repair has no winning branch")?;
8608 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8609 if self.state.gate_head.as_deref() != Some(after.as_str())
8610 || !git::is_clean(&winner.worktree).await?
8611 {
8612 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8613 return Ok(false);
8614 }
8615 let expected = self
8616 .state
8617 .pr_pushed_head
8618 .as_deref()
8619 .unwrap_or(&request.head)
8620 .to_owned();
8621 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8622 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8625 let refspec = format!("{after}:refs/heads/{public}");
8626 let push = git::git_raw(
8627 &winner.worktree,
8628 &["push", &lease, &self.state.config.merge.remote, &refspec],
8629 )
8630 .await;
8631 match push {
8632 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8633 Ok(out) => {
8634 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8635 return Ok(false);
8636 }
8637 Err(e) => {
8638 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8639 return Ok(false);
8640 }
8641 }
8642 self.state.merge_repairs[i].phase = Phase::Complete;
8643 self.state.status = RunStatus::Landing;
8644 self.state.event(
8645 "repair",
8646 "new review and gate completed; fresh owner merge decision required",
8647 );
8648 self.state.save()?;
8649 Ok(true)
8650 }
8651
8652 fn merge_repair_artifacts(&self) -> PathBuf {
8653 let artifacts = agent::artifacts_dir(&self.state.dir());
8654 match self
8655 .state
8656 .merge_repairs
8657 .last()
8658 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8659 {
8660 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8661 None => artifacts,
8662 }
8663 }
8664
8665 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8666 self.state.reviews.clear();
8667 self.state.gate.clear();
8668 self.state.gate_ran = false;
8669 self.state.gate_head = None;
8670 self.state.gate_fixes.clear();
8671 self.state.pre_gate.clear();
8672 self.state.pre_gate_commit = None;
8673 self.state.contested_handoff = None;
8674 self.state.land_reruns = None;
8675 self.state.land_approval = None;
8676 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8677 self.state.status = RunStatus::Reviewing;
8678 self.state.save()
8679 }
8680
8681 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8682 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8683 self.state.merge_repairs[index].failure = Some(why.clone());
8684 self.state.status = RunStatus::Blocked;
8685 self.state.event("repair", why);
8686 self.state.save()
8687 }
8688
8689 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8693 if let Some(existing) = self.state.seats.get(key)
8694 && existing.agent == agent
8695 {
8696 return existing.clone();
8697 }
8698 let fresh = if self.state.seats.contains_key(key) {
8701 handover_seat(key, agent, self.state.next_seat_seed())
8702 } else {
8703 SeatState::new(key, agent, self.state.seed)
8704 };
8705 self.state.seats.insert(key.to_owned(), fresh.clone());
8706 fresh
8707 }
8708
8709 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8715 match self.state.seats.get(key) {
8716 Some(s) if s.agent != spec.id => {
8717 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8718 }
8719 _ => spec,
8720 }
8721 }
8722
8723 fn view(&self, c: &Candidate) -> CandidateView {
8725 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8726 .unwrap_or_default();
8727 let (patch, _) = blind::sanitize_patch(
8728 &format!("candidate {} patch", c.label),
8729 &raw,
8730 &self.state.config.blind,
8731 );
8732 CandidateView {
8733 label: c.label,
8734 branch: c.branch.clone(),
8735 summary: c.summary.clone(),
8736 stat: c.stat.clone(),
8737 patch,
8738 }
8739 }
8740
8741 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8743 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8744 prompt::judge(
8745 "(see above)",
8746 &views,
8747 self.roles.judges.len(),
8748 base_short,
8749 "en",
8750 )
8751 }
8752
8753 fn vote_prompt_full(
8759 &self,
8760 j: usize,
8761 labels: &[char],
8762 language: &str,
8763 candidates: &[Candidate],
8764 base_short: &str,
8765 ) -> String {
8766 let mut text = format!(
8767 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8768 prompt::final_vote(labels, language),
8769 self.state.instruction,
8770 self.candidate_block(candidates, base_short)
8771 );
8772 if let Some(own) = self
8773 .state
8774 .judgements
8775 .get(j)
8776 .filter(|r| !r.ranking.is_empty())
8777 {
8778 let reasons = own
8779 .reasons
8780 .iter()
8781 .map(|(k, v)| format!("- {k}: {v}"))
8782 .collect::<Vec<_>>()
8783 .join("\n");
8784 text.push_str(&format!(
8785 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8786 own.ranking.iter().collect::<String>(),
8787 if reasons.is_empty() {
8788 ""
8789 } else {
8790 ", because:\n"
8791 }
8792 ));
8793 }
8794 if !self.state.deliberation.is_empty() {
8795 text.push_str("\n# What was argued before this vote\n");
8796 for t in self.transcript(&[], j) {
8797 text.push_str(&format!(
8798 "\n## {}{}\n\n{}\n",
8799 t.who,
8800 if t.is_self { " (you)" } else { "" },
8801 t.body.trim()
8802 ));
8803 }
8804 }
8805 text
8806 }
8807
8808 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8815 let mut turns = Vec::new();
8816 for j in &self.state.judgements {
8817 if j.ranking.is_empty() {
8818 continue;
8819 }
8820 let reasons = j
8821 .reasons
8822 .iter()
8823 .map(|(k, v)| format!("- {k}: {v}"))
8824 .collect::<Vec<_>>()
8825 .join("\n");
8826 turns.push(Turn {
8827 who: format!("Judge {} (opening ranking)", j.judge),
8828 is_self: j.judge == self_idx + 1,
8829 body: format!(
8830 "Ranked {}{}{reasons}",
8831 j.ranking.iter().collect::<String>(),
8832 if reasons.is_empty() {
8833 ""
8834 } else {
8835 ", because:\n"
8836 }
8837 ),
8838 });
8839 }
8840 for t in self
8841 .state
8842 .deliberation
8843 .iter()
8844 .flat_map(|r| r.turns.iter())
8845 .chain(current)
8846 {
8847 turns.push(Turn {
8848 who: format!("Judge {}", t.judge),
8849 is_self: t.judge == self_idx + 1,
8850 body: t.body.clone(),
8851 });
8852 }
8853 turns
8854 }
8855}
8856
8857fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8859 agent::has_session(spec.kind, seat, sessions)
8860}
8861
8862fn next_untried_in_roster<'a>(
8883 roster: &'a [AgentSpec],
8884 start: usize,
8885 tried: &BTreeSet<String>,
8886) -> Option<&'a AgentSpec> {
8887 roster
8888 .get(start + 1..)?
8889 .iter()
8890 .find(|s| !tried.contains(&s.id))
8891}
8892
8893fn pick_successor<'a>(
8914 roster: &'a [AgentSpec],
8915 start: usize,
8916 tried: &BTreeSet<String>,
8917 carried: Option<&BTreeSet<String>>,
8918 others: &BTreeSet<String>,
8919) -> Option<&'a AgentSpec> {
8920 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8921 roster
8922 .get(start + 1..)
8923 .and_then(|tail| {
8924 tail.iter()
8925 .filter(free)
8926 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8927 })
8928 .or_else(|| {
8929 carried
8930 .is_some()
8931 .then(|| roster.iter().find(free))
8932 .flatten()
8933 })
8934 .or_else(|| match carried {
8935 Some(c) => next_for_seat(roster, start, tried, c),
8936 None => next_untried_in_roster(roster, start, tried),
8937 })
8938}
8939
8940fn next_for_seat<'a>(
8950 roster: &'a [AgentSpec],
8951 start: usize,
8952 round_tried: &BTreeSet<String>,
8953 carried_failed: &BTreeSet<String>,
8954) -> Option<&'a AgentSpec> {
8955 roster
8956 .get(start + 1..)?
8957 .iter()
8958 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8959 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8960}
8961
8962fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8968 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8969 return spec;
8970 };
8971 let ok = |id: &str| !h.failed.contains(id);
8972 if let Some(last) = h.last_ok.as_deref()
8973 && ok(last)
8974 && let Some(s) = roster.iter().find(|s| s.id == last)
8975 {
8976 return s.clone();
8977 }
8978 if ok(&spec.id) {
8979 return spec;
8980 }
8981 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8982 roster
8983 .get(from + 1..)
8984 .into_iter()
8985 .flatten()
8986 .chain(roster.iter())
8987 .find(|s| ok(&s.id))
8988 .cloned()
8989 .unwrap_or(spec)
8990}
8991
8992pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8996 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8997}
8998
8999const TIMED_OUT: &str = "timed out";
9002
9003impl FailClass {
9004 fn of(out: &AgentOutcome) -> Option<Self> {
9006 match out {
9007 AgentOutcome::Ok(_) => None,
9008 AgentOutcome::Quota(_) => Some(Self::Quota),
9009 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
9010 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
9011 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
9012 AgentOutcome::ResourceBlocked(_) => None,
9015 }
9016 }
9017
9018 fn stem_word(&self) -> &'static str {
9020 match self {
9021 Self::Quota => "quota",
9022 _ => "handover",
9023 }
9024 }
9025}
9026
9027fn failure_signature(msg: &str) -> String {
9031 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9032 if let Some(code) = line
9035 .strip_prefix("process exited with no usable output (exit code ")
9036 .and_then(|s| s.strip_suffix(')'))
9037 .and_then(|s| s.parse::<i32>().ok())
9038 {
9039 line = format!("exited with some({code}) and no usable output");
9040 } else if matches!(
9041 line.as_str(),
9042 "process exited with no usable output (no exit code reported)"
9043 | "process exited with no usable output (exit code unavailable)"
9044 ) {
9045 line = "exited with none and no usable output".to_owned();
9046 }
9047 let mut out = Vec::new();
9048 for word in line.split_whitespace() {
9049 if word.contains('/') || word.contains('\\') {
9050 out.push("<path>".to_owned());
9051 continue;
9052 }
9053 let mut w = String::new();
9054 let mut in_digits = false;
9055 for c in word.chars() {
9056 if c.is_ascii_digit() {
9057 if !in_digits {
9058 w.push('#');
9059 }
9060 in_digits = true;
9061 } else {
9062 in_digits = false;
9063 w.push(c);
9064 }
9065 }
9066 out.push(w);
9067 }
9068 out.join(" ").chars().take(120).collect()
9069}
9070
9071fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9078 match cur {
9079 FailClass::Quota | FailClass::Timeout => true,
9080 FailClass::Other(_) => prev != Some(cur),
9081 }
9082}
9083
9084fn fail_reason(out: &AgentOutcome) -> String {
9086 match out {
9087 AgentOutcome::Ok(_) => String::new(),
9088 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9089 AgentOutcome::Dropped(o) => format!(
9090 "the CLI dropped the stream ({})",
9091 o.dropped
9092 .as_ref()
9093 .map(|d| d.why.as_str())
9094 .unwrap_or("it ended without delivering its answer")
9095 ),
9096 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9097 e.lines().next().unwrap_or("").chars().take(160).collect()
9098 }
9099 }
9100}
9101
9102pub(crate) fn record_handover(
9105 state: &mut RunState,
9106 node: &str,
9107 seat: &str,
9108 from: &str,
9109 to: &str,
9110 class: &FailClass,
9111 reason: &str,
9112) {
9113 let message = if *class == FailClass::Quota {
9114 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9115 } else {
9116 format!("{seat}: handed over {from} -> {to} ({reason})")
9117 };
9118 state.event(node, message);
9119 state.handovers.push(Handover {
9120 at: Timestamp::now(),
9121 node: node.to_owned(),
9122 seat: seat.to_owned(),
9123 from: from.to_owned(),
9124 to: to.to_owned(),
9125 reason: reason.to_owned(),
9126 });
9127}
9128
9129fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9143 commands.iter().any(|c| c.exit_code.is_none())
9144}
9145
9146fn verified_noop_claim(
9159 usable: bool,
9160 commands: &[agent::CommandEvidence],
9161 text: &str,
9162) -> Option<String> {
9163 (usable && !has_unconfirmed_command(commands))
9164 .then(|| verdict::verified_noop(text))
9165 .flatten()
9166}
9167
9168fn short(commit: &str) -> String {
9169 commit.chars().take(7).collect()
9170}
9171
9172fn make_executable(path: &Path) -> Result<()> {
9173 #[cfg(unix)]
9174 {
9175 use std::os::unix::fs::PermissionsExt as _;
9176 let mut perms = std::fs::metadata(path)?.permissions();
9177 perms.set_mode(0o755);
9178 std::fs::set_permissions(path, perms)?;
9179 }
9180 #[cfg(not(unix))]
9181 {
9182 let _ = path;
9183 }
9184 Ok(())
9185}
9186
9187struct WaveCtx<'a> {
9194 run: &'a str,
9197 node: &'a str,
9199 prompts: &'a Prompts,
9200 cache: Option<&'a Path>,
9202 round: Option<usize>,
9205 carry_seats: bool,
9209 pauses: Option<(&'a Pause, &'a Pause)>,
9212}
9213
9214fn repository_wait(
9216 state: &RunState,
9217 pauses: Option<(&Pause, &Pause)>,
9218) -> crate::repo_lock::WaitPolicy {
9219 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9220 let inherited = crate::repo_lock::WaitPolicy::current();
9221 crate::repo_lock::WaitPolicy::new(
9222 Duration::from_secs(state.config.graph.timeout_resource_wait),
9223 move || {
9224 inherited.cancelled()
9225 || pauses
9226 .as_ref()
9227 .is_some_and(|(a, b)| a.parked() || b.parked())
9228 },
9229 )
9230}
9231
9232fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9233 if error.is::<crate::repo_lock::ResourceBlocked>() {
9234 state.resource_block = Some(crate::run::ResourceBlock {
9235 node: "repository".into(),
9236 kind: if error.to_string().contains("disk pressure") {
9237 "disk pressure"
9238 } else {
9239 "repository coordination"
9240 }
9241 .into(),
9242 detail: error.to_string(),
9243 at: Timestamp::now(),
9244 });
9245 state.event("resource", error.to_string());
9246 }
9247}
9248
9249async fn coordinate_operation(
9250 state: &mut RunState,
9251 cwd: &Path,
9252 wait: &crate::repo_lock::WaitPolicy,
9253) -> Result<Option<crate::repo_lock::SeatLease>> {
9254 let result =
9255 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9256 .await;
9257 if let Err(e) = &result {
9258 record_repository_block(state, e);
9259 }
9260 result
9261}
9262
9263async fn run_one(
9265 job: SeatJob,
9266 sem: Arc<Semaphore>,
9267 ctx: &WaveCtx<'_>,
9268 state: &mut RunState,
9269 attempt: usize,
9270) -> (SeatState, AgentOutcome) {
9271 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9272 .await
9273 .pop()
9274 .expect("one job in, one result out");
9275 (seat, out)
9276}
9277
9278async fn wave(
9284 jobs: Vec<SeatJob>,
9285 sem: Arc<Semaphore>,
9286 ctx: &WaveCtx<'_>,
9287 state: &mut RunState,
9288 attempt: usize,
9289) -> Vec<(usize, SeatState, AgentOutcome)> {
9290 let WaveCtx {
9291 run,
9292 node,
9293 prompts,
9294 cache,
9295 round,
9296 carry_seats: _,
9297 pauses,
9298 } = *ctx;
9299 for job in &jobs {
9300 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9301 }
9302 if let Err(e) = state.save() {
9303 tracing::warn!("could not persist in-progress seats: {e:#}");
9308 }
9309 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9325 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9329 let cache_guard = if let Some(cache_dir) = cache {
9330 if jobs_had_a_writer {
9331 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9332 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9333 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9334 {
9335 Ok(Some(g)) => Some(g),
9336 Ok(None) => {
9339 state.resource_block = Some(crate::run::ResourceBlock {
9340 node: node.to_owned(),
9341 kind: "shared build cache".to_owned(),
9342 detail: "the run is parking".to_owned(),
9343 at: Timestamp::now(),
9344 });
9345 return block_wave(
9346 jobs,
9347 state,
9348 node,
9349 attempt,
9350 "parked while waiting for the shared build cache",
9351 );
9352 }
9353 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9356 Err(e) => {
9357 let why = format!("{e:#}");
9361 state.resource_block = Some(crate::run::ResourceBlock {
9362 node: node.to_owned(),
9363 kind: "shared build cache".to_owned(),
9364 detail: why.clone(),
9365 at: Timestamp::now(),
9366 });
9367 state.event(
9368 "resource",
9369 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9370 );
9371 return block_wave(jobs, state, node, attempt, &why);
9372 }
9373 }
9374 } else {
9375 None
9376 }
9377 } else {
9378 None
9379 };
9380 state.seats_spawning(node, attempt);
9383 if let Err(e) = state.save() {
9384 tracing::warn!("could not persist the start of a wave: {e:#}");
9385 }
9386 let slot_limit = resource_limit
9390 + jobs
9391 .iter()
9392 .map(|j| j.timeout)
9393 .max()
9394 .unwrap_or_default()
9395 .saturating_mul(jobs.len() as u32);
9396 let mut set = tokio::task::JoinSet::new();
9397 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9400 let overlay = prompts.overlay(node);
9401 let repository_wait = repository_wait(state, pauses);
9402 for (i, mut job) in jobs.into_iter().enumerate() {
9403 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9404 if cache.is_some() {
9405 job.prompt.push('\n');
9406 job.prompt
9407 .push_str(&prompt::build_cache_note(node, job.allow_write));
9408 }
9409 let sem = Arc::clone(&sem);
9410 let run = run.to_owned();
9411 let node = node.to_owned();
9412 let attachments = if node == "implement" {
9415 state.attachments.clone()
9416 } else {
9417 Vec::new()
9418 };
9419 let cache = cache
9430 .filter(|_| job.allow_write && cache_guard.is_some())
9431 .map(Path::to_path_buf);
9432 let spawned_tx = spawned_tx.clone();
9433 let repository_wait = repository_wait.clone();
9434 let coordination = state.config.coordination.clone();
9435 let fetched_base = state.fetched_base.clone();
9436 set.spawn(async move {
9437 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9441 Ok(p) => p,
9442 Err(_) => {
9443 return (
9444 i,
9445 job.seat,
9446 AgentOutcome::ResourceBlocked(format!(
9447 "no parallel slot became free within {}s",
9448 slot_limit.as_secs()
9449 )),
9450 false,
9451 );
9452 }
9453 };
9454 spawned_tx.send(job.seat.key.clone()).ok();
9455 let mut seat = job.seat;
9456 let out = repository_wait
9457 .scope(agent::invoke(
9458 &job.spec,
9459 &mut seat,
9460 &Invocation {
9461 scope: crate::agent::InvocationScope::Repository,
9462 coordination: coordination.as_ref(),
9463 fetched_base: fetched_base.as_ref(),
9464 cwd: &job.cwd,
9465 prompt: &job.prompt,
9466 timeout: job.timeout,
9467 allow_write: job.allow_write,
9468 unsandboxed: false,
9469 sessions: job.sessions,
9470 artifacts: &job.artifacts,
9471 stem: &job.stem,
9472 run: &run,
9473 node: &node,
9474 cache_dir: cache.as_deref(),
9475 attachments: &attachments,
9476 writable: &[],
9477 },
9478 ))
9479 .await;
9480 let interrupted = out
9481 .as_ref()
9482 .err()
9483 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9484 let out = match out {
9485 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9486 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9487 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9495 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9496 Ok(o) => AgentOutcome::Failed(format!(
9497 "process exited with no usable output ({})",
9498 crate::notices::exit_status(o.exit_code)
9499 )),
9500 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9501 AgentOutcome::ResourceBlocked(e.to_string())
9502 }
9503 Err(e) => AgentOutcome::Failed(e.to_string()),
9504 };
9505 (i, seat, out, interrupted)
9506 });
9507 }
9508 drop(spawned_tx);
9509 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9510 loop {
9511 let joined = tokio::select! {
9512 Some(key) = spawned_rx.recv() => {
9513 state.seat_spawned(&key);
9515 if let Err(e) = state.save() {
9516 tracing::warn!("could not persist a seat's start: {e:#}");
9517 }
9518 continue;
9519 }
9520 joined = set.join_next() => joined,
9521 };
9522 let Some(joined) = joined else {
9523 break;
9524 };
9525 let (i, seat, out, interrupted) = match joined {
9526 Ok(v) => v,
9527 Err(e) => {
9531 tracing::error!("agent task panicked: {e}");
9532 continue;
9533 }
9534 };
9535 state.seat_finished(&seat.key);
9536 if interrupted
9537 && node == "implement"
9538 && let Some(label) = state
9539 .candidates
9540 .iter()
9541 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9542 .map(|candidate| candidate.label)
9543 {
9544 state.candidate_preservation.insert(
9545 label,
9546 "interrupted implementation awaiting persistence".into(),
9547 );
9548 state.interrupted_implementations.insert(label);
9549 }
9550 if let AgentOutcome::ResourceBlocked(why) = &out
9551 && state.resource_block.is_none()
9552 {
9553 state.resource_block = Some(crate::run::ResourceBlock {
9554 node: node.to_owned(),
9555 kind: if why.contains("disk pressure") {
9556 "disk pressure"
9557 } else if why.contains("repository coordination") {
9558 "repository coordination"
9559 } else {
9560 "parallel slot"
9561 }
9562 .to_owned(),
9563 detail: why.clone(),
9564 at: Timestamp::now(),
9565 });
9566 let activity = if interrupted {
9567 "agent execution interrupted"
9568 } else {
9569 "no agent was started"
9570 };
9571 state.event("resource", format!("{node}: {activity} - {why}"));
9572 }
9573 record_jobs(state, node, round, &seat.key, &out);
9574 if let Err(e) = state.save() {
9575 tracing::warn!("could not persist a seat's completion: {e:#}");
9576 }
9577 if collected.len() <= i {
9578 collected.resize_with(i + 1, || None);
9579 }
9580 collected[i] = Some((i, seat, out));
9581 }
9582 if state
9588 .active
9589 .values()
9590 .any(|a| a.node == node && a.attempt == attempt)
9591 {
9592 state
9593 .active
9594 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9595 if let Err(e) = state.save() {
9596 tracing::warn!("could not persist the end of a wave: {e:#}");
9597 }
9598 }
9599 if let Some(cache_dir) = cache
9606 && jobs_had_a_writer
9607 {
9608 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9609 }
9610 if let Some(guard) = cache_guard {
9611 guard.release();
9612 }
9613 collected.into_iter().flatten().collect()
9614}
9615
9616fn record_jobs(
9627 state: &mut RunState,
9628 node: &str,
9629 round: Option<usize>,
9630 seat: &str,
9631 out: &AgentOutcome,
9632) {
9633 let commands: &[agent::CommandEvidence] = match out {
9634 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9635 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9636 };
9637 let checked_at = Timestamp::now();
9638 for c in commands {
9639 state.jobs.push(JobRecord {
9640 node: node.to_owned(),
9641 round,
9642 seat: seat.to_owned(),
9643 id: c.id.clone(),
9644 description: c.description.clone(),
9645 checked_at,
9646 status: match c.exit_code {
9647 Some(0) => JobStatus::Completed,
9648 Some(_) => JobStatus::Failed,
9649 None => JobStatus::Unknown,
9650 },
9651 exit_code: c.exit_code,
9652 result_summary: c.result_summary.clone(),
9653 source: c.source.clone(),
9654 });
9655 }
9656}
9657
9658fn round_is_clean(
9679 blocking: usize,
9680 e2e_ok: bool,
9681 answered: usize,
9682 expected: usize,
9683 quota_missing: usize,
9684 policy: IncompleteReviewPolicy,
9685) -> bool {
9686 if blocking != 0 || !e2e_ok {
9687 return false;
9688 }
9689 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9690 return true;
9691 }
9692 answered > 0 && expected - answered <= quota_missing
9693}
9694
9695fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9719 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9720 return Some(RunStatus::Gating);
9721 }
9722 let last = reviews.last()?;
9723 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9724 if reviews.len() < max_rounds && !stagnant {
9725 return None;
9726 }
9727 if last.incomplete() && last.blocking == 0 {
9728 return Some(RunStatus::Blocked);
9729 }
9730 if last.e2e_status() == E2eStatus::ResourceBlocked {
9731 return None;
9732 }
9733 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9734 RunStatus::Gating
9735 } else {
9736 RunStatus::Blocked
9737 })
9738}
9739
9740pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9755 if nudged {
9756 (full / 4).max(Duration::from_secs(120)).min(full)
9757 } else {
9758 full
9759 }
9760}
9761
9762#[allow(clippy::too_many_arguments)]
9791async fn ask_json_wave<T>(
9792 jobs: Vec<SeatJob>,
9793 sem: Arc<Semaphore>,
9794 retries: usize,
9795 roster: &[AgentSpec],
9796 ctx: &WaveCtx<'_>,
9797 losses: &mut Vec<QuotaLoss>,
9798 state: &mut RunState,
9799 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9800) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9801where
9802 T: serde::de::DeserializeOwned + Send + 'static,
9803{
9804 ask_wave_with(
9805 jobs,
9806 sem,
9807 retries,
9808 roster,
9809 ctx,
9810 losses,
9811 state,
9812 &|text: &str| {
9813 let v = verdict::extract_json::<T>(text)?;
9814 validate(&v)?;
9815 Ok(v)
9816 },
9817 )
9818 .await
9819}
9820
9821#[allow(clippy::too_many_arguments)]
9830async fn ask_wave_with<T>(
9831 jobs: Vec<SeatJob>,
9832 sem: Arc<Semaphore>,
9833 retries: usize,
9834 roster: &[AgentSpec],
9835 ctx: &WaveCtx<'_>,
9836 losses: &mut Vec<QuotaLoss>,
9837 state: &mut RunState,
9838 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9839) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9840where
9841 T: Send + 'static,
9842{
9843 let n = jobs.len();
9844 let originals: Vec<SeatJob> = jobs;
9845 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9846 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9847 let mut nudges: Vec<usize> = vec![0; n];
9854 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9858 let mut tried: Vec<BTreeSet<String>> = specs
9859 .iter()
9860 .map(|s| BTreeSet::from([s.id.clone()]))
9861 .collect();
9862 let starts: Vec<usize> = specs
9863 .iter()
9864 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9865 .collect();
9866 let carry = ctx.carry_seats && !roster.is_empty();
9867 let carried: Vec<BTreeSet<String>> = originals
9870 .iter()
9871 .map(|j| {
9872 state
9873 .seat_history
9874 .get(&j.seat.key)
9875 .filter(|_| carry)
9876 .map(|h| h.failed.clone())
9877 .unwrap_or_default()
9878 })
9879 .collect();
9880 let mut prev: Vec<Option<FailClass>> = originals
9881 .iter()
9882 .map(|j| {
9883 state
9884 .seat_history
9885 .get(&j.seat.key)
9886 .filter(|_| carry)
9887 .and_then(|h| h.last_fail.clone())
9888 })
9889 .collect();
9890 let next_agent =
9891 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9892 let others: BTreeSet<String> = specs
9893 .iter()
9894 .enumerate()
9895 .filter(|(j, _)| *j != i)
9896 .map(|(_, s)| s.id.clone())
9897 .collect();
9898 pick_successor(
9899 roster,
9900 starts[i],
9901 tried,
9902 carry.then(|| &carried[i]),
9903 &others,
9904 )
9905 .cloned()
9906 };
9907 let mut fresh: Vec<Option<String>> = vec![None; n];
9908 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9909 let mut pending: Vec<usize> = (0..n).collect();
9910
9911 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9916 for round in 0..max_rounds {
9917 if pending.is_empty() {
9918 break;
9919 }
9920 let mut batch = Vec::with_capacity(pending.len());
9921 let mut renudged: Vec<&str> = Vec::new();
9922 for &i in &pending {
9923 let src = &originals[i];
9924 let full: &str = match &src.handover {
9929 Some(h) if specs[i].id != src.spec.id => h,
9930 _ => &src.prompt,
9931 };
9932 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9935 (
9936 full.to_owned(),
9937 src.timeout,
9938 format!("{}-{word}-{}", src.stem, specs[i].id),
9939 )
9940 } else if nudges[i] == 0 {
9941 (full.to_owned(), src.timeout, src.stem.clone())
9942 } else {
9943 renudged.push(src.seat.key.as_str());
9944 let why = done[i]
9945 .as_ref()
9946 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9947 .unwrap_or_else(|| "no parsable answer".to_owned());
9948 let nudge = prompt::nudge(&why);
9949 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9950 let prompt = if nudged {
9951 nudge
9952 } else {
9953 format!("{full}\n\n---\n\n{nudge}")
9954 };
9955 (
9956 prompt,
9957 retry_budget(src.timeout, nudged),
9958 format!("{}-retry{}", src.stem, nudges[i]),
9959 )
9960 };
9961 batch.push(SeatJob {
9962 spec: specs[i].clone(),
9963 seat: seats[i].clone(),
9964 cwd: src.cwd.clone(),
9965 prompt,
9966 timeout,
9967 allow_write: src.allow_write,
9968 sessions: src.sessions,
9969 artifacts: src.artifacts.clone(),
9970 stem,
9971 handover: None,
9972 });
9973 }
9974
9975 if !renudged.is_empty() {
9976 state.event(
9977 ctx.node,
9978 format!("retry {round}: re-asking {}", renudged.join(", ")),
9979 );
9980 }
9981 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9982 let mut still = Vec::new();
9983 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9984 seats[i] = seat;
9985 if let AgentOutcome::ResourceBlocked(why) = &out {
9988 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9989 continue;
9990 }
9991 let class = FailClass::of(&out);
9992 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9995 && nudges[i] < retries
9996 && !roster.is_empty();
9997 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9998 let next = should_hand_over(prev[i].as_ref(), &cur)
9999 .then(|| next_agent(i, &tried[i], &specs))
10000 .flatten();
10001 if carry {
10002 let h = state
10003 .seat_history
10004 .entry(originals[i].seat.key.clone())
10005 .or_default();
10006 h.failed.insert(specs[i].id.clone());
10007 h.last_fail = Some(cur.clone());
10008 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
10009 h.last_ok = None;
10010 }
10011 if let Err(e) = state.save() {
10014 tracing::warn!("could not persist a seat's failure history: {e:#}");
10015 }
10016 }
10017 if let Some(next) = next {
10018 record_handover(
10019 state,
10020 ctx.node,
10021 &originals[i].seat.key,
10022 &specs[i].id,
10023 &next.id,
10024 &cur,
10025 &fail_reason(&out),
10026 );
10027 tried[i].insert(next.id.clone());
10028 prev[i] = Some(cur.clone());
10029 seats[i] =
10030 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10031 specs[i] = next;
10032 fresh[i] = Some(cur.stem_word().to_owned());
10033 nudges[i] = 0;
10034 done[i] = Some(Err(anyhow::anyhow!(
10035 "handed over after: {}",
10036 fail_reason(&out)
10037 )));
10038 still.push(i);
10039 continue;
10040 }
10041 }
10042 if carry
10043 && !nudge_first
10044 && let Some(cur) = class.clone()
10045 {
10046 let h = state
10048 .seat_history
10049 .entry(originals[i].seat.key.clone())
10050 .or_default();
10051 h.failed.insert(specs[i].id.clone());
10052 h.last_fail = Some(cur);
10053 }
10054 let parsed = match out {
10055 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10056 AgentOutcome::Quota(o) => {
10057 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10058 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10059 }
10060 AgentOutcome::Dropped(o) => {
10064 let why = o
10065 .dropped
10066 .as_ref()
10067 .map(|d| d.why.as_str())
10068 .unwrap_or("the CLI ended the stream without delivering its answer");
10069 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10070 }
10071 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10072 Err(anyhow::anyhow!(e))
10073 }
10074 };
10075 let quota = class == Some(FailClass::Quota);
10076 let failed = parsed.is_err();
10077 done[i] = Some(parsed);
10078 if carry && !failed {
10079 let h = state
10080 .seat_history
10081 .entry(originals[i].seat.key.clone())
10082 .or_default();
10083 h.failed.remove(&specs[i].id);
10084 h.last_ok = Some(specs[i].id.clone());
10085 h.last_fail = None;
10086 }
10087 let agent_failure = class.is_some()
10095 && !nudge_first
10096 && !roster.is_empty()
10097 && next_agent(i, &tried[i], &specs).is_some();
10098 if failed && !quota && !agent_failure && nudges[i] < retries {
10099 nudges[i] += 1;
10100 still.push(i);
10101 }
10102 }
10103 pending = still;
10104 }
10105
10106 for (i, q) in last_quota.into_iter().enumerate() {
10109 if let Some(reset) = q {
10110 losses.push(QuotaLoss {
10111 seat: originals[i].seat.key.clone(),
10112 node: ctx.node.to_owned(),
10113 at: Timestamp::now(),
10114 reset,
10115 });
10116 }
10117 }
10118
10119 seats
10120 .into_iter()
10121 .zip(done)
10122 .zip(nudges)
10123 .map(|((seat, res), attempts)| {
10124 (
10125 seat,
10126 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10127 attempts,
10128 )
10129 })
10130 .collect()
10131}
10132
10133fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10135 results
10136 .iter()
10137 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10138}
10139
10140fn block_wave(
10144 jobs: Vec<SeatJob>,
10145 state: &mut RunState,
10146 node: &str,
10147 attempt: usize,
10148 why: &str,
10149) -> Vec<(usize, SeatState, AgentOutcome)> {
10150 state.resource_wait = None;
10151 state
10152 .active
10153 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10154 if let Err(e) = state.save() {
10155 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10156 }
10157 jobs.into_iter()
10158 .enumerate()
10159 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10160 .collect()
10161}
10162
10163async fn acquire_cache_lease(
10176 state: &mut RunState,
10177 cache_dir: &Path,
10178 owner: &crate::cache::Owner,
10179 budget: Duration,
10180 context: &str,
10181 cancel: &(dyn Fn() -> bool + Sync),
10182) -> Result<Option<crate::cache::Guard>> {
10183 let home = crate::run::home();
10184 let started = Instant::now();
10185 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10186 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10187 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10188 Err(e) => {
10189 state.event(
10190 "verify",
10191 format!("{context}: could not check the shared build cache: {e:#}"),
10192 );
10193 if let Err(e2) = state.save() {
10194 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10195 }
10196 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10197 }
10198 };
10199 state.event(
10200 "verify",
10201 format!(
10202 "{context}: waiting for the shared build cache at {} ({})",
10203 cache_dir.display(),
10204 busy.describe()
10205 ),
10206 );
10207 state.resource_wait = Some(crate::run::ResourceWait {
10210 node: context.to_owned(),
10211 kind: "shared build cache".to_owned(),
10212 holder: busy.describe(),
10213 since: Timestamp::now(),
10214 limit_secs: budget.as_secs(),
10215 });
10216 if let Err(e) = state.save() {
10217 tracing::warn!("could not persist a cache wait: {e:#}");
10218 }
10219 let remaining = budget.saturating_sub(started.elapsed());
10220 let waited = crate::cache::wait_for_unless(
10221 &home,
10222 cache_dir,
10223 owner,
10224 remaining,
10225 Duration::from_secs(5),
10226 cancel,
10227 )
10228 .await;
10229 state.resource_wait = None;
10230 match waited {
10231 Ok(g) => Ok(g),
10232 Err(e) => {
10233 state.event("verify", format!("{context}: {e:#}"));
10234 if let Err(e2) = state.save() {
10235 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10236 }
10237 Err(e)
10238 }
10239 }
10240}
10241
10242#[derive(Debug)]
10247struct CacheCheckError(String);
10248
10249impl std::fmt::Display for CacheCheckError {
10250 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10251 f.write_str(&self.0)
10252 }
10253}
10254
10255impl std::error::Error for CacheCheckError {}
10256
10257#[allow(clippy::too_many_arguments)]
10278async fn with_cache_lease<'s, F, Fut>(
10279 state: &'s mut RunState,
10280 cache_dir: Option<&Path>,
10281 node: &str,
10282 seat: &str,
10283 worktree: &Path,
10284 head: &str,
10285 budget: Duration,
10286 context: &str,
10287 wait: &crate::repo_lock::WaitPolicy,
10288 body: F,
10289) -> Result<(Vec<CommandOutcome>, bool)>
10290where
10291 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10292 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10293{
10294 let started = Instant::now();
10297 let home = crate::run::home();
10298 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10299 let guard = if let Some(cache_dir) = cache_dir {
10300 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10301 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10302 wait.cancelled()
10303 })
10304 .await
10305 .and_then(|g| {
10306 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10307 }) {
10308 Ok(g) => Some(g),
10309 Err(e) => {
10310 return Ok((
10311 vec![CommandOutcome {
10312 command: "(waiting for the shared build cache)".into(),
10313 code: None,
10314 output_tail: e.to_string(),
10315 duration_ms: started.elapsed().as_millis() as u64,
10316 resource_blocked: true,
10317 }],
10318 false,
10319 ));
10320 }
10321 }
10322 } else {
10323 None
10324 };
10325 let _coordination = coordinate_operation(state, worktree, wait).await?;
10326 if let Some(cache_dir) = cache_dir {
10327 let identity = crate::cache::Identity::new(worktree, head);
10328 if let Err(e) = crate::cache::ensure_fresh(
10329 &home,
10330 cache_dir,
10331 &identity,
10332 &state.config.verify.cargo_manifests(),
10333 ) {
10334 state.event(
10335 "verify",
10336 format!(
10337 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10338 worktree.display(),
10339 short(head)
10340 ),
10341 );
10342 if let Some(guard) = guard {
10343 guard.release();
10344 }
10345 return Ok((
10346 vec![CommandOutcome {
10347 command: "(confirming the shared build cache is fresh)".into(),
10348 code: None,
10349 output_tail: e.to_string(),
10350 duration_ms: started.elapsed().as_millis() as u64,
10351 resource_blocked: true,
10352 }],
10353 false,
10354 ));
10355 }
10356 }
10357 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10358 if !timed_out_pids.is_empty() {
10359 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10360 }
10361 drop(_coordination);
10364 if let Some(guard) = guard {
10365 guard.release();
10366 }
10367 Ok((outcomes, retried))
10368}
10369
10370async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10382 wait_for_pids_with(
10383 pids,
10384 crate::proc::pid_alive,
10385 LEASE_RELEASE_POLL,
10386 LEASE_RELEASE_MAX_WAIT,
10387 )
10388 .await;
10389}
10390
10391async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10397 pids: &[u32],
10398 alive: F,
10399 poll: Duration,
10400 max_wait: Duration,
10401) {
10402 let deadline = Instant::now() + max_wait;
10403 loop {
10404 if pids.iter().all(|&pid| !alive(pid)) {
10405 return;
10406 }
10407 if Instant::now() >= deadline {
10408 return;
10409 }
10410 tokio::time::sleep(poll).await;
10411 }
10412}
10413
10414fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10421 outcomes.iter().any(|o| o.resource_blocked)
10422}
10423
10424enum GateFix {
10426 Retry,
10428 Stop,
10431 Defer,
10434}
10435
10436fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10444 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10445 red.peek().is_some()
10446 && red.all(|o| {
10447 !o.resource_blocked
10448 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10449 && !o.output_tail.trim().is_empty()
10450 })
10451}
10452
10453fn e2e_outcome_label(o: &CommandOutcome) -> String {
10457 if o.ok() {
10458 return "pass".to_owned();
10459 }
10460 let reason = if o.build_failed() {
10461 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10462 } else {
10463 format!("FAIL ({:?})", o.code)
10464 };
10465 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10466}
10467
10468async fn run_e2e_with_retry(
10476 state: &mut RunState,
10477 shell: &[String],
10478 commands: &[String],
10479 worktree: &Path,
10480 timeout: Duration,
10481 context: &str,
10482) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10483 let (mut e2e, mut timed_out_pids) = run_commands(
10484 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10485 )
10486 .await;
10487 for o in &e2e {
10488 state.event(
10489 "verify",
10490 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10491 );
10492 }
10493 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10497 if verify_retried {
10498 state.event(
10499 "verify",
10500 format!(
10501 "{context}: verify could not build/link, not a test result — retrying once \
10502 before concluding"
10503 ),
10504 );
10505 let retried = run_commands(
10506 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10507 )
10508 .await;
10509 e2e = retried.0;
10510 timed_out_pids.extend(retried.1);
10513 for o in &e2e {
10514 state.event(
10515 "verify",
10516 format!(
10517 "{context}: retry `{}` -> {}",
10518 o.command,
10519 e2e_outcome_label(o)
10520 ),
10521 );
10522 }
10523 }
10524 (e2e, verify_retried, timed_out_pids)
10525}
10526
10527#[allow(clippy::too_many_arguments)]
10543async fn run_commands(
10544 state: &mut RunState,
10545 node: &str,
10546 task: &str,
10547 attempt: usize,
10548 shell: &[String],
10549 commands: &[String],
10550 cwd: &Path,
10551 timeout: Duration,
10552) -> (Vec<CommandOutcome>, Vec<u32>) {
10553 if commands.is_empty() {
10554 return (Vec::new(), Vec::new());
10559 }
10560 let mut out = Vec::new();
10561 let mut timed_out_pids = Vec::new();
10562 let total = commands.len();
10563 for (idx, command) in commands.iter().enumerate() {
10564 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10565 if let Err(e) = state.save() {
10566 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10567 }
10568 let started = Instant::now();
10569 let mut cmd = tokio::process::Command::new(&shell[0]);
10570 cmd.quiet();
10571 cmd.args(&shell[1..])
10572 .arg(command)
10573 .current_dir(cwd)
10574 .stdin(std::process::Stdio::null())
10575 .stdout(std::process::Stdio::piped())
10576 .stderr(std::process::Stdio::piped())
10577 .kill_on_drop(true);
10578 let mut outputs = Vec::new();
10579 let protected = (|| -> Result<()> {
10580 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10581 outputs.push(guard);
10582 }
10583 if let Some(cache) = state.config.cache_dir() {
10584 outputs.push(crate::retention::OutputLease::cache(
10585 &crate::run::home(),
10586 &cache,
10587 false,
10588 )?);
10589 }
10590 Ok(())
10591 })();
10592 if let Err(e) = protected {
10593 out.push(CommandOutcome {
10594 command: command.clone(),
10595 code: None,
10596 output_tail: format!("output ownership unavailable: {e:#}"),
10597 duration_ms: 0,
10598 resource_blocked: true,
10599 });
10600 continue;
10601 }
10602 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10603 let (code, body) = match spawned {
10604 Ok(mut running) => {
10605 let pid = running.child().id();
10606 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10607 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10608 let mut paths = vec![cwd.to_owned()];
10609 if let Some(cache) = state.config.cache_dir() {
10610 paths.push(cache);
10611 }
10612 let mut pressure = None;
10613 let status = tokio::select! {
10614 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10615 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10616 pressure = Some(reason); None
10617 }
10618 };
10619 let (code, failure) = match status {
10620 Some(Ok(Ok(status))) => (
10621 status.code(),
10622 status
10623 .code()
10624 .is_none()
10625 .then(|| format!("command terminated: {status}")),
10626 ),
10627 Some(Ok(Err(error))) => {
10628 (None, Some(format!("failed waiting for command: {error}")))
10629 }
10630 Some(Err(_)) => {
10631 if !running.coordinated()
10635 && let Some(pid) = pid
10636 {
10637 timed_out_pids.push(pid);
10638 }
10639 (
10640 None,
10641 Some(format!("timed out after {}s", timeout.as_secs())),
10642 )
10643 }
10644 None => (None, None),
10645 };
10646 running.finish().await;
10647 if let Some(reason) = pressure {
10648 state.resource_block = Some(crate::run::ResourceBlock {
10649 node: node.to_owned(),
10650 kind: "disk pressure".to_owned(),
10651 detail: reason.clone(),
10652 at: Timestamp::now(),
10653 });
10654 out.push(CommandOutcome {
10655 command: command.clone(),
10656 code: None,
10657 output_tail: reason,
10658 duration_ms: started.elapsed().as_millis() as u64,
10659 resource_blocked: true,
10660 });
10661 break;
10662 }
10663 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10664 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10665 if let Some(failure) = failure {
10666 body.push('\n');
10667 body.push_str(&failure);
10668 }
10669 (code, body)
10670 }
10671 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10672 };
10673 out.push(CommandOutcome {
10674 command: command.clone(),
10675 code,
10676 output_tail: tail(&body, OUTPUT_TAIL),
10677 duration_ms: started.elapsed().as_millis() as u64,
10678 resource_blocked: false,
10679 });
10680 }
10681 state.task_finished(task);
10682 if let Err(e) = state.save() {
10683 tracing::warn!("could not persist the end of {task}: {e:#}");
10684 }
10685 (out, timed_out_pids)
10686}
10687
10688fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10700 let repo = repo.display();
10701 match style {
10702 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10703 MergeStyle::Squash => {
10704 let subject = message
10707 .lines()
10708 .next()
10709 .unwrap_or(branch)
10710 .replace(['\\', '"', '$', '`'], "");
10711 format!(
10712 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10713 )
10714 }
10715 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10716 }
10717}
10718
10719const PR_TITLE_MAX: usize = 240;
10730
10731fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10742 let line = queue::first_line(text)?;
10743 let chars: Vec<char> = line.chars().collect();
10744 let (kept, mark) = if chars.len() <= max {
10745 (line.to_owned(), "")
10746 } else if let Some(end) = sentence_end(&chars, max) {
10747 (chars[..end].iter().collect::<String>(), "")
10748 } else {
10749 let room = max.saturating_sub(3);
10750 let cut = if chars[room].is_whitespace() {
10753 room
10754 } else {
10755 chars[..room]
10756 .iter()
10757 .rposition(|c| c.is_whitespace())
10758 .unwrap_or(room)
10759 };
10760 let head: String = chars[..cut].iter().collect();
10761 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10762 (head.to_owned(), "...")
10763 };
10764 if kept.is_empty() {
10765 return None;
10766 }
10767 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10768 return None;
10769 }
10770 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10771 return None;
10772 }
10773 Some(format!("{kept}{mark}"))
10774}
10775
10776fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10780 const MIN: usize = 20;
10781 for i in MIN..max.min(chars.len()) {
10782 if !matches!(chars[i], '.' | '!' | '?') {
10783 continue;
10784 }
10785 let Some(&next) = chars.get(i + 1) else {
10786 continue;
10787 };
10788 if !next.is_whitespace() {
10789 continue;
10790 }
10791 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10792 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10793 continue;
10794 }
10795 let word: String = chars[..i]
10796 .iter()
10797 .rev()
10798 .take_while(|c| !c.is_whitespace())
10799 .collect::<Vec<_>>()
10800 .into_iter()
10801 .rev()
10802 .collect();
10803 let word = word.to_ascii_lowercase();
10804 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10805 continue;
10806 }
10807 let end = chars[..i]
10808 .iter()
10809 .rposition(|c| !c.is_whitespace())
10810 .map_or(i, |p| p + 1);
10811 return Some(end);
10812 }
10813 None
10814}
10815
10816struct PrMessage {
10821 title: String,
10822 body: String,
10823}
10824
10825impl PrMessage {
10826 fn commit_message(&self) -> String {
10830 format!("{}\n\n{}", self.title, self.body)
10831 }
10832}
10833
10834fn title_marker(line: &str) -> Option<&str> {
10836 let line = line.trim();
10837 let head = line.get(..6)?;
10838 head.eq_ignore_ascii_case("title:")
10839 .then(|| line[6..].trim())
10840}
10841
10842fn summary_title(summary: &str) -> Option<String> {
10847 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10848 let raw = title_marker(first)?;
10849 if raw.is_empty() {
10850 return None;
10851 }
10852 let title = queue::title_from(raw, PR_TITLE_MAX);
10853 let lower = title.to_ascii_lowercase();
10854 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10855 return None;
10856 }
10857 Some(title)
10858}
10859
10860const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10862
10863pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10865
10866fn is_review_run(state: &RunState) -> bool {
10870 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10871}
10872
10873fn review_title(state: &RunState) -> Option<String> {
10878 english_subject(
10879 state.reviewed_commits.as_ref()?.first()?,
10880 state.config.graph.github_english(),
10881 )
10882}
10883
10884fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10887 let raw = raw.trim();
10888 if raw.is_empty() {
10889 return None;
10890 }
10891 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10892 return None;
10893 }
10894 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10895 return None;
10896 }
10897 let title = queue::title_from(raw, PR_TITLE_MAX);
10898 let lower = title.to_ascii_lowercase();
10899 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10900 return None;
10901 }
10902 Some(title)
10903}
10904
10905#[derive(Debug, Clone, PartialEq, Eq)]
10908struct BranchFacts {
10909 commits: Vec<(String, String)>,
10911 stat: String,
10913}
10914
10915const STAT_FILE_LINES: usize = 25;
10917const COMMITS_MAX_CHARS: usize = 20_000;
10919
10920async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10923 let commits = git::commit_log(repo, base, branch).await.ok()?;
10924 if commits.is_empty() {
10925 return None;
10926 }
10927 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10928 let lines: Vec<&str> = stat.lines().collect();
10929 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10930 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10931 let more = format!(" ... {omitted} more file(s)");
10932 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10933 kept.push(&more);
10934 kept.push(lines[lines.len() - 1]);
10935 kept.join("\n")
10936 } else {
10937 lines.join("\n")
10938 };
10939 Some(BranchFacts { commits, stat })
10940}
10941
10942fn markdown_safe(text: &str) -> String {
10945 text.replace("</details>", "</details>")
10946 .replace("\x60\x60\x60", "~~~")
10947}
10948
10949fn neutral_title(state: &RunState, winner: char) -> String {
10950 format!(
10951 "chore: land candidate {} of run {}",
10952 winner.to_ascii_uppercase(),
10953 state.id
10954 )
10955}
10956
10957pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10963 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10964 ""
10965 } else {
10966 pr_title
10967 }
10968}
10969
10970pub fn landing_subject_source(state: &RunState) -> String {
10974 if is_review_run(state) {
10975 let winner = state.candidates.first().map_or('A', |c| c.label);
10976 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10977 }
10978 state.instruction.clone()
10979}
10980
10981fn summary_without_title(summary: &str) -> String {
10984 let mut lines = summary.trim().lines().peekable();
10985 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10986 lines.next();
10987 }
10988 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10989}
10990
10991#[cfg(test)]
11003fn pr_message(state: &RunState, winner: char) -> PrMessage {
11004 pr_message_with(state, winner, None)
11005}
11006
11007#[cfg(test)]
11010fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11011 let raw = pr_message_raw(
11012 state,
11013 winner,
11014 facts,
11015 crate::pr_outcome::FinalHead::Unreported,
11016 ChangeNote::NotAsked,
11017 );
11018 let id = crate::scrub::Identity::current();
11019 PrMessage {
11020 title: crate::scrub::scrub(&raw.title, &id),
11021 body: crate::scrub::scrub(&raw.body, &id),
11022 }
11023}
11024
11025#[derive(Debug, Clone, Copy)]
11027enum ChangeNote<'a> {
11028 NotAsked,
11030 Summary(&'a crate::run::ChangeSummary),
11032 Unavailable(&'a str),
11034}
11035
11036fn pr_message_raw(
11037 state: &RunState,
11038 winner: char,
11039 facts: Option<&BranchFacts>,
11040 head: crate::pr_outcome::FinalHead<'_>,
11041 change: ChangeNote<'_>,
11042) -> PrMessage {
11043 if !matches!(change, ChangeNote::NotAsked) {
11044 let (title, mut body) = match change {
11045 ChangeNote::Summary(c) => (
11046 c.title.clone(),
11047 format!("## Summary\n\n{}\n\n", c.body.trim()),
11048 ),
11049 ChangeNote::Unavailable(why) => (
11050 neutral_title(state, winner),
11051 format!(
11052 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11053 ),
11054 ),
11055 ChangeNote::NotAsked => unreachable!(),
11056 };
11057 body.push_str(&public_pr_evidence(state, head));
11058 body.push_str(&format!(
11059 "\n---\nmagi:run/{} magi:candidate-{}\n",
11060 state.id,
11061 winner.to_ascii_lowercase()
11062 ));
11063 return PrMessage { title, body };
11064 }
11065 pr_preview(state, winner, facts, head)
11066}
11067
11068fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11071 let mut body = crate::pr_outcome::review_rounds(state);
11072 body.push_str(&crate::pr_outcome::final_verification(state, head));
11073 let open = state.last_round_findings();
11074 if !open.is_empty() {
11075 body.push_str("## Open review findings\n\n");
11076 for f in open {
11077 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11078 }
11079 body.push('\n');
11080 }
11081 if let Some(c) = state.change_summary.as_ref()
11082 && head == crate::pr_outcome::FinalHead::At(&c.head)
11083 {
11084 body.push_str(&format!(
11085 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11086 c.base, c.head
11087 ));
11088 }
11089 body
11090}
11091
11092fn pr_preview(
11094 state: &RunState,
11095 winner: char,
11096 facts: Option<&BranchFacts>,
11097 head: crate::pr_outcome::FinalHead<'_>,
11098) -> PrMessage {
11099 let summary = state
11100 .candidates
11101 .iter()
11102 .find(|c| c.label == winner)
11103 .map(|c| c.summary.as_str())
11104 .unwrap_or_default();
11105 let review = is_review_run(state);
11108 let title = if review {
11109 facts
11110 .and_then(|f| {
11111 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11112 })
11113 .or_else(|| review_title(state))
11114 .or_else(|| {
11115 state
11116 .candidates
11117 .iter()
11118 .find(|c| c.label == winner)
11119 .filter(|c| !c.branch.starts_with("magi/"))
11120 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11121 })
11122 .unwrap_or_else(|| neutral_title(state, winner))
11123 } else {
11124 summary_title(summary).unwrap_or_else(|| {
11125 english_title(
11126 &state.instruction,
11127 PR_TITLE_MAX,
11128 state.config.graph.github_english(),
11129 )
11130 .unwrap_or_else(|| neutral_title(state, winner))
11131 })
11132 };
11133
11134 let mut body = String::new();
11135 let what = summary_without_title(summary);
11136 if !what.is_empty() {
11137 body.push_str("## Summary\n\n");
11138 body.push_str(&what);
11139 body.push_str("\n\n");
11140 }
11141
11142 body.push_str(&crate::pr_outcome::review_rounds(state));
11144 body.push_str(&crate::pr_outcome::final_verification(state, head));
11145 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11146
11147 let open = state.open_findings();
11148 if !open.is_empty() {
11149 body.push_str("## Open review findings\n\n");
11150 for f in &open {
11151 body.push_str(&format!(
11152 "- `{}` [{:?}] {}\n",
11153 f.id, f.severity, "Context retained in local review records."
11154 ));
11155 }
11156 body.push('\n');
11157 }
11158
11159 if let Some(fix) = fix
11160 && !fix.rejected.is_empty()
11161 {
11162 body.push_str("## Declined by the fixer\n\n");
11163 for r in &fix.rejected {
11164 body.push_str(&format!(
11165 "- `{}`: {}\n",
11166 r.id, "Reason retained in local review records."
11167 ));
11168 }
11169 body.push('\n');
11170 }
11171
11172 if review {
11173 body.push_str("## Commits under review\n\n");
11175 if let Some(facts) = facts {
11176 let mut left = COMMITS_MAX_CHARS;
11177 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11178 let mut entry = format!("- {}\n", markdown_safe(subject));
11179 for l in markdown_safe(text).lines() {
11180 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11181 }
11182 if left == 0 {
11183 body.push_str(&format!(
11184 "- ... {} more commit(s)\n",
11185 facts.commits.len() - i
11186 ));
11187 break;
11188 }
11189 if entry.len() > left {
11190 let mut end = left;
11193 while !entry.is_char_boundary(end) {
11194 end -= 1;
11195 }
11196 entry.truncate(end);
11197 entry.push_str("\n ... (truncated)\n");
11198 left = 0;
11199 } else {
11200 left -= entry.len();
11201 }
11202 body.push_str(&entry);
11203 }
11204 if !facts.stat.trim().is_empty() {
11205 body.push_str(&format!(
11206 "\n## Diff stat\n\n```\n{}\n```\n",
11207 markdown_safe(facts.stat.trim())
11208 ));
11209 }
11210 } else {
11211 match &state.reviewed_commits {
11212 Some(subjects) => {
11213 for s in subjects {
11214 body.push_str(&format!("- {}\n", s.trim()));
11215 }
11216 }
11217 None => {
11218 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11221 body.push_str(rest.trim());
11222 body.push('\n');
11223 }
11224 }
11225 }
11226 } else {
11227 let task = state.instruction.trim();
11228 let task = if task.is_empty() {
11229 "(empty task)"
11230 } else {
11231 task
11232 };
11233 body.push_str(&format!(
11234 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11235 task.replace("</details>", "</details>")
11236 ));
11237 }
11238
11239 body.push_str(&format!(
11240 "\n---\nmagi:run/{} magi:candidate-{}\n",
11241 state.id,
11242 winner.to_ascii_lowercase()
11243 ));
11244
11245 PrMessage { title, body }
11246}
11247
11248fn seeded_instruction(state: &RunState) -> String {
11252 match refs::describe(&state.seeds) {
11253 Some(facts) => format!(
11254 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11255 Candidates start from the unmerged branch named above, when there \
11256 is one, and carry any unmerged commit named by sha as a \
11257 cherry-pick. Check that this is what the task meant before \
11258 building on it.",
11259 state.instruction
11260 ),
11261 None => state.instruction.clone(),
11262 }
11263}
11264
11265async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11269 let base = &state.base_branch;
11270 let mut against = base.clone();
11271 if mode == MergeMode::Pr {
11272 let remote = &state.config.merge.remote;
11276 let tracking = format!("{remote}/{base}");
11277 let fetched = git::fetch(repo, remote, base).await;
11278 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11279 against = tracking;
11280 } else {
11281 return false;
11282 }
11283 }
11284 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11285}
11286
11287fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11290 let mut detail = format!(
11291 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11292 nothing to open a pull request for"
11293 );
11294 match refs::describe(&state.seeds) {
11295 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11296 None => detail.push_str(
11297 "\nThe task names no existing branch or commit; if it means to land work \
11298 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11299 ),
11300 }
11301 detail
11302}
11303
11304#[derive(Debug, PartialEq, Eq)]
11307enum PrPlan {
11308 Create,
11309 Adopt { url: String, title: String },
11310 Stop(String),
11311}
11312
11313fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11316 match found {
11317 Ok(land::OpenPr::None) => PrPlan::Create,
11318 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11319 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11320 "several open pull requests exist for this branch, not picking one: {}",
11321 urls.join(" ")
11322 )),
11323 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11324 }
11325}
11326
11327pub(crate) async fn gh_pr_create(
11329 cwd: &Path,
11330 base: &str,
11331 head: &str,
11332 title: &str,
11333 body: &str,
11334) -> Result<String> {
11335 let out = tokio::process::Command::new("gh")
11336 .args([
11337 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11338 ])
11339 .current_dir(cwd)
11340 .quiet()
11341 .stdin(std::process::Stdio::null())
11342 .output()
11343 .await
11344 .context("spawn gh")?;
11345 if out.status.success() {
11346 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11347 } else {
11348 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11349 }
11350}
11351
11352async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11354 match std::fs::symlink_metadata(path) {
11355 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11356 git::worktree_remove_clean(repo, path).await?;
11359 return Ok(());
11360 }
11361 Err(e) => return Err(e.into()),
11362 Ok(_) => {}
11363 }
11364 if !git::is_clean(path).await? {
11365 bail!(
11366 "worktree has uncommitted work; refusing removal: {}",
11367 path.display()
11368 );
11369 }
11370 if !git::worktree_remove_clean(repo, path).await? {
11371 bail!("worktree removal refused: {}", path.display());
11372 }
11373 Ok(())
11374}
11375
11376pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11385 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11386 bail!("refusing fold: {reason}");
11387 }
11388 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11389 bail!("candidate preservation incomplete; refusing cleanup");
11390 }
11391 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11395 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11396 let _driver = crate::driver::maintenance(home, &state.id)?;
11397 let mut output_guards = Vec::new();
11398 for entry in std::fs::read_dir(state.worktree_root())
11399 .into_iter()
11400 .flatten()
11401 .flatten()
11402 {
11403 if entry.path().join(".git").is_file() {
11404 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11405 }
11406 }
11407 let repo = state.repo.clone();
11408 let wait = repository_wait(state, None);
11411 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11412 Ok(lease) => lease,
11413 Err(e) => {
11414 state.save_under(home)?;
11415 return Err(e);
11416 }
11417 };
11418 let _source_outputs = if state.status == RunStatus::Merged
11422 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11423 {
11424 verify_merged_fold_sources(state, drop_winner, home).await?
11425 } else {
11426 Vec::new()
11427 };
11428 let root = state.worktree_root();
11429 let winner = state.tally.as_ref().map(|t| t.winner);
11430 let mut removed = Vec::new();
11431
11432 for i in 0..state.candidates.len() {
11433 let c = state.candidates[i].clone();
11434 let is_winner = Some(c.label) == winner;
11435 if is_winner && !drop_winner {
11436 continue;
11437 }
11438 let existed = c.worktree.exists();
11439 remove_preserved_worktree(&repo, &c.worktree).await?;
11442 if existed {
11443 removed.push(c.worktree.to_string_lossy().into_owned());
11444 }
11445 let handed_over = state.released_branches.contains(&c.branch);
11448 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11449 if !git::branch_delete(&repo, &c.branch).await? {
11450 bail!("candidate branch removal refused");
11451 }
11452 removed.push(c.branch.clone());
11453 }
11454 state.candidates[i].folded = true;
11455 }
11456
11457 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11458 let path = name.path();
11459 let keep = !drop_winner
11460 && winner.is_some_and(|w| {
11461 path.file_name()
11462 .is_some_and(|n| n == format!("cand-{w}").as_str())
11463 });
11464 if keep {
11465 continue;
11466 }
11467 remove_preserved_worktree(&repo, &path).await?;
11468 removed.push(path.to_string_lossy().into_owned());
11469 }
11470
11471 remove_if_empty(&root);
11480
11481 if state.enabled_worktree_config && drop_winner {
11482 git::release_worktree_config(&repo).await.ok();
11486 state.enabled_worktree_config = false;
11487 }
11488 state.save_under(home)?;
11489 Ok(removed)
11490}
11491
11492async fn verify_merged_fold_sources(
11493 state: &RunState,
11494 drop_winner: bool,
11495 home: &Path,
11496) -> Result<Vec<std::fs::File>> {
11497 let winner = state.tally.as_ref().map(|t| t.winner);
11498 let mut heads = BTreeSet::new();
11499 for candidate in &state.candidates {
11500 if Some(candidate.label) == winner && !drop_winner {
11501 continue;
11502 }
11503 match std::fs::symlink_metadata(&candidate.worktree) {
11504 Ok(_) => anyhow::ensure!(
11505 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11506 "candidate recovery source is outside the inventoried run root: {}",
11507 candidate.worktree.display()
11508 ),
11509 Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11510 Err(error) => {
11511 return Err(error).context(format!(
11512 "cannot inspect candidate recovery source: {}",
11513 candidate.worktree.display()
11514 ));
11515 }
11516 }
11517 if git::branch_exists(&state.repo, &candidate.branch).await? {
11518 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11519 heads.extend(
11520 git::git(
11521 &state.repo,
11522 &["reflog", "show", "--format=%H", &candidate.branch],
11523 )
11524 .await?
11525 .lines()
11526 .map(str::to_owned),
11527 );
11528 }
11529 }
11530 let mut cargo_guards = Vec::new();
11531 let entries = match std::fs::read_dir(state.worktree_root()) {
11532 Ok(entries) => Some(entries),
11533 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11534 Err(error) => return Err(error.into()),
11535 };
11536 for entry in entries.into_iter().flatten() {
11537 let path = entry?.path();
11538 let metadata = std::fs::symlink_metadata(&path)?;
11539 anyhow::ensure!(
11540 metadata.is_dir() && !metadata.file_type().is_symlink(),
11541 "non-directory or symlink recovery source retained: {}",
11542 path.display()
11543 );
11544 if !drop_winner
11545 && winner.is_some_and(|w| {
11546 path.file_name()
11547 .is_some_and(|name| name == format!("cand-{w}").as_str())
11548 })
11549 {
11550 continue;
11551 }
11552 anyhow::ensure!(
11553 path.join(".git").is_file(),
11554 "unregistered recovery source retained: {}",
11555 path.display()
11556 );
11557 anyhow::ensure!(
11558 git::git(
11559 &path,
11560 &[
11561 "status",
11562 "--porcelain",
11563 "--untracked-files=all",
11564 "--ignored",
11565 "--",
11566 ".",
11567 ":(exclude)target"
11568 ]
11569 )
11570 .await?
11571 .is_empty(),
11572 "source changes or ignored recovery files retained: {}",
11573 path.display()
11574 );
11575 match std::fs::symlink_metadata(path.join("target")) {
11576 Ok(_) => {
11577 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11578 }
11579 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11580 Err(error) => return Err(error.into()),
11581 }
11582 for checkpoint in [
11583 "rebase-merge",
11584 "rebase-apply",
11585 "sequencer",
11586 "CHERRY_PICK_HEAD",
11587 "MERGE_HEAD",
11588 "REBASE_HEAD",
11589 "REVERT_HEAD",
11590 "BISECT_LOG",
11591 "AUTO_MERGE",
11592 "MERGE_MSG",
11593 ] {
11594 let location = git::git(
11595 &path,
11596 &[
11597 "rev-parse",
11598 "--path-format=absolute",
11599 "--git-path",
11600 checkpoint,
11601 ],
11602 )
11603 .await?;
11604 anyhow::ensure!(
11605 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11606 if e.kind() == std::io::ErrorKind::NotFound),
11607 "Git recovery checkpoint retained: {checkpoint}"
11608 );
11609 }
11610 heads.insert(git::rev_parse(&path, "HEAD").await?);
11611 heads.extend(
11612 git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11613 .await?
11614 .lines()
11615 .map(str::to_owned),
11616 );
11617 if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11618 heads.insert(original);
11619 }
11620 }
11621 if !heads.is_empty() {
11622 let reference = format!(
11626 "refs/remotes/{}/{}",
11627 state.config.merge.remote, state.base_branch
11628 );
11629 let base = git::rev_parse(&state.repo, &reference)
11630 .await
11631 .context("cannot prove merged sources against the fetched base")?;
11632 let mut squash_winner = None;
11633 let mut checked_squash = false;
11634 for head in heads {
11635 if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11636 .await?
11637 .ok()
11638 {
11639 continue;
11640 }
11641 if !checked_squash {
11642 squash_winner = proven_squash_winner(state, &base).await?;
11643 checked_squash = true;
11644 }
11645 let incorporated = if let Some(winner) = &squash_winner {
11646 git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11647 .await?
11648 .ok()
11649 } else {
11650 false
11651 };
11652 anyhow::ensure!(
11653 incorporated,
11654 "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11655 );
11656 }
11657 }
11658 Ok(cargo_guards)
11659}
11660
11661async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11667 let Some(winner) = state.winner() else {
11668 return Ok(None);
11669 };
11670 let head = git::rev_parse(&state.repo, &winner.branch).await?;
11671 if !git::git_raw(
11672 &state.repo,
11673 &["merge-base", "--is-ancestor", &state.base_commit, &head],
11674 )
11675 .await?
11676 .ok()
11677 {
11678 return Ok(None);
11679 }
11680 let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11681 if patch.is_empty() {
11682 return Ok(None);
11683 }
11684 let range = format!("{}..{base}", state.base_commit);
11685 let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11686 for commit in commits.lines() {
11687 let parent = format!("{commit}^");
11688 if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11689 return Ok(Some(head));
11690 }
11691 }
11692 Ok(None)
11693}
11694
11695async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11696 git::git(
11697 repo,
11698 &[
11699 "-c",
11700 "core.quotePath=true",
11701 "diff",
11702 "--binary",
11703 "--full-index",
11704 "--no-renames",
11705 "--no-ext-diff",
11706 "--no-textconv",
11707 "--no-color",
11708 "--ignore-submodules=none",
11709 "--submodule=short",
11710 from,
11711 to,
11712 "--",
11713 ],
11714 )
11715 .await
11716}
11717
11718fn remove_if_empty(dir: &Path) {
11729 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11730 std::fs::remove_dir(dir).ok();
11731 }
11732}
11733
11734pub fn worst_open(state: &RunState) -> Option<Severity> {
11736 state
11737 .reviews
11738 .last()?
11739 .reviews
11740 .iter()
11741 .flat_map(|r| r.findings.iter())
11742 .map(|f| f.severity)
11743 .max()
11744}
11745
11746async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11750 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11751 git::acquire_worktree_config(repo).await?;
11752 state.enabled_worktree_config = true;
11753 }
11754 Ok(())
11755}
11756
11757#[cfg(test)]
11758mod tests {
11759 #[tokio::test]
11760 async fn launcher_base_fetch_requires_remote_success() {
11761 let tmp = tempfile::tempdir().unwrap();
11762 let repo = tmp.path().join("repo");
11763 let remote = tmp.path().join("remote");
11764 std::fs::create_dir(&repo).unwrap();
11765 std::fs::create_dir(&remote).unwrap();
11766 git::git(&remote, &["init", "--bare", "-b", "main"])
11767 .await
11768 .unwrap();
11769 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11770 git::git(
11771 &repo,
11772 &[
11773 "-c",
11774 "user.name=Fixture",
11775 "-c",
11776 "user.email=fixture@example.invalid",
11777 "commit",
11778 "--allow-empty",
11779 "-m",
11780 "fixture",
11781 ],
11782 )
11783 .await
11784 .unwrap();
11785 git::git(
11786 &repo,
11787 &["remote", "add", "origin", remote.to_str().unwrap()],
11788 )
11789 .await
11790 .unwrap();
11791 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11792 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11793 assert_eq!(
11794 resolve_base(&repo, "main", "origin", None)
11795 .await
11796 .unwrap()
11797 .commit,
11798 head
11799 );
11800 #[cfg(unix)]
11801 {
11802 let policy: crate::config::Coordination = toml::from_str(
11803 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11804 ).unwrap();
11805 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11806 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11807 let held = crate::repo_lock::SeatLease::operation_with(
11808 &repo,
11809 Some(&policy),
11810 &crate::repo_lock::WaitPolicy::default(),
11811 )
11812 .await
11813 .unwrap();
11814 assert!(held.is_some());
11815 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11816 tokio::pin!(fetch);
11817 assert!(
11818 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11819 .await
11820 .is_err()
11821 );
11822 assert!(
11823 !repo.join(".git/FETCH_HEAD").exists(),
11824 "no fetch before admission"
11825 );
11826 drop(held);
11827 let fetched = fetch.await.unwrap();
11828 assert_eq!(fetched.commit, head);
11829 assert_eq!(fetched.remote, "origin");
11830 assert_eq!(fetched.branch, "main");
11831 }
11832 git::git(
11834 &repo,
11835 &[
11836 "remote",
11837 "set-url",
11838 "origin",
11839 tmp.path().join("missing").to_str().unwrap(),
11840 ],
11841 )
11842 .await
11843 .unwrap();
11844 let error = resolve_base(&repo, "main", "origin", None)
11845 .await
11846 .unwrap_err()
11847 .to_string();
11848 assert!(error.contains("cannot read origin/main"));
11849 assert!(error.contains("refusing to branch off"));
11850 }
11851
11852 #[test]
11853 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11854 let leaked = vec!["chore(deps): update a crate".to_owned()];
11855 let own = "fix(daemon): apply a chosen action";
11856 assert!(should_retitle("", own, &leaked));
11857 assert!(should_retitle(
11858 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11859 own,
11860 &leaked
11861 ));
11862 assert!(should_retitle(
11863 "chore: land candidate A of run 1",
11864 own,
11865 &leaked
11866 ));
11867 assert!(should_retitle(
11868 "magi: candidate A (uncommitted work)",
11869 own,
11870 &leaked
11871 ));
11872 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11873 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11874 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11875 }
11876
11877 #[test]
11878 fn pr_merge_plan_creates_adopts_or_stops() {
11879 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11880 assert_eq!(
11881 pr_merge_plan(Ok(land::OpenPr::One {
11882 url: "u".into(),
11883 title: "t".into()
11884 })),
11885 PrPlan::Adopt {
11886 url: "u".into(),
11887 title: "t".into()
11888 }
11889 );
11890 let PrPlan::Stop(many) =
11891 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11892 else {
11893 panic!("many must stop");
11894 };
11895 assert!(many.contains('a') && many.contains('b'));
11896 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11897 panic!("a failed lookup must stop");
11898 };
11899 assert!(err.contains("bad token"));
11900 }
11901
11902 use super::*;
11903 use crate::run::GateStatus;
11904 use std::collections::BTreeMap;
11905 use std::time::Duration;
11906
11907 fn conductor() -> AgentSpec {
11908 AgentSpec {
11909 id: "conductor".to_owned(),
11910 kind: crate::config::AgentKind::Command,
11911 model: None,
11912 command: vec!["true".to_owned()],
11913 extra_args: Vec::new(),
11914 env: BTreeMap::new(),
11915 prompt_delivery: None,
11916 }
11917 }
11918
11919 fn spec(id: &str) -> AgentSpec {
11920 AgentSpec {
11921 id: id.to_owned(),
11922 kind: crate::config::AgentKind::Command,
11923 model: None,
11924 command: vec!["true".to_owned()],
11925 extra_args: Vec::new(),
11926 env: BTreeMap::new(),
11927 prompt_delivery: None,
11928 }
11929 }
11930
11931 fn ids(xs: &[&str]) -> BTreeSet<String> {
11932 xs.iter().map(|s| (*s).to_owned()).collect()
11933 }
11934
11935 #[test]
11936 fn pick_successor_skips_an_agent_another_seat_holds() {
11937 let roster = [spec("a"), spec("b"), spec("c")];
11939 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11940 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11941 }
11942
11943 #[test]
11944 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11945 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11947 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11948 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11949 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11950 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11951 }
11952
11953 #[test]
11954 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11955 let roster = [spec("a"), spec("b")];
11956 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11957 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11958 let carried = ids(&["b"]);
11959 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11960 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11961 }
11962
11963 #[test]
11964 fn pick_successor_returns_none_without_any_untried_successor() {
11965 let roster = [spec("a"), spec("b")];
11966 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11967 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11968 let carried = ids(&["a"]);
11969 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11970 }
11971
11972 #[test]
11973 fn pick_successor_rescue_avoids_another_seats_occupant() {
11974 let roster = [spec("a"), spec("b"), spec("c")];
11976 let carried = ids(&["b", "c"]);
11977 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11978 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11979 }
11980
11981 #[test]
11982 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11983 let roster = [spec("a"), spec("b"), spec("c")];
11984 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11985 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11986 }
11987
11988 #[test]
11989 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11990 let roster = [spec("a"), spec("b"), spec("c")];
11991 let failed = ids(&["a", "b", "c"]);
11992 let mut tried = ids(&["b"]);
11994 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11995 assert_eq!(first.id, "a");
11996 tried.insert(first.id.clone());
11997 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11998 assert_eq!(second.id, "c");
11999 tried.insert(second.id.clone());
12000 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12001 }
12002
12003 #[test]
12004 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12005 let roster = [spec("a"), spec("b")];
12006 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12007 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12008 }
12009
12010 #[test]
12011 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12012 let roster = [spec("a"), spec("b"), spec("c")];
12013 let h = SeatHistory {
12014 failed: ids(&["a"]),
12015 last_ok: Some("b".to_owned()),
12016 last_fail: None,
12017 };
12018 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12019 let gone = SeatHistory {
12021 last_ok: Some("zzz".to_owned()),
12022 ..h.clone()
12023 };
12024 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12025 let failed = SeatHistory {
12026 failed: ids(&["a", "b"]),
12027 last_ok: Some("b".to_owned()),
12028 last_fail: None,
12029 };
12030 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12031 }
12032
12033 #[test]
12034 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12035 let first = SeatState::new("review-1", "alpha", 7);
12036 let next = handover_seat("review-1", "gamma", 7);
12037 assert_ne!(first.claude_session, next.claude_session);
12038 }
12039
12040 #[test]
12041 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12042 let mut state = state_with_summary("x", "y");
12043 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12044 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12045 assert_ne!(a.claude_session, b.claude_session);
12046 }
12047
12048 #[test]
12049 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12050 let roster = [spec("a"), spec("b")];
12051 let h = SeatHistory {
12052 failed: ids(&["a", "b"]),
12053 ..SeatHistory::default()
12054 };
12055 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12056 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12057 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12058 }
12059
12060 #[test]
12067 fn failure_signature_ignores_numbers_and_paths() {
12068 assert_eq!(
12069 failure_signature("exited with Some(2) and no usable output"),
12070 failure_signature("exited with Some(137) and no usable output")
12071 );
12072 assert_eq!(
12073 failure_signature("process exited with no usable output (exit code 2)"),
12074 failure_signature("process exited with no usable output (exit code 137)")
12075 );
12076 for (old, new) in [
12077 (
12078 "exited with Some(2) and no usable output",
12079 "process exited with no usable output (exit code 2)",
12080 ),
12081 (
12082 "exited with None and no usable output",
12083 "process exited with no usable output (exit code unavailable)",
12084 ),
12085 ] {
12086 let previous = FailClass::Other(failure_signature(old));
12087 let current = FailClass::Other(failure_signature(new));
12088 assert_eq!(previous, current, "resumption retains the failure class");
12089 assert!(!should_hand_over(Some(&previous), ¤t));
12090 }
12091 assert_eq!(
12092 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12093 failure_signature("cannot open /var/x.txt: denied")
12094 );
12095 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12096 }
12097
12098 #[test]
12099 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12100 let other = FailClass::Other("x".into());
12101 assert!(should_hand_over(None, &FailClass::Quota));
12102 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12103 assert!(should_hand_over(
12104 Some(&FailClass::Timeout),
12105 &FailClass::Timeout
12106 ));
12107 assert!(should_hand_over(None, &other));
12108 assert!(!should_hand_over(Some(&other), &other));
12109 assert!(should_hand_over(
12110 Some(&other),
12111 &FailClass::Other("y".into())
12112 ));
12113 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12115 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12116 }
12117
12118 #[test]
12119 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12120 let a = SeatState::new("judge-1", "alpha", 7);
12121 let b = handover_seat("judge-1", "beta", 7);
12122 assert_ne!(a.claude_session, b.claude_session);
12123 assert_eq!(b.turns, 0);
12124 }
12125
12126 #[test]
12127 fn a_timeout_is_classified_apart_from_other_failures() {
12128 assert_eq!(
12129 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12130 Some(FailClass::Timeout)
12131 );
12132 assert!(matches!(
12133 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12134 Some(FailClass::Other(_))
12135 ));
12136 }
12137
12138 #[test]
12139 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12140 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12141 let tried = BTreeSet::from(["beta".to_owned()]);
12142 let next = next_untried_in_roster(&roster, 1, &tried);
12145 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12146 }
12147
12148 #[test]
12149 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12150 let roster = vec![spec("alpha"), spec("beta")];
12151 let tried = BTreeSet::from(["beta".to_owned()]);
12152 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12156 }
12157
12158 #[test]
12159 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12160 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12161 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12162 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12166 }
12167
12168 #[test]
12169 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12170 let roster = vec![spec("a"), spec("a"), spec("b")];
12171 let tried = BTreeSet::from(["a".to_owned()]);
12172 let next = next_untried_in_roster(&roster, 0, &tried);
12173 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12174 }
12175
12176 #[test]
12177 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12178 let roster = vec![spec("a"), spec("b")];
12179 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12180 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12181 }
12182
12183 #[test]
12184 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12185 let dir = tempfile::tempdir().unwrap();
12186 let bay = dir.path().join("ffff");
12187
12188 remove_if_empty(&bay);
12190 assert!(!bay.exists());
12191
12192 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12195 remove_if_empty(&bay);
12196 assert!(bay.exists(), "non-empty directory must survive");
12197
12198 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12200 remove_if_empty(&bay);
12201 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12202 }
12203
12204 #[test]
12213 fn a_full_panel_that_found_nothing_is_clean() {
12214 assert!(round_is_clean(
12215 0,
12216 true,
12217 2,
12218 2,
12219 0,
12220 IncompleteReviewPolicy::Block
12221 ));
12222 }
12223
12224 #[test]
12225 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12226 assert!(!round_is_clean(
12227 0,
12228 true,
12229 1,
12230 2,
12231 0,
12232 IncompleteReviewPolicy::Block
12233 ));
12234 }
12235
12236 #[test]
12237 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12238 assert!(!round_is_clean(
12239 1,
12240 true,
12241 1,
12242 2,
12243 0,
12244 IncompleteReviewPolicy::Warn
12245 ));
12246 }
12247
12248 #[test]
12249 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12250 assert!(round_is_clean(
12251 0,
12252 true,
12253 1,
12254 2,
12255 0,
12256 IncompleteReviewPolicy::Warn
12257 ));
12258 }
12259
12260 #[test]
12261 fn a_full_panel_with_an_open_finding_is_not_clean() {
12262 assert!(!round_is_clean(
12263 1,
12264 true,
12265 2,
12266 2,
12267 0,
12268 IncompleteReviewPolicy::Block
12269 ));
12270 }
12271
12272 #[test]
12273 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12274 assert!(!round_is_clean(
12275 0,
12276 false,
12277 2,
12278 2,
12279 0,
12280 IncompleteReviewPolicy::Block
12281 ));
12282 }
12283
12284 #[test]
12291 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12292 assert!(round_is_clean(
12295 0,
12296 true,
12297 1,
12298 2,
12299 1,
12300 IncompleteReviewPolicy::Block
12301 ));
12302 }
12303
12304 #[test]
12305 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12306 assert!(!round_is_clean(
12309 0,
12310 true,
12311 1,
12312 2,
12313 0,
12314 IncompleteReviewPolicy::Block
12315 ));
12316 }
12317
12318 #[test]
12319 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12320 assert!(!round_is_clean(
12321 1,
12322 true,
12323 1,
12324 2,
12325 1,
12326 IncompleteReviewPolicy::Block
12327 ));
12328 assert!(!round_is_clean(
12329 0,
12330 false,
12331 1,
12332 2,
12333 1,
12334 IncompleteReviewPolicy::Block
12335 ));
12336 }
12337
12338 #[test]
12339 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12340 assert!(!round_is_clean(
12344 0,
12345 true,
12346 0,
12347 2,
12348 2,
12349 IncompleteReviewPolicy::Block
12350 ));
12351 }
12352
12353 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12354 CommandOutcome {
12355 command: "test".to_owned(),
12356 code,
12357 output_tail: String::new(),
12358 duration_ms: 0,
12359 resource_blocked,
12360 }
12361 }
12362
12363 #[test]
12364 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12365 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12366 assert!(
12367 !verify_inconclusive(&[outcome(Some(1), false)]),
12368 "an ordinary failure is still evidence about the patch"
12369 );
12370 assert!(verify_inconclusive(&[outcome(None, true)]));
12371 assert!(
12372 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12373 "one inconclusive outcome taints the whole batch"
12374 );
12375 assert!(!verify_inconclusive(&[]));
12376 }
12377
12378 #[tokio::test]
12379 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12380 let calls = std::sync::atomic::AtomicUsize::new(0);
12384 let started = Instant::now();
12385 wait_for_pids_with(
12386 &[123],
12387 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12388 Duration::from_millis(5),
12389 Duration::from_secs(5),
12390 )
12391 .await;
12392 assert!(
12393 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12394 "must keep checking rather than deciding on the first answer"
12395 );
12396 assert!(
12397 started.elapsed() < Duration::from_secs(1),
12398 "must return the moment it is confirmed dead, not wait out the ceiling"
12399 );
12400 }
12401
12402 #[tokio::test]
12403 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12404 let started = Instant::now();
12405 wait_for_pids_with(
12406 &[123],
12407 |_| true, Duration::from_millis(5),
12409 Duration::from_millis(30),
12410 )
12411 .await;
12412 let elapsed = started.elapsed();
12413 assert!(
12414 elapsed >= Duration::from_millis(30),
12415 "must not give up before its own ceiling: {elapsed:?}"
12416 );
12417 assert!(
12418 elapsed < Duration::from_secs(1),
12419 "must not wait past its own ceiling either: {elapsed:?}"
12420 );
12421 }
12422
12423 #[tokio::test]
12424 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12425 let started = Instant::now();
12426 wait_for_pids_with(
12427 &[],
12428 |_| true,
12429 Duration::from_secs(5),
12430 Duration::from_secs(5),
12431 )
12432 .await;
12433 assert!(
12434 started.elapsed() < Duration::from_millis(200),
12435 "an empty pid list has nothing to confirm"
12436 );
12437 }
12438
12439 fn review_round(
12445 clean: bool,
12446 blocking: usize,
12447 answered: usize,
12448 expected: usize,
12449 progressed: bool,
12450 e2e_ok: bool,
12451 ) -> ReviewRound {
12452 ReviewRound {
12453 round: 1,
12454 head: "h".to_owned(),
12455 verified_head: None,
12456 verified_at: None,
12457 reviews: Vec::new(),
12458 e2e: vec![CommandOutcome {
12459 command: "test".to_owned(),
12460 code: Some(if e2e_ok { 0 } else { 1 }),
12461 output_tail: String::new(),
12462 duration_ms: 0,
12463 resource_blocked: false,
12464 }],
12465 verify_retried: false,
12466 e2e_deferred: false,
12467 e2e_defer_reason: None,
12468 fix: None,
12469 blocking,
12470 answered,
12471 expected,
12472 clean,
12473 progressed,
12474 vote_split: false,
12475 reconsideration: Vec::new(),
12476 verdict: None,
12477 }
12478 }
12479
12480 #[test]
12481 fn review_conclusion_is_none_when_nothing_has_run() {
12482 assert_eq!(review_conclusion(&[], 3), None);
12483 }
12484
12485 #[test]
12486 fn review_conclusion_is_none_while_rounds_remain() {
12487 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12488 assert_eq!(review_conclusion(&rounds, 3), None);
12489 }
12490
12491 #[test]
12492 fn review_conclusion_is_gating_once_a_round_is_clean() {
12493 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12494 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12495 }
12496
12497 #[test]
12498 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12499 let rounds = vec![
12500 review_round(false, 1, 2, 2, true, true),
12501 review_round(false, 1, 2, 2, true, true),
12502 ];
12503 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12504 }
12505
12506 #[test]
12507 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12508 let rounds = vec![
12509 review_round(false, 1, 2, 2, true, true),
12510 review_round(false, 1, 2, 2, true, false),
12511 ];
12512 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12513 }
12514
12515 #[test]
12516 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12517 let mut blocked = review_round(false, 1, 2, 2, true, false);
12524 blocked.e2e[0].resource_blocked = true;
12525 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12526 assert_eq!(review_conclusion(&rounds, 2), None);
12527 }
12528
12529 #[test]
12530 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12531 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12533 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12534 }
12535
12536 #[test]
12537 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12538 let rounds = vec![
12539 review_round(false, 1, 2, 2, false, true),
12540 review_round(false, 1, 2, 2, false, true),
12541 ];
12542 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12543 }
12544
12545 fn secs(n: u64) -> Duration {
12546 Duration::from_secs(n)
12547 }
12548
12549 #[tokio::test]
12552 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12553 ask_test_home();
12554 let tmp = tempfile::tempdir().unwrap();
12555 let repo = tmp.path().join("repo");
12556 std::fs::create_dir_all(&repo).unwrap();
12557 init_repo(&repo);
12558 let g = |args: &[&str]| {
12559 let out = std::process::Command::new("git")
12560 .args(args)
12561 .current_dir(&repo)
12562 .quiet()
12563 .output()
12564 .unwrap();
12565 assert!(
12566 out.status.success(),
12567 "{}",
12568 String::from_utf8_lossy(&out.stderr)
12569 );
12570 };
12571 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12572 g(&["checkout", "-b", "historical"]);
12573 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12574 g(&["add", "."]);
12575 g(&["commit", "-m", "old alternative"]);
12576 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12577 g(&["checkout", "-b", "magi/task-selected", &base]);
12578 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12579 g(&["add", "."]);
12580 g(&["commit", "-m", "measurement infrastructure"]);
12581 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12582 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12583 g(&["add", "."]);
12584 g(&["commit", "-m", "dependent implementation"]);
12585 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12586 g(&["checkout", "main"]);
12587 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12588 g(&["add", "."]);
12589 g(&["commit", "-m", "unrelated base change"]);
12590 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12591 let origin = tmp.path().join("origin.git");
12592 g(&[
12593 "clone",
12594 "--bare",
12595 repo.to_str().unwrap(),
12596 origin.to_str().unwrap(),
12597 ]);
12598 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12599 g(&["fetch", "origin"]);
12600 let selection = crate::pr_recovery::Selection {
12601 url: "https://example.test/project/pull/1".into(),
12602 branch: "magi/task-selected".into(),
12603 head: head.clone(),
12604 source_head: None,
12605 source_run: None,
12606 source_proof: None,
12607 disposition: Some(crate::pr_recovery::Disposition::Review),
12608 instruction: "preserve measured behavior".into(),
12609 approved_at: jiff::Timestamp::now(),
12610 };
12611 let mut template = runner_at(RunStatus::Reviewing);
12612 template.state.repo = repo.clone();
12613 template.state.base_commit = current.clone();
12614 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12615 template.state.origin = Some(crate::run::Origin::queue("selected"));
12616 template.state.pr_head = Some(selection.branch.clone());
12617 template.state.pr_recovery = Some(selection.clone());
12618 let mut runner =
12619 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12620 .await
12621 .unwrap();
12622 let wt = runner.state.winner().unwrap().worktree.clone();
12623 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12624 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12625 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12626 assert!(runner.state.seeds.is_empty());
12627 assert_eq!(runner.state.instruction, "preserve measured behavior");
12628 runner.sync_to_base().await.unwrap();
12629 assert_eq!(
12630 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12631 "prerequisite"
12632 );
12633 assert_eq!(
12634 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12635 "measured implementation"
12636 );
12637 assert_eq!(
12638 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12639 "current main change"
12640 );
12641 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12642 assert_eq!(
12643 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12644 head
12645 );
12646
12647 let mut partial = runner_at(RunStatus::Reviewing);
12649 partial.state.repo = repo.clone();
12650 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12651 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12652 run: Some("exit 1".into()),
12653 ..Default::default()
12654 }];
12655 let selection = runner.state.pr_recovery.clone().unwrap();
12656 partial.state.pr_recovery = Some(selection.clone());
12657 partial.state.pr_head = Some(selection.branch.clone());
12658 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12659 let id = partial.state.id.clone();
12660 let error = match Runner::open_selected(
12661 &repo,
12662 partial.state,
12663 partial.roles,
12664 1,
12665 runner.state.base_commit.clone(),
12666 selection,
12667 )
12668 .await
12669 {
12670 Ok(_) => panic!("setup must fail"),
12671 Err(e) => e,
12672 };
12673 assert_eq!(
12674 error
12675 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12676 .unwrap()
12677 .run,
12678 id
12679 );
12680 let failed = RunState::load(&id).unwrap();
12681 assert_eq!(failed.status, RunStatus::Blocked);
12682 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12683 assert!(Runner::resume(&id).is_err());
12684 runner.state.status = RunStatus::Ready;
12685 runner.state.driver_exited = true;
12686 runner.state.save().unwrap();
12687 let mut retry = runner_at(RunStatus::Reviewing);
12688 retry.state.repo = repo.clone();
12689 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12690 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12691 reselected.approved_at = jiff::Timestamp::now();
12692 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12693 retry.state.pr_head = Some(reselected.branch.clone());
12694 retry.state.pr_recovery = Some(reselected.clone());
12695 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12696 .await
12697 .unwrap();
12698 let restarted = Runner::open_selected(
12699 &repo,
12700 retry.state,
12701 retry.roles,
12702 1,
12703 runner.state.base_commit.clone(),
12704 reselected,
12705 )
12706 .await
12707 .unwrap();
12708 assert_eq!(
12709 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12710 .await
12711 .unwrap(),
12712 head
12713 );
12714 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12715 assert_eq!(
12716 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12717 head
12718 );
12719 }
12720
12721 #[tokio::test]
12722 #[cfg(unix)]
12723 async fn fallback_and_interrupted_restoration_require_coordination() {
12724 use std::os::fd::AsRawFd as _;
12725 ask_test_home();
12726 let tmp = tempfile::tempdir().unwrap();
12727 let mut runner = registered_runner(tmp.path()).await;
12728 runner.state.config.graph.timeout_resource_wait = 1;
12729 let winner = runner.state.candidates[0].clone();
12730 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12731 let leftover = winner.worktree.join("leftover");
12732 std::fs::write(&leftover, "untrusted edits").unwrap();
12733 let next = tmp.path().join("next-fixer");
12734 let mut failed = spec("first");
12735 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12736 let mut successor = spec("second");
12737 successor.command = vec![
12738 "sh".into(),
12739 "-c".into(),
12740 format!("touch '{}'; echo done", next.display()),
12741 ];
12742 let attempts = vec![
12743 (failed.clone(), "first".into()),
12744 (successor.clone(), "second".into()),
12745 ];
12746 let lock_path = runner.state.repo.join(".git/updater.lock");
12747 let external = std::fs::File::create(&lock_path).unwrap();
12748 assert_eq!(
12749 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12750 0
12751 );
12752 let artifacts = tmp.path().join("artifacts");
12753 let build = |spec, seat| SeatJob {
12754 spec,
12755 seat,
12756 cwd: winner.worktree.clone(),
12757 prompt: "fixture".into(),
12758 timeout: Duration::from_secs(10),
12759 allow_write: true,
12760 sessions: false,
12761 artifacts: artifacts.clone(),
12762 stem: "fix".into(),
12763 handover: None,
12764 };
12765 let (_, _, out) = runner
12766 .ask_fixer_among(
12767 attempts.clone(),
12768 "graft",
12769 None,
12770 Some((&winner.worktree, &head)),
12771 build,
12772 )
12773 .await;
12774 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12775 assert!(leftover.exists(), "contention must prevent clean");
12776 assert!(!next.exists());
12777 assert!(runner.state.handovers.is_empty());
12778 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12780 from_run: "earlier".into(),
12781 generation: 1,
12782 mode: crate::recovery::Mode::VerifyRepair,
12783 reasons: Vec::new(),
12784 reviewed_head: None,
12785 rounds: Vec::new(),
12786 open_findings: Vec::new(),
12787 evidence: None,
12788 criteria_inherited: true,
12789 repair: crate::recovery::Repair::Started {
12790 before: head.clone(),
12791 },
12792 });
12793 assert!(
12794 runner
12795 .recovery_repair(&winner)
12796 .await
12797 .unwrap_err()
12798 .is::<crate::repo_lock::ResourceBlocked>()
12799 );
12800 assert!(matches!(
12801 runner.state.recovery.as_ref().unwrap().repair,
12802 crate::recovery::Repair::Started { .. }
12803 ));
12804 assert!(leftover.exists());
12805 drop(external);
12806 runner.recovery_repair(&winner).await.unwrap();
12807 assert!(!leftover.exists());
12808 assert!(matches!(
12809 runner.state.recovery.as_ref().unwrap().repair,
12810 crate::recovery::Repair::Done { .. }
12811 ));
12812 std::fs::remove_file(&lock_path).unwrap();
12814 std::fs::create_dir(&lock_path).unwrap();
12815 std::fs::write(&leftover, "untrusted edits").unwrap();
12816 let (_, _, out) = runner
12817 .ask_fixer_among(
12818 attempts,
12819 "graft",
12820 None,
12821 Some((&winner.worktree, &head)),
12822 build,
12823 )
12824 .await;
12825 assert!(matches!(out, AgentOutcome::Failed(_)));
12826 assert!(leftover.exists());
12827 assert!(!next.exists());
12828 assert_eq!(
12829 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12830 head
12831 );
12832 }
12833
12834 #[cfg(unix)]
12835 async fn registered_runner(root: &Path) -> Runner {
12836 ask_test_home();
12837 let repo = root.join("repo");
12838 std::fs::create_dir_all(&repo).unwrap();
12839 init_repo(&repo);
12840 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12841 let mut runner = runner_at(RunStatus::Prep);
12842 runner.state.repo = repo.clone();
12843 runner.state.config.coordination = crate::config::test_coordination();
12844 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12845 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12846 runner.state.config.graph.advise = false;
12847 runner.roles.implementers = vec![spec("implement")];
12848 runner.roles.judges = vec![spec("judge")];
12849 runner.prep().await.unwrap();
12850 runner
12851 }
12852
12853 #[tokio::test]
12854 #[cfg(unix)]
12855 async fn parked_implementer_persists_completed_work() {
12856 for registered in [false, true] {
12857 let tmp = tempfile::tempdir().unwrap();
12858 let mut runner = registered_runner(tmp.path()).await;
12859 if !registered {
12860 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12861 }
12862 let marker = tmp.path().join("started");
12863 let release = tmp.path().join("release");
12864 let mut implementer = spec("implement");
12865 implementer.command = vec![
12866 "sh".into(),
12867 "-c".into(),
12868 format!(
12869 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12870 marker.display(),
12871 release.display()
12872 ),
12873 ];
12874 runner.roles.implementers = vec![implementer.clone()];
12875 runner.state.config.agents = vec![implementer];
12876 let pause = runner.pause.clone();
12877 let (result, ()) = tokio::join!(runner.implement(), async {
12878 tokio::time::timeout(Duration::from_secs(10), async {
12879 while !marker.exists() {
12880 tokio::time::sleep(Duration::from_millis(10)).await;
12881 }
12882 })
12883 .await
12884 .unwrap();
12885 pause.park();
12886 std::fs::write(&release, "go").unwrap();
12887 });
12888 result.unwrap();
12889 let candidate = &runner.state.candidates[0];
12890 assert!(candidate.commits > 0, "{candidate:?}");
12891 assert!(candidate.summary.contains("saved-result"));
12892 assert_eq!(candidate.files, 1);
12893 let before = candidate.commits;
12894 runner.implement().await.unwrap();
12895 assert_eq!(runner.state.candidates[0].commits, before);
12896 }
12897 }
12898
12899 #[tokio::test]
12900 #[cfg(unix)]
12901 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12902 let tmp = tempfile::tempdir().unwrap();
12903 let mut runner = registered_runner(tmp.path()).await;
12904 let winner = runner.state.candidates[0].clone();
12905 let cache = tmp.path().join("cache");
12906 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12907 runner.state.config.graph.timeout_resource_wait = 0;
12908 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12909 let crate::cache::AcquireOutcome::Acquired(held) =
12910 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12911 else {
12912 panic!("cache must be free")
12913 };
12914 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12915 from_run: "earlier".into(),
12916 generation: 1,
12917 mode: crate::recovery::Mode::VerifyRepair,
12918 reasons: Vec::new(),
12919 reviewed_head: None,
12920 rounds: Vec::new(),
12921 open_findings: Vec::new(),
12922 evidence: None,
12923 criteria_inherited: true,
12924 repair: crate::recovery::Repair::NotStarted,
12925 });
12926 runner.recovery_repair(&winner).await.unwrap();
12927 assert!(matches!(
12928 runner.state.recovery.as_ref().unwrap().repair,
12929 crate::recovery::Repair::WaitingForRestore { .. }
12930 ));
12931 drop(held);
12932 runner.state.resource_block = None;
12933 runner.state.config.graph.timeout_resource_wait = 1;
12934 runner.recovery_repair(&winner).await.unwrap();
12935 assert!(matches!(
12936 runner.state.recovery.as_ref().unwrap().repair,
12937 crate::recovery::Repair::Done { .. }
12938 ));
12939 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12940 }
12941
12942 #[tokio::test]
12943 #[cfg(unix)]
12944 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12945 use std::os::fd::AsRawFd as _;
12946 for graft in [false, true] {
12947 let tmp = tempfile::tempdir().unwrap();
12948 let mut runner = registered_runner(tmp.path()).await;
12949 runner.state.config.graph.timeout_resource_wait = 1;
12950 let winner = runner.state.candidates[0].clone();
12951 let marker = tmp.path().join("fixer-started");
12952 let mut fixer = spec("fixer");
12953 fixer.command = vec![
12954 "sh".into(),
12955 "-c".into(),
12956 format!("echo started >> '{}'; echo done", marker.display()),
12957 ];
12958 runner.roles.fixer = Some(vec![fixer]);
12959 if graft {
12960 let mut loser = winner.clone();
12961 loser.label = 'B';
12962 loser.index = 1;
12963 runner.state.candidates.push(loser);
12964 runner.state.config.graph.graft_runner_up = true;
12965 runner.state.tally = Some(Tally {
12966 first_choice: BTreeMap::new(),
12967 borda: BTreeMap::new(),
12968 winner: winner.label,
12969 rankings: 1,
12970 unanimous_initial: true,
12971 deliberated: false,
12972 changed_votes: 0,
12973 unanimous_final: true,
12974 tie_break: None,
12975 judges: 1,
12976 present: 1,
12977 quorum: 1,
12978 met_quorum: true,
12979 uncontested: None,
12980 });
12981 runner.state.judgements.push(crate::run::Judgement {
12982 judge: 1,
12983 seat: "judge-1".into(),
12984 agent: "judge".into(),
12985 ranking: vec![winner.label, 'B'],
12986 reasons: BTreeMap::new(),
12987 confidence: None,
12988 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12989 order: vec![0, 1],
12990 failed: None,
12991 duration_ms: 0,
12992 });
12993 } else {
12994 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12995 from_run: "earlier".into(),
12996 generation: 1,
12997 mode: crate::recovery::Mode::VerifyRepair,
12998 reasons: Vec::new(),
12999 reviewed_head: None,
13000 rounds: Vec::new(),
13001 open_findings: Vec::new(),
13002 evidence: None,
13003 criteria_inherited: true,
13004 repair: crate::recovery::Repair::NotStarted,
13005 });
13006 }
13007 let reader = crate::repo_lock::SeatLease::for_checkout(
13008 &winner.worktree,
13009 false,
13010 crate::config::test_coordination().as_ref(),
13011 )
13012 .await
13013 .unwrap();
13014 let pause = runner.pause.clone();
13015 let (result, ()) = tokio::join!(
13016 async {
13017 if graft {
13018 runner.graft().await
13019 } else {
13020 runner.recovery_repair(&winner).await
13021 }
13022 },
13023 async {
13024 tokio::time::sleep(Duration::from_millis(100)).await;
13025 pause.park();
13026 }
13027 );
13028 result.unwrap();
13029 assert!(!marker.exists(), "a blocked seat must not spawn");
13030 runner.state = RunState::load(&runner.state.id).unwrap();
13032 if graft {
13033 assert_eq!(
13034 runner.state.graft.as_ref().unwrap().outcome,
13035 GraftOutcome::WaitingForRestore
13036 );
13037 } else {
13038 assert!(matches!(
13039 runner.state.recovery.as_ref().unwrap().repair,
13040 crate::recovery::Repair::WaitingForRestore { .. }
13041 ));
13042 }
13043 drop(reader);
13044 runner.pause = Pause::new();
13045 runner.state.resource_block = None;
13046 let external =
13048 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13049 assert_eq!(
13050 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13051 0
13052 );
13053 let leftover = winner.worktree.join("leftover");
13054 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13055 let result = if graft {
13056 runner.graft().await
13057 } else {
13058 runner.recovery_repair(&winner).await
13059 };
13060 assert!(
13061 result
13062 .unwrap_err()
13063 .is::<crate::repo_lock::ResourceBlocked>()
13064 );
13065 assert!(leftover.exists());
13066 assert!(!marker.exists());
13067 drop(external);
13068 runner.state.resource_block = None;
13069 let result = if graft {
13070 runner.graft().await
13071 } else {
13072 runner.recovery_repair(&winner).await
13073 };
13074 result.unwrap();
13075 assert!(!leftover.exists(), "restore must precede retry");
13076 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13077 if graft {
13078 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13079 } else {
13080 assert!(matches!(
13081 runner.state.recovery.as_ref().unwrap().repair,
13082 crate::recovery::Repair::Done { .. }
13083 ));
13084 }
13085 }
13086 }
13087
13088 #[tokio::test]
13089 #[cfg(unix)]
13090 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13091 use std::os::fd::AsRawFd as _;
13092 let tmp = tempfile::tempdir().unwrap();
13093 let mut runner = registered_runner(tmp.path()).await;
13094 runner.state.config.graph.timeout_resource_wait = 1;
13095 let winner = runner.state.candidates[0].clone();
13096 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13097 runner.state.tally = Some(Tally {
13098 first_choice: BTreeMap::new(),
13099 borda: BTreeMap::new(),
13100 winner: winner.label,
13101 rankings: 0,
13102 unanimous_initial: false,
13103 deliberated: false,
13104 changed_votes: 0,
13105 unanimous_final: false,
13106 tie_break: None,
13107 judges: 0,
13108 present: 0,
13109 quorum: 0,
13110 met_quorum: true,
13111 uncontested: Some("fixture".into()),
13112 });
13113 runner.state.graft = Some(GraftRecord {
13114 pre: pre.clone(),
13115 post: None,
13116 lines: None,
13117 limit: None,
13118 agent: None,
13119 outcome: GraftOutcome::Running,
13120 requested: Vec::new(),
13121 adoption: None,
13122 adoption_error: None,
13123 });
13124 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13125 git::commit_all(&winner.worktree, "unfinished graft")
13126 .await
13127 .unwrap();
13128 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13129 let leftover = winner.worktree.join("leftover");
13130 std::fs::write(&leftover, "untrusted edits").unwrap();
13131 let path = runner.state.repo.join(".git/updater.lock");
13132 let external = std::fs::File::create(&path).unwrap();
13133 assert_eq!(
13134 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13135 0
13136 );
13137 assert!(
13138 runner
13139 .graft()
13140 .await
13141 .unwrap_err()
13142 .is::<crate::repo_lock::ResourceBlocked>()
13143 );
13144 assert!(matches!(
13145 runner.state.graft.as_ref().unwrap().outcome,
13146 GraftOutcome::Running
13147 ));
13148 assert_eq!(
13149 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13150 unfinished
13151 );
13152 assert!(leftover.exists());
13153 drop(external);
13154 std::fs::remove_file(&path).unwrap();
13155 std::fs::create_dir(&path).unwrap();
13156 assert!(runner.graft().await.is_err());
13157 assert!(matches!(
13158 runner.state.graft.as_ref().unwrap().outcome,
13159 GraftOutcome::Running
13160 ));
13161 assert_eq!(
13162 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13163 unfinished
13164 );
13165 assert!(leftover.exists());
13166 std::fs::remove_dir(&path).unwrap();
13167 runner.graft().await.unwrap();
13168 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13169 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13170 assert!(!leftover.exists());
13171 }
13172
13173 #[tokio::test]
13174 #[cfg(unix)]
13175 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13176 use std::os::fd::AsRawFd as _;
13177 let tmp = tempfile::tempdir().unwrap();
13178 let mut runner = registered_runner(tmp.path()).await;
13179 runner.state.prep_complete = Some(false);
13180 runner.state.config.graph.timeout_resource_wait = 1;
13181 runner.state.save().unwrap();
13182 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13183 assert_eq!(
13184 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13185 0
13186 );
13187 runner.execute().await.unwrap();
13188 let disk = RunState::load(&runner.state.id).unwrap();
13189 assert!(disk.driver_exited);
13190 assert_eq!(disk.driver_pid, None);
13191 assert_eq!(runner.state.status, RunStatus::Blocked);
13192 assert_eq!(
13193 runner.state.resource_block.as_ref().unwrap().kind,
13194 "repository coordination"
13195 );
13196 assert!(runner.state.prep_failure.is_none());
13197 assert!(runner.state.seats.is_empty());
13198 runner.pause.park();
13199 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13202 .await
13203 .unwrap()
13204 .unwrap();
13205 assert!(runner.state.parked);
13206 assert!(runner.state.prep_failure.is_none());
13207 let disk = RunState::load(&runner.state.id).unwrap();
13208 assert!(disk.driver_exited);
13209 assert_eq!(disk.driver_pid, None);
13210 }
13211
13212 #[tokio::test]
13213 #[cfg(unix)]
13214 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13215 use std::os::fd::AsRawFd as _;
13216 let tmp = tempfile::tempdir().unwrap();
13217 let mut runner = registered_runner(tmp.path()).await;
13218 runner.state.prep_complete = Some(false);
13219 runner.state.config.graph.timeout_resource_wait = 30;
13220 runner.state.save().unwrap();
13221 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13222 assert_eq!(
13223 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13224 0
13225 );
13226 assert!(
13227 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13228 .await
13229 .is_err()
13230 );
13231 let disk = RunState::load(&runner.state.id).unwrap();
13232 assert!(disk.driver_exited);
13233 assert_eq!(disk.driver_pid, None);
13234 assert!(disk.status.resumable());
13235 assert!(runner.state.driver_exited);
13236 assert_eq!(runner.state.driver_pid, None);
13237 runner.state.save().unwrap();
13238 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13239 }
13240
13241 #[tokio::test]
13242 #[cfg(unix)]
13243 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13244 let tmp = tempfile::tempdir().unwrap();
13245 let mut runner = registered_runner(tmp.path()).await;
13246 let winner = runner.state.candidates[0].clone();
13247 let cache = tmp.path().join("cache");
13248 let started = tmp.path().join("pre-gate-started");
13249 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13250 runner.state.config.verify.gate =
13251 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13252 let home = crate::run::home();
13253 let owner =
13254 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13255 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13256 crate::cache::AcquireOutcome::Acquired(g) => g,
13257 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13258 };
13259 let mut gate = Box::pin(runner.run_gate(&winner));
13260 assert!(
13261 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13262 .await
13263 .is_err()
13264 );
13265 assert!(!started.exists(), "pre-gate must wait for the cache too");
13266 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13270 let editor = crate::repo_lock::SeatLease::checkout_with(
13271 &winner.worktree,
13272 true,
13273 crate::config::test_coordination().as_ref(),
13274 &wait,
13275 )
13276 .await
13277 .unwrap();
13278 drop(editor);
13279 guard.release();
13280 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13281 .await
13282 .unwrap()
13283 .unwrap();
13284 drop(gate);
13285 assert!(outcomes.iter().all(CommandOutcome::ok));
13286 assert!(started.exists());
13287 }
13288
13289 #[tokio::test]
13290 #[cfg(unix)]
13291 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13292 use std::os::fd::AsRawFd as _;
13293
13294 ask_test_home();
13295 let tmp = tempfile::tempdir().unwrap();
13296 let repo = tmp.path().join("repo");
13297 std::fs::create_dir_all(&repo).unwrap();
13298 init_repo(&repo);
13299 let common = repo.join(".git");
13300 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13301 let mut runner = runner_at(RunStatus::Prep);
13302 runner.state.repo = repo.clone();
13303 runner.state.config.coordination = crate::config::test_coordination();
13304 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13305 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13306 runner.state.config.graph.advise = false;
13307 runner.roles.implementers = vec![spec("implement")];
13308 runner.roles.judges = vec![spec("judge")];
13309 runner.prep().await.unwrap();
13310 let winner = runner.state.candidates[0].clone();
13311 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13314 let completed = tmp.path().join("seat-completed");
13315 runner.roles.implementers[0].command = vec![
13316 "sh".into(),
13317 "-c".into(),
13318 format!("touch '{}'; echo completed", completed.display()),
13319 ];
13320 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13321 assert_eq!(
13322 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13323 0
13324 );
13325 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13326 let mut implement = Box::pin(runner.implement());
13327 tokio::select! {
13328 result = &mut implement => panic!("rescue must wait: {result:?}"),
13329 result = tokio::time::timeout(Duration::from_secs(10), async {
13330 while !completed.exists() {
13331 tokio::time::sleep(Duration::from_millis(20)).await;
13332 }
13333 }) => result.unwrap(),
13334 }
13335 assert!(
13336 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13337 .await
13338 .is_err()
13339 );
13340 assert_eq!(
13341 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13342 before
13343 );
13344 assert_eq!(
13345 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13346 0
13347 );
13348 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13349 .await
13350 .unwrap()
13351 .unwrap();
13352 drop(implement);
13353 assert_ne!(
13354 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13355 before
13356 );
13357 assert!(git::is_clean(&winner.worktree).await.unwrap());
13358
13359 let started = tmp.path().join("pre-gate-started");
13360 let proceed = tmp.path().join("pre-gate-proceed");
13361 let checked = tmp.path().join("gate-ran");
13362 runner.state.config.verify.pre_gate = vec![format!(
13363 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13364 started.display(),
13365 proceed.display()
13366 )];
13367 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13368 assert_eq!(
13369 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13370 0
13371 );
13372 let mut gate = Box::pin(runner.run_gate(&winner));
13373 assert!(
13374 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13375 .await
13376 .is_err()
13377 );
13378 assert!(!started.exists());
13379 assert!(!checked.exists());
13380 assert!(git::is_clean(&winner.worktree).await.unwrap());
13381 assert_eq!(
13382 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13383 0
13384 );
13385 tokio::select! {
13386 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13387 result = tokio::time::timeout(Duration::from_secs(10), async {
13388 while !started.exists() {
13389 tokio::time::sleep(Duration::from_millis(20)).await;
13390 }
13391 }) => result.unwrap(),
13392 }
13393 assert_eq!(
13394 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13395 -1,
13396 "mechanical edits must hold the real lock"
13397 );
13398 std::fs::write(proceed, "ready").unwrap();
13399 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13400 .await
13401 .unwrap()
13402 .unwrap();
13403 drop(gate);
13404 assert!(outcomes.iter().all(CommandOutcome::ok));
13405 assert!(checked.exists());
13406 assert!(runner.state.pre_gate_commit.is_some());
13407 assert!(git::is_clean(&winner.worktree).await.unwrap());
13408 assert_eq!(
13409 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13410 0,
13411 "gate completion must release coordination"
13412 );
13413 drop(outsider);
13414 std::fs::remove_file(common.join("updater.lock")).unwrap();
13415 std::fs::create_dir(common.join("updater.lock")).unwrap();
13416 std::fs::remove_file(&started).unwrap();
13417 std::fs::remove_file(&checked).unwrap();
13418 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13419 let error = runner.run_gate(&winner).await.unwrap_err();
13420 assert!(
13421 error
13422 .to_string()
13423 .contains("acquire mandatory repository lock")
13424 );
13425 assert!(
13426 !started.exists(),
13427 "failed acquisition must prevent pre-gate"
13428 );
13429 assert!(!checked.exists(), "failed acquisition must prevent gate");
13430 assert!(git::is_clean(&winner.worktree).await.unwrap());
13431 assert_eq!(
13432 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13433 head
13434 );
13435 }
13436
13437 #[tokio::test]
13438 #[cfg(unix)]
13439 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13440 use std::os::fd::AsRawFd as _;
13441 use std::time::Duration;
13442
13443 ask_test_home();
13444 let tmp = tempfile::tempdir().unwrap();
13445 let repo = tmp.path().join("repo");
13446 std::fs::create_dir_all(&repo).unwrap();
13447 init_repo(&repo);
13448 let common = repo.join(".git");
13449 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13450 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13451 assert_eq!(
13452 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13453 0
13454 );
13455 let mut runner = runner_at(RunStatus::Prep);
13456 runner.state.repo = repo.clone();
13457 runner.state.config.coordination = crate::config::test_coordination();
13458 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13459 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13460 runner.state.config.graph.advise = true;
13461 runner.state.config.graph.advisors = 1;
13462 runner.roles.implementers = vec![spec("implement")];
13463 runner.roles.judges = vec![spec("judge")];
13464 let root = runner.state.worktree_root();
13465 let started = tmp.path().join("setup-started");
13466 let proceed = tmp.path().join("setup-proceed");
13467 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13468 run: Some(format!(
13469 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13470 started.display(),
13471 proceed.display()
13472 )),
13473 ..Default::default()
13474 }];
13475 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13476 let config = git::git(&repo, &["config", "--local", "--list"])
13477 .await
13478 .unwrap();
13479 let mut prep = Box::pin(runner.prep());
13480 assert!(
13481 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13482 .await
13483 .is_err()
13484 );
13485 assert!(
13486 !root.exists(),
13487 "contention must prevent even checkout creation"
13488 );
13489 assert!(!started.exists(), "contention must prevent setup");
13490 assert_eq!(
13491 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13492 branches
13493 );
13494 assert_eq!(
13495 git::git(&repo, &["config", "--local", "--list"])
13496 .await
13497 .unwrap(),
13498 config
13499 );
13500 drop(lock);
13501 tokio::select! {
13502 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13503 result = tokio::time::timeout(Duration::from_secs(10), async {
13504 while !started.exists() {
13505 tokio::time::sleep(Duration::from_millis(20)).await;
13506 }
13507 }) => result.unwrap(),
13508 }
13509 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13510 assert_eq!(
13511 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13512 -1
13513 );
13514 assert!(
13515 tokio::time::timeout(
13516 Duration::from_millis(100),
13517 crate::repo_lock::SeatLease::for_checkout(
13518 &repo,
13519 true,
13520 crate::config::test_coordination().as_ref()
13521 )
13522 )
13523 .await
13524 .is_err(),
13525 "preparation must also exclude local editors"
13526 );
13527 std::fs::write(proceed, "ready").unwrap();
13528 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13529 .await
13530 .unwrap()
13531 .unwrap();
13532 drop(prep);
13533 assert_eq!(runner.state.prep_complete, Some(true));
13534 assert!(root.join("judge-1").exists());
13535 assert!(root.join("advisor-1").exists());
13536 assert_eq!(
13537 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13538 0
13539 );
13540 }
13541
13542 #[tokio::test]
13543 #[cfg(unix)]
13544 async fn preparation_lock_failure_prevents_repository_changes() {
13545 ask_test_home();
13546 let tmp = tempfile::tempdir().unwrap();
13547 let repo = tmp.path().join("repo");
13548 std::fs::create_dir_all(&repo).unwrap();
13549 init_repo(&repo);
13550 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13551 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13554 let mut runner = runner_at(RunStatus::Prep);
13555 runner.state.repo = repo.clone();
13556 runner.state.config.coordination = crate::config::test_coordination();
13557 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13558 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13559 runner.roles.implementers = vec![spec("implement")];
13560 let error = runner.prep().await.unwrap_err();
13561 assert!(
13562 error
13563 .to_string()
13564 .contains("acquire mandatory repository lock")
13565 );
13566 assert!(!runner.state.worktree_root().exists());
13567 assert!(runner.state.candidates.is_empty());
13568 assert!(!repo.join(".git/config.worktree").exists());
13569 assert_eq!(
13570 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13571 "* main"
13572 );
13573 }
13574
13575 #[tokio::test]
13576 async fn preparation_records_creation_intent_before_git_can_fail() {
13577 ask_test_home();
13578 let tmp = tempfile::tempdir().unwrap();
13579 let repo = tmp.path().join("repo");
13580 std::fs::create_dir_all(&repo).unwrap();
13581 init_repo(&repo);
13582 let mut runner = runner_at(RunStatus::Prep);
13583 runner.state.repo = repo.clone();
13584 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13585 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13586 runner.state.config.blind.commit_msg_hook = false;
13587 runner.roles.implementers = vec![spec("one")];
13588 let label = blind::assign_labels(1, runner.state.seed)[0];
13589 let branch = runner.state.branch_for(label);
13590 git::git(&repo, &["branch", &branch]).await.unwrap();
13593 assert!(runner.prep().await.is_err());
13594 let saved = RunState::load(&runner.state.id).unwrap();
13595 assert!(saved.candidates.is_empty());
13596 let event = saved
13597 .events
13598 .iter()
13599 .find(|e| e.node == "prep-intent")
13600 .unwrap();
13601 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13602 assert_eq!(intent["branch"], branch);
13603 assert_eq!(intent["start"], runner.state.base_commit);
13604 assert_eq!(
13605 intent["worktree"],
13606 runner
13607 .state
13608 .worktree_root()
13609 .join(format!("cand-{label}"))
13610 .to_string_lossy()
13611 .as_ref()
13612 );
13613 assert_eq!(intent["picks"], serde_json::json!([]));
13614 }
13615
13616 #[tokio::test]
13617 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13618 ask_test_home();
13619 let tmp = tempfile::tempdir().unwrap();
13620 let repo = tmp.path().join("repo");
13621 std::fs::create_dir_all(&repo).unwrap();
13622 init_repo(&repo);
13623 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13624 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13625 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13626 git::git(&repo, &["add", "."]).await.unwrap();
13627 git::git(&repo, &["commit", "-m", "required seed"])
13628 .await
13629 .unwrap();
13630 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13631 git::git(&repo, &["checkout", "main"]).await.unwrap();
13632 let mut runner = runner_at(RunStatus::Prep);
13633 runner.state.repo = repo.clone();
13634 runner.state.base_commit = base.clone();
13635 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13636 runner.state.config.graph.advise = false;
13637 runner.state.config.blind.commit_msg_hook = false;
13638 runner.state.seeds = vec![refs::Seed {
13639 token: seed.clone(),
13640 sha: seed.clone(),
13641 kind: refs::SeedKind::Unmerged,
13642 branch: false,
13643 detail: String::new(),
13644 }];
13645 runner.roles.implementers = vec![spec("one"), spec("two")];
13646 runner.roles.judges = vec![spec("judge")];
13647 let label = blind::assign_labels(2, runner.state.seed)[0];
13648 let branch = runner.state.branch_for(label);
13649 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13650 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13651 .await
13652 .unwrap();
13653 runner.state.candidates.push(Candidate {
13654 index: 0,
13655 label,
13656 agent: "one".into(),
13657 branch,
13658 worktree,
13659 summary: String::new(),
13660 stat: String::new(),
13661 files: 0,
13662 commits: 0,
13663 empty: false,
13664 failed: None,
13665 verified_noop: None,
13666 duration_ms: 0,
13667 folded: false,
13668 });
13669 runner.state.save().unwrap();
13670 runner.state = RunState::load(&runner.state.id).unwrap();
13672 assert_eq!(runner.state.prep_complete, Some(false));
13673 runner.prep().await.unwrap();
13674 assert_eq!(runner.state.prep_complete, Some(true));
13675 assert_eq!(runner.state.candidates.len(), 2);
13676 for candidate in &runner.state.candidates {
13677 assert_eq!(
13678 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13679 git::tree_of(&repo, &seed).await.unwrap()
13680 );
13681 assert_eq!(
13682 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13683 "required commit"
13684 );
13685 }
13686 assert!(runner.state.worktree_root().join("judge-1").exists());
13687 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13688 runner.prep().await.unwrap();
13689 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13690
13691 runner.state.prep_complete = Some(false);
13692 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13693 run: Some(format!(
13694 "test -f '{}'",
13695 tmp.path().join("dependency-ready").display()
13696 )),
13697 ..Default::default()
13698 }];
13699 assert!(runner.prep().await.is_err());
13700 assert!(runner.state.prep_failure.is_some());
13701 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13702 runner.state = RunState::load(&runner.state.id).unwrap();
13703 assert!(
13704 runner.prep().await.is_err(),
13705 "correcting the cause alone must not auto-retry"
13706 );
13707 let failed_at = runner.state.events.last().unwrap().at;
13708 assert!(
13709 runner.retry_preparation(failed_at).is_err(),
13710 "old approvals cannot unlock the hold"
13711 );
13712 let mut task = crate::queue::Task::new(
13713 "retry".into(),
13714 "retry setup".into(),
13715 repo.clone(),
13716 crate::queue::Source::Human,
13717 );
13718 task.start(runner.state.id.clone());
13719 task.hold_machine(Some("setup failed".into()));
13720 task.release_operator("cli", None);
13721 runner
13722 .retry_preparation(task.resumption.as_ref().unwrap().at)
13723 .unwrap();
13724 runner.prep().await.unwrap();
13725 assert!(runner.state.prep_failure.is_none());
13726 assert_eq!(runner.state.prep_complete, Some(true));
13727 assert_eq!(runner.state.candidates.len(), 2);
13728 assert!(runner.state.worktree_root().join("judge-1").exists());
13729 for candidate in &runner.state.candidates {
13730 assert_eq!(
13731 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13732 git::tree_of(&repo, &seed).await.unwrap()
13733 );
13734 }
13735 }
13736
13737 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13738 let mut heads = Vec::new();
13739 for candidate in candidates {
13740 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13741 }
13742 heads
13743 }
13744
13745 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13746 ask_test_home();
13747 let tmp = tempfile::tempdir().unwrap();
13748 let repo = tmp.path().join("repo");
13749 std::fs::create_dir(&repo).unwrap();
13750 init_repo(&repo);
13751 let mut runner = runner_at(RunStatus::Reviewing);
13752 runner.state.repo = repo.clone();
13753 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13754 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13755 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13756 let branch = format!("candidate-{label}");
13757 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13758 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13759 .await
13760 .unwrap();
13761 if label == 'B' {
13762 std::fs::write(worktree.join("change"), "winner").unwrap();
13763 git::rescue_commit(&worktree, "candidate change")
13764 .await
13765 .unwrap();
13766 }
13767 runner.state.candidates.push(Candidate {
13768 index,
13769 label,
13770 agent: "alpha".into(),
13771 branch,
13772 worktree,
13773 summary: "completed reply".into(),
13774 stat: String::new(),
13775 files: usize::from(label == 'B'),
13776 commits: usize::from(label == 'B'),
13777 empty: label == 'A',
13778 failed: None,
13779 verified_noop: None,
13780 duration_ms: 1,
13781 folded: false,
13782 });
13783 }
13784 runner.state.judge_skipped = true;
13785 runner.state.tally = Some(Tally {
13786 first_choice: BTreeMap::new(),
13787 borda: BTreeMap::new(),
13788 winner: 'B',
13789 rankings: 0,
13790 unanimous_initial: false,
13791 deliberated: false,
13792 changed_votes: 0,
13793 unanimous_final: false,
13794 tie_break: None,
13795 judges: 0,
13796 present: 0,
13797 quorum: 0,
13798 met_quorum: true,
13799 uncontested: Some("only B produced a change".into()),
13800 });
13801 runner.state.save().unwrap();
13802 (tmp, runner)
13803 }
13804
13805 #[tokio::test]
13806 #[cfg(unix)]
13807 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13808 for (solo, failed_checkpoint) in
13809 [(true, false), (false, false), (true, true), (false, true)]
13810 {
13811 let (_tmp, mut runner) = preservation_resume_fixture().await;
13812 if solo {
13813 runner.state.candidates.retain(|c| c.label == 'A');
13814 }
13815 runner.state.candidates[0].empty = false;
13816 runner.state.candidates[0].summary.clear();
13817 runner.state.tally = None;
13818 runner.state.judge_skipped = false;
13819 runner.state.status = RunStatus::Implementing;
13820 runner.state.config.disk.min_free_bytes = u64::MAX;
13821 let mut implementer = spec("alpha");
13822 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13823 runner.roles.implementers = vec![implementer];
13824 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13825 if failed_checkpoint {
13826 std::fs::create_dir_all(&patch).unwrap();
13827 }
13828 runner.state.save().unwrap();
13829 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13830 runner.state = RunState::load(&runner.state.id).unwrap();
13831 assert_eq!(runner.state.status, RunStatus::Blocked);
13832 assert!(runner.state.interrupted_implementations.contains(&'A'));
13833 assert_eq!(runner.state.candidates[0].commits, 0);
13834 assert!(!runner.state.candidates[0].empty);
13835 assert!(runner.state.candidates[0].failed.is_none());
13836 if failed_checkpoint {
13837 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13838 std::fs::remove_dir(patch).unwrap();
13839 }
13840 runner.state.resource_block = None;
13841 runner.state.config.disk.min_free_bytes = 0;
13842 runner.roles.implementers[0].command = vec![
13843 "sh".into(),
13844 "-c".into(),
13845 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13846 ];
13847 runner.implement().await.unwrap();
13848 assert!(runner.state.interrupted_implementations.is_empty());
13849 assert!(runner.state.candidate_preservation.is_empty());
13850 assert_eq!(runner.state.candidates[0].commits, 1);
13851 assert!(!runner.state.candidates[0].empty);
13852 assert!(
13853 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13854 .unwrap()
13855 .contains("retried empty interrupted seat")
13856 );
13857 if !solo {
13858 assert_eq!(runner.state.candidates[1].commits, 1);
13859 }
13860 }
13861 }
13862
13863 #[tokio::test]
13864 #[cfg(unix)]
13865 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13866 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13867 let (_tmp, mut runner) = preservation_resume_fixture().await;
13868 runner.state.candidates.retain(|c| c.label == 'A');
13869 runner.state.candidates[0].empty = false;
13870 runner.state.candidates[0].summary.clear();
13871 runner.state.tally = None;
13872 runner.state.judge_skipped = false;
13873 runner.state.status = RunStatus::Implementing;
13874 runner.state.config.disk.min_free_bytes = u64::MAX;
13877 let wt = runner.state.candidates[0].worktree.clone();
13878 std::fs::write(
13879 wt.join("interrupted-source"),
13880 "retain implementation work\n",
13881 )
13882 .unwrap();
13883 if committed {
13884 git::rescue_commit(&wt, "implementation checkpoint")
13885 .await
13886 .unwrap();
13887 }
13888 let lock = git::git(
13889 &wt,
13890 &[
13891 "rev-parse",
13892 "--path-format=absolute",
13893 "--git-path",
13894 "index.lock",
13895 ],
13896 )
13897 .await
13898 .unwrap();
13899 if block_rescue {
13900 std::fs::write(&lock, "fixture-owned lock").unwrap();
13901 }
13902 let mut implementer = spec("alpha");
13903 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13904 runner.roles.implementers = vec![implementer];
13905 runner.state.save().unwrap();
13906 let result = runner.implement().await;
13907 assert_eq!(result.is_err(), block_rescue);
13908 assert_eq!(
13909 runner.state.resource_block.as_ref().unwrap().kind,
13910 "disk pressure"
13911 );
13912 assert!(
13913 runner
13914 .state
13915 .events
13916 .iter()
13917 .any(|e| e.message.contains("agent execution interrupted"))
13918 );
13919 assert!(
13920 runner
13921 .state
13922 .dir()
13923 .join("artifacts/candidate-A-recovery/staged.patch")
13924 .exists()
13925 );
13926 if block_rescue {
13927 let saved = RunState::load(&runner.state.id).unwrap();
13928 assert!(saved.candidate_preservation.contains_key(&'A'));
13929 assert!(!saved.candidates[0].empty);
13930 assert_eq!(
13931 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13932 "retain implementation work\n"
13933 );
13934 std::fs::remove_file(lock).unwrap();
13935 runner.state = saved;
13936 } else {
13937 assert_eq!(runner.state.status, RunStatus::Blocked);
13938 assert_eq!(runner.state.candidates[0].commits, 1);
13939 }
13940 runner.roles.implementers.clear();
13943 runner.state.config.disk.min_free_bytes = 0;
13944 runner.state.resource_block = None;
13945 runner.implement().await.unwrap();
13946 assert!(runner.state.candidate_preservation.is_empty());
13947 assert_eq!(runner.state.candidates[0].commits, 1);
13948 assert!(!runner.state.candidates[0].empty);
13949 assert!(git::is_clean(&wt).await.unwrap());
13950 assert!(
13951 crate::run::read_artifact(&runner.state, "cand-A.patch")
13952 .unwrap()
13953 .contains("retain implementation work")
13954 );
13955 }
13956 }
13957
13958 #[tokio::test]
13959 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13960 let (_tmp, mut runner) = preservation_resume_fixture().await;
13961 runner.implement().await.unwrap();
13963 assert!(runner.state.candidate_preservation.is_empty());
13964 assert!(
13965 !runner
13966 .state
13967 .dir()
13968 .join("artifacts/candidate-A-recovery")
13969 .exists()
13970 );
13971 runner.fold_losers().await.unwrap();
13972 assert!(!runner.state.candidates[0].worktree.exists());
13973 for schema in [14, crate::run::SCHEMA] {
13974 runner.state.schema = schema;
13975 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13976 runner.state.save().unwrap();
13977 runner.state = RunState::load(&runner.state.id).unwrap();
13978 runner.implement().await.unwrap();
13979 assert!(runner.state.candidate_preservation.is_empty());
13980 assert!(runner.state.candidates[0].folded);
13981 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13982 }
13983 }
13984
13985 #[tokio::test]
13986 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13987 let (_tmp, mut runner) = preservation_resume_fixture().await;
13988 let wt = runner.state.candidates[0].worktree.clone();
13989 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13990 runner.state.schema = 14;
13991 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13992 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13993 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13994 for _ in 0..2 {
13995 assert!(runner.implement().await.is_err());
13996 assert_eq!(runner.state.status, RunStatus::Blocked);
13997 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13998 assert!(
13999 runner.state.candidates[0].empty,
14000 "old verdict must remain unusable"
14001 );
14002 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14003 assert!(runner.state.judge_skipped);
14004 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14005 assert!(
14006 crate::run::read_artifact(&runner.state, "cand-A.patch")
14007 .unwrap()
14008 .contains("owned edits")
14009 );
14010 assert!(wt.exists());
14011 assert_eq!(
14012 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14013 .await
14014 .unwrap(),
14015 1
14016 );
14017 runner.state = RunState::load(&runner.state.id).unwrap();
14018 }
14019 }
14020
14021 #[tokio::test]
14022 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14023 let (_tmp, mut runner) = preservation_resume_fixture().await;
14024 let wt = runner.state.candidates[0].worktree.clone();
14025 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14026 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14027 let index_lock = PathBuf::from(
14028 git::git(
14029 &wt,
14030 &[
14031 "rev-parse",
14032 "--path-format=absolute",
14033 "--git-path",
14034 "index.lock",
14035 ],
14036 )
14037 .await
14038 .unwrap(),
14039 );
14040 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14041 runner.state.schema = 14;
14042 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14043 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14044 assert!(runner.implement().await.is_err());
14045 assert!(runner.after_implement().is_err());
14046 assert!(runner.fold_losers().await.is_err());
14047 assert_eq!(
14048 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14049 "failed seat edits"
14050 );
14051 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14052 runner.state = RunState::load(&runner.state.id).unwrap();
14053 std::fs::remove_file(index_lock).unwrap();
14054 assert!(runner.implement().await.is_err());
14056 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14057 assert!(runner.state.candidates[0].empty);
14058 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14059 assert!(
14060 crate::run::read_artifact(&runner.state, "cand-A.patch")
14061 .unwrap()
14062 .contains("failed seat edits")
14063 );
14064 assert!(wt.exists());
14065 }
14066
14067 #[tokio::test]
14068 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14069 let (_tmp, mut runner) = preservation_resume_fixture().await;
14070 let repo = runner.state.repo.clone();
14071 let common = git::git(
14072 &repo,
14073 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14074 )
14075 .await
14076 .unwrap();
14077 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14078 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14079 assert!(runner.fold_losers().await.is_err());
14080 assert!(!runner.state.candidates[0].worktree.exists());
14081 assert!(!runner.state.candidates[0].folded);
14082 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14083 runner.state = RunState::load(&runner.state.id).unwrap();
14084 std::fs::remove_file(branch_lock).unwrap();
14085 let judge = runner.state.worktree_root().join("judge-1");
14086 git::worktree_add_detached(&repo, &judge, "main")
14087 .await
14088 .unwrap();
14089 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14090 runner.roles.judges = vec![spec("judge")];
14091 assert!(runner.fold_losers().await.is_err());
14092 runner.state = RunState::load(&runner.state.id).unwrap();
14093 assert!(
14094 runner.state.candidates[0].folded,
14095 "candidate progress saved before judge failure"
14096 );
14097 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14098 assert!(judge.join("uncommitted").exists());
14099 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14100 runner.fold_losers().await.unwrap();
14101 assert!(!judge.exists());
14102 runner.fold_losers().await.unwrap();
14103 }
14104
14105 #[tokio::test]
14106 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14107 let (_tmp, mut runner) = preservation_resume_fixture().await;
14108 runner.state.candidates.retain(|c| c.label == 'B');
14109 runner.state.tally = None;
14110 runner.state.judge_skipped = false;
14111 runner.state.prep_complete = Some(true);
14112 runner.state.config.graph.advise = false;
14113 runner.state.config.graph.selected_implementer = Some("alpha".into());
14114 let wt = runner.state.candidates[0].worktree.clone();
14115 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14116 runner
14117 .state
14118 .candidate_preservation
14119 .insert('B', "awaiting persistence".into());
14120 let index_lock = git::git(
14121 &wt,
14122 &[
14123 "rev-parse",
14124 "--path-format=absolute",
14125 "--git-path",
14126 "index.lock",
14127 ],
14128 )
14129 .await
14130 .unwrap();
14131 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14132 runner.state.save().unwrap();
14133 assert!(runner.execute_graph().await.is_err());
14134 assert_eq!(runner.state.status, RunStatus::Blocked);
14135 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14136 std::fs::remove_file(index_lock).unwrap();
14137 for _ in 0..2 {
14140 runner.state = RunState::load(&runner.state.id).unwrap();
14141 runner.pause.park();
14142 runner.execute_graph().await.unwrap();
14143 assert_eq!(runner.state.status, RunStatus::Implementing);
14144 assert!(runner.state.candidate_preservation.is_empty());
14145 assert_eq!(runner.state.candidates[0].commits, 2);
14146 assert!(runner.state.tally.is_none());
14147 }
14148 runner.implement().await.unwrap();
14149 assert_eq!(runner.state.status, RunStatus::Judging);
14150 runner.state.status = RunStatus::Blocked;
14152 runner.state.candidates[0].failed = Some("implementation failed".into());
14153 runner.state.save().unwrap();
14154 let error = runner.execute_graph().await.unwrap_err();
14155 assert!(
14156 error
14157 .to_string()
14158 .contains("selected implementation is held")
14159 );
14160 }
14161
14162 #[tokio::test]
14163 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14164 let (_tmp, mut runner) = preservation_resume_fixture().await;
14165 let repo = runner.state.repo.clone();
14166 let candidate = runner.state.candidates[0].clone();
14167 let registered = git::worktree_holding(&repo, &candidate.branch)
14170 .await
14171 .unwrap()
14172 .unwrap();
14173 assert_eq!(
14174 registered.canonicalize().unwrap(),
14175 candidate.worktree.canonicalize().unwrap()
14176 );
14177 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14178 assert_eq!(
14179 git::worktree_holding(&repo, &candidate.branch)
14180 .await
14181 .unwrap(),
14182 Some(registered)
14183 );
14184 assert!(
14185 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14186 "the stale registration initially keeps the branch checked out"
14187 );
14188 runner.fold_losers().await.unwrap();
14189 assert!(runner.state.candidates[0].folded);
14190 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14191 assert_eq!(
14192 git::worktree_holding(&repo, &candidate.branch)
14193 .await
14194 .unwrap(),
14195 None
14196 );
14197 runner.state = RunState::load(&runner.state.id).unwrap();
14198 runner.fold_losers().await.unwrap();
14199 }
14200
14201 #[tokio::test]
14202 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14203 let (_tmp, mut runner) = preservation_resume_fixture().await;
14204 let repo = runner.state.repo.clone();
14205 let candidate = runner.state.candidates[0].clone();
14206 let winner = runner.state.candidates[1].clone();
14207 let registered = git::worktree_holding(&repo, &candidate.branch)
14208 .await
14209 .unwrap()
14210 .unwrap();
14211 assert_eq!(
14212 registered.canonicalize().unwrap(),
14213 candidate.worktree.canonicalize().unwrap()
14214 );
14215 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14216 assert_eq!(
14217 git::worktree_holding(&repo, &candidate.branch)
14218 .await
14219 .unwrap(),
14220 Some(registered)
14221 );
14222 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14223
14224 for _ in 0..2 {
14225 fold_run(&mut runner.state, false, &crate::run::home())
14226 .await
14227 .unwrap();
14228 runner.state = RunState::load(&runner.state.id).unwrap();
14229 assert!(runner.state.candidates[0].folded);
14230 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14231 assert_eq!(
14232 git::worktree_holding(&repo, &candidate.branch)
14233 .await
14234 .unwrap(),
14235 None
14236 );
14237 assert!(!runner.state.candidates[1].folded);
14238 assert!(winner.worktree.exists());
14239 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14240 }
14241 }
14242
14243 #[tokio::test]
14244 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14245 ask_test_home();
14246 let tmp = tempfile::tempdir().unwrap();
14247 let repo = tmp.path().join("repo");
14248 std::fs::create_dir(&repo).unwrap();
14249 init_repo(&repo);
14250 let mut runner = runner_at(RunStatus::Implementing);
14251 runner.state.repo = repo.clone();
14252 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14253 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14254 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14255 let branch = format!("candidate-{label}");
14256 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14257 git::worktree_add_branch(&repo, &wt, &branch, "main")
14258 .await
14259 .unwrap();
14260 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14261 git::git(&wt, &["add", "README.md"]).await.unwrap();
14262 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14263 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14264 runner.state.candidates.push(Candidate {
14265 index,
14266 label,
14267 agent: "alpha".to_owned(),
14268 branch,
14269 worktree: wt,
14270 summary: "completed implementation".to_owned(),
14271 stat: String::new(),
14272 files: 0,
14273 commits: 0,
14274 empty: false,
14275 failed: None,
14276 verified_noop: None,
14277 duration_ms: 42,
14278 folded: false,
14279 });
14280 runner
14281 .state
14282 .candidate_preservation
14283 .insert(label, "awaiting persistence".to_owned());
14284 }
14285 let wt = runner.state.candidates[0].worktree.clone();
14286 let index_lock = git::git(
14287 &wt,
14288 &[
14289 "rev-parse",
14290 "--path-format=absolute",
14291 "--git-path",
14292 "index.lock",
14293 ],
14294 )
14295 .await
14296 .unwrap();
14297 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14298 runner.state.save().unwrap();
14299 assert!(runner.persist_candidates().await.is_err());
14300 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14301 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14302 assert_eq!(
14303 runner.state.candidates[1].commits, 1,
14304 "other results are still collected"
14305 );
14306 assert!(
14307 !runner.state.candidates[0].empty,
14308 "failed rescue is not no change"
14309 );
14310 assert_eq!(
14311 runner.state.candidates[0].summary,
14312 "completed implementation"
14313 );
14314 assert!(runner.after_implement().is_err());
14315 let home = tmp.path().join("fold-home");
14316 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14317 assert!(wt.join("new-file").exists());
14318 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14319 assert!(recovery.join("staged.patch").exists());
14320 assert!(recovery.join("unstaged.patch").exists());
14321 assert_eq!(
14322 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14323 "untracked edits\n"
14324 );
14325 runner.state.begin_pass();
14326 runner.state.end_pass_parked(ParkCause::Upgrade {
14327 from: "1.0".into(),
14328 to: Some("1.1".into()),
14329 });
14330 runner.state.save().unwrap();
14331 runner.state = RunState::load(&runner.state.id).unwrap();
14332 assert!(matches!(
14333 runner.state.passes.last().unwrap().end,
14334 Some(crate::run::PassEnd::Parked {
14335 cause: ParkCause::Upgrade { .. }
14336 })
14337 ));
14338 std::fs::remove_file(index_lock).unwrap();
14339 runner.implement().await.unwrap();
14341 assert!(runner.state.candidate_preservation.is_empty());
14342 assert!(
14343 runner
14344 .state
14345 .candidates
14346 .iter()
14347 .all(|c| c.commits == 1 && !c.empty)
14348 );
14349 assert!(git::is_clean(&wt).await.unwrap());
14350 assert!(
14351 crate::run::read_artifact(&runner.state, "cand-A.patch")
14352 .unwrap()
14353 .contains("later tracked edits")
14354 );
14355 }
14356
14357 #[tokio::test]
14358 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14359 ask_test_home();
14360 let tmp = tempfile::tempdir().unwrap();
14361 let repo = tmp.path().join("repo");
14362 std::fs::create_dir(&repo).unwrap();
14363 init_repo(&repo);
14364 let wt = tmp.path().join("candidate");
14365 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14366 .await
14367 .unwrap();
14368 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14369 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14370 assert!(wt.join("uncommitted").exists());
14371 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14372 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14373 let mut runner = runner_at(RunStatus::Implementing);
14374 runner.state.repo = repo.clone();
14375 runner.state.base_commit = "missing-base".to_owned();
14376 runner.state.candidates.push(Candidate {
14377 index: 0,
14378 label: 'A',
14379 agent: "alpha".into(),
14380 branch: "candidate".into(),
14381 worktree: wt.clone(),
14382 summary: "completed".into(),
14383 stat: String::new(),
14384 files: 0,
14385 commits: 0,
14386 empty: false,
14387 failed: None,
14388 verified_noop: None,
14389 duration_ms: 1,
14390 folded: false,
14391 });
14392 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14393 assert!(
14394 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14395 .await
14396 .is_err()
14397 );
14398 assert!(!runner.state.candidates[0].folded);
14399 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14400 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14401 runner
14402 .state
14403 .candidate_preservation
14404 .insert('A', "pending".into());
14405 assert!(runner.persist_candidates().await.is_err());
14406 assert!(!runner.state.candidates[0].empty);
14407 assert!(!runner.state.candidates[0].folded);
14408 assert!(wt.exists());
14409 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14410 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14411 std::fs::create_dir_all(&patch_path).unwrap();
14412 assert!(
14413 runner.persist_candidates().await.is_err(),
14414 "artifact failure stays pending"
14415 );
14416 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14417 assert!(!runner.state.candidates[0].empty);
14418 std::fs::remove_dir(patch_path).unwrap();
14419 runner.persist_candidates().await.unwrap();
14420 assert!(
14421 runner.state.candidates[0].empty,
14422 "only successful inspection confirms no change"
14423 );
14424 }
14425
14426 fn init_repo(dir: &Path) {
14427 let run = |args: &[&str]| {
14428 let out = std::process::Command::new("git")
14429 .args(args)
14430 .current_dir(dir)
14431 .quiet()
14432 .output()
14433 .expect("spawn git");
14434 assert!(
14435 out.status.success(),
14436 "git {args:?} failed: {}",
14437 String::from_utf8_lossy(&out.stderr)
14438 );
14439 };
14440 run(&["init", "-b", "main"]);
14441 run(&["config", "user.name", "magi test"]);
14442 run(&["config", "user.email", "magi@example.com"]);
14443 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14444 run(&["add", "-A"]);
14445 run(&["commit", "-m", "init"]);
14446 }
14447
14448 fn ask_test_home() {
14456 crate::run::pin_test_home();
14457 }
14458
14459 fn runner_at(status: RunStatus) -> Runner {
14462 let mut state = RunState::new(
14463 PathBuf::from("/nonexistent/repo"),
14464 "main".to_owned(),
14465 "deadbeef".to_owned(),
14466 "task".to_owned(),
14467 Config::default(),
14468 );
14469 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14472 state.id = format!(
14473 "20000101-000000-{:04x}",
14474 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14475 );
14476 state.status = status;
14477 Runner {
14478 state,
14479 roles: ResolvedRoles {
14480 implementers: Vec::new(),
14481 judges: Vec::new(),
14482 reviewers: Vec::new(),
14483 fixer: None,
14484 conductor: conductor(),
14485 implementer_roster: Vec::new(),
14486 judge_roster: Vec::new(),
14487 reviewer_roster: Vec::new(),
14488 },
14489 sem: Arc::new(Semaphore::new(1)),
14490 pause: Pause::new(),
14491 interrupt: Pause::new(),
14492 }
14493 }
14494
14495 #[test]
14496 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14497 crate::run::pin_test_home();
14498 let mut runner = runner_at(RunStatus::Gating);
14499 runner.state.save().unwrap();
14500 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14503 assert!(runner.claim_driver().is_err());
14504 assert!(runner.state.driver_exited);
14505 assert_eq!(runner.state.driver_pid, None);
14506 let disk = RunState::load(&runner.state.id).unwrap();
14507 assert!(disk.driver_exited);
14508 assert_eq!(disk.driver_pid, None);
14509 assert_eq!(disk.status, RunStatus::Gating);
14510 }
14511
14512 #[tokio::test]
14513 async fn an_early_graph_error_releases_its_driver() {
14514 crate::run::pin_test_home();
14515 let mut runner = runner_at(RunStatus::Blocked);
14516 runner.state.config.graph.selected_implementer = Some("selected".into());
14517 runner.state.save().unwrap();
14518 assert!(runner.execute().await.is_err());
14519 let disk = RunState::load(&runner.state.id).unwrap();
14520 assert!(disk.driver_exited);
14521 assert_eq!(disk.driver_pid, None);
14522 assert_eq!(disk.status, RunStatus::Blocked);
14523 }
14524
14525 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14526 crate::run::MergeRepair {
14527 question: "repair-question".into(),
14528 head: "old-head".into(),
14529 pr: "https://github.com/o/r/pull/42".into(),
14530 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14531 gate: vec![outcome(Some(0), false)],
14532 gate_head: Some("old-head".into()),
14533 gate_fixes: Vec::new(),
14534 pre_gate: Vec::new(),
14535 phase,
14536 failure: None,
14537 }
14538 }
14539
14540 #[tokio::test]
14541 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14542 crate::run::pin_test_home();
14543 let mut runner = runner_at(RunStatus::Landing);
14544 runner.state.merge_repairs.push(merge_repair_request(
14545 crate::run::MergeRepairPhase::Requested,
14546 ));
14547 runner.state.save().unwrap();
14548 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14549 assert!(
14550 runner
14551 .execute()
14552 .await
14553 .unwrap_err()
14554 .to_string()
14555 .contains("already running")
14556 );
14557 assert_eq!(
14558 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14559 crate::run::MergeRepairPhase::Requested
14560 );
14561 drop(claim);
14562 let mut disk = runner.state.clone();
14563 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14564 disk.save().unwrap();
14565 assert!(
14566 runner
14567 .execute()
14568 .await
14569 .unwrap_err()
14570 .to_string()
14571 .contains("decision changed")
14572 );
14573 assert_eq!(
14574 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14575 crate::run::MergeRepairPhase::Complete
14576 );
14577 }
14578
14579 #[test]
14580 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14581 crate::run::pin_test_home();
14582 let mut runner = runner_at(RunStatus::Landing);
14583 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14584 runner.state.reviews = request.reviews.clone();
14585 runner.state.gate = request.gate.clone();
14586 runner.state.gate_ran = true;
14587 runner.state.land_approval = Some(crate::run::LandApproval {
14588 question: "old".into(),
14589 head: "old-head".into(),
14590 });
14591 runner.state.merge_repairs.push(request);
14592 runner.begin_merge_repair_review(0).unwrap();
14593 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14594 assert!(!runner.state.gate_status().ok());
14595 assert!(runner.state.land_approval.is_none());
14596 let disk = RunState::load(&runner.state.id).unwrap();
14597 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14598 assert!(disk.merge_repairs[0].gate[0].ok());
14599 assert_eq!(
14600 disk.merge_repairs[0].phase,
14601 crate::run::MergeRepairPhase::Reviewing
14602 );
14603 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14604 assert!(disk.reviews.is_empty());
14605 }
14606
14607 #[tokio::test]
14608 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14609 crate::run::pin_test_home();
14610 for phase in [
14611 crate::run::MergeRepairPhase::Fixing,
14612 crate::run::MergeRepairPhase::Requested,
14613 ] {
14614 let mut runner = runner_at(RunStatus::Landing);
14615 runner.state.config.graph.land_rounds = 0;
14616 runner.state.merge_repairs.push(merge_repair_request(phase));
14617 runner.state.save().unwrap();
14618 assert!(!runner.repair_merge_review().await.unwrap());
14619 assert_eq!(runner.state.status, RunStatus::Blocked);
14620 assert_eq!(
14621 runner.state.merge_repairs[0].phase,
14622 crate::run::MergeRepairPhase::Failed
14623 );
14624 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14625 assert!(
14626 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14627 "interrupted"
14628 } else {
14629 "budget exhausted"
14630 })
14631 );
14632 assert!(runner.state.merge.is_none());
14633 }
14634 }
14635
14636 #[test]
14640 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14641 crate::run::pin_test_home();
14642 let mut runner = runner_at(RunStatus::Implementing);
14643 let interrupt = Pause::new();
14644 runner.watch_interrupt(interrupt.clone());
14645
14646 interrupt.park_because("task a1b2 asked to run first");
14647
14648 assert!(runner.park_here().expect("park_here"));
14649 assert!(runner.state.parked);
14650 let last = runner.state.events.last().expect("a park event");
14651 assert_eq!(last.node, "park");
14652 assert!(
14653 last.message.contains("task a1b2 asked to run first"),
14654 "expected the interrupt reason in {:?}",
14655 last.message
14656 );
14657 }
14658
14659 #[test]
14662 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14663 crate::run::pin_test_home();
14664 let mut runner = runner_at(RunStatus::Implementing);
14665 runner.state.begin_pass();
14666 let shutdown = Pause::new();
14667 runner.on_pause(shutdown.clone());
14668 let upgrade = ParkCause::Upgrade {
14669 from: "0.121.0".to_owned(),
14670 to: Some("0.121.1".to_owned()),
14671 };
14672 shutdown.park_for(upgrade.clone());
14673 shutdown.park_for(ParkCause::Operator);
14674
14675 assert!(runner.park_here().expect("park_here"));
14676 assert_eq!(
14677 runner.state.passes[0].end,
14678 Some(crate::run::PassEnd::Parked { cause: upgrade })
14679 );
14680 let last = runner.state.events.last().expect("a park event");
14681 assert!(last.message.contains("0.121.0"), "{}", last.message);
14682 }
14683
14684 #[test]
14692 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14693 crate::run::pin_test_home();
14694 let mut runner = runner_at(RunStatus::Implementing);
14695 let shutdown = Pause::new();
14696 runner.on_pause(shutdown.clone());
14697 let interrupt = Pause::new();
14698 runner.watch_interrupt(interrupt.clone());
14699
14700 assert!(!runner.park_here().expect("park_here"));
14702 assert!(!runner.state.parked);
14703
14704 interrupt.park_because("test");
14706 assert!(!shutdown.parked());
14707 assert!(runner.park_here().expect("park_here"));
14708 }
14709
14710 #[tokio::test]
14724 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14725 crate::run::pin_test_home();
14726 let mut runner = runner_at(RunStatus::Implementing);
14727 let interrupt = Pause::new();
14728 runner.watch_interrupt(interrupt.clone());
14729
14730 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14731 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14732
14733 let node = async move {
14737 started_tx.send(()).expect("send started");
14738 finish_rx.await.expect("recv finish");
14739 "node finished"
14740 };
14741
14742 let interrupter = async move {
14743 started_rx.await.expect("recv started");
14744 interrupt.park_because("higher-priority task waiting");
14746 tokio::task::yield_now().await;
14750 finish_tx.send(()).expect("send finish");
14751 };
14752
14753 let (node_result, ()) = tokio::join!(node, interrupter);
14754 assert_eq!(
14755 node_result, "node finished",
14756 "the in-flight call ran to completion"
14757 );
14758
14759 assert!(runner.park_here().expect("park_here"));
14762 assert!(runner.state.parked);
14763 }
14764
14765 #[test]
14771 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14772 crate::run::pin_test_home();
14773 let mut runner = runner_at(RunStatus::Judging);
14774 runner.state.config.agents = vec![conductor()];
14778 runner.state.candidates = vec![Candidate {
14779 index: 0,
14780 label: 'A',
14781 agent: "alpha".to_owned(),
14782 branch: "magi/x/A".to_owned(),
14783 worktree: PathBuf::from("/nonexistent/worktree"),
14784 summary: "did the thing".to_owned(),
14785 stat: "1 file changed".to_owned(),
14786 files: 1,
14787 commits: 1,
14788 empty: false,
14789 failed: None,
14790 verified_noop: None,
14791 duration_ms: 1234,
14792 folded: false,
14793 }];
14794 let run_id = runner.state.id.clone();
14795
14796 let interrupt = Pause::new();
14797 runner.watch_interrupt(interrupt.clone());
14798 interrupt.park_because("task c3d4 asked to run first");
14799 assert!(runner.park_here().expect("park_here"));
14800
14801 let resumed = Runner::resume(&run_id).expect("resume");
14802 assert_eq!(resumed.state.candidates.len(), 1);
14803 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14804 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14805 assert_eq!(resumed.state.status, runner.state.status);
14806 assert!(
14807 resumed.state.parked,
14808 "still parked until `execute` actually walks the graph again"
14809 );
14810 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14811 }
14812
14813 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14815 let mut q = ask::Question::new(
14816 run.to_owned(),
14817 "implement".to_owned(),
14818 "impl-A".to_owned(),
14819 "Which storage backend should the cache use?".to_owned(),
14820 String::new(),
14821 vec!["SQLite".to_owned(), "Redis".to_owned()],
14822 );
14823 store.put(&mut q).unwrap();
14824 q
14825 }
14826
14827 #[test]
14828 fn a_failed_runs_open_question_is_abandoned() {
14829 ask_test_home();
14830 let store = ask::Questions::open();
14831 let mut runner = runner_at(RunStatus::Failed);
14832 let run = runner.state.id.clone();
14833 let q = ask_open_question(&store, &run);
14834
14835 runner.settle_questions();
14836
14837 let back = store.get(&q.id).unwrap();
14838 assert!(
14839 !back.status.open(),
14840 "the seat that asked died with the run; nobody is left to read an answer"
14841 );
14842 assert!(
14843 back.detail.contains(&run) && back.detail.contains("failed"),
14844 "the reason names what the run became, not just that it is gone: {}",
14845 back.detail
14846 );
14847 }
14848
14849 #[test]
14850 fn a_merged_runs_open_question_is_abandoned_too() {
14851 ask_test_home();
14852 let store = ask::Questions::open();
14853 for status in [RunStatus::Merged, RunStatus::Ready] {
14856 let mut runner = runner_at(status);
14857 let run = runner.state.id.clone();
14858 let q = ask_open_question(&store, &run);
14859
14860 runner.settle_questions();
14861
14862 let back = store.get(&q.id).unwrap();
14863 assert!(
14864 !back.status.open(),
14865 "{status:?} run's question must not outlive the run"
14866 );
14867 }
14868 }
14869
14870 #[test]
14871 fn a_still_resumable_runs_open_question_is_left_alone() {
14872 ask_test_home();
14873 let store = ask::Questions::open();
14874 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14880 let mut runner = runner_at(status);
14881 let run = runner.state.id.clone();
14882 let q = ask_open_question(&store, &run);
14883
14884 runner.settle_questions();
14885
14886 let back = store.get(&q.id).unwrap();
14887 assert!(
14888 back.status.open(),
14889 "{status:?} is still alive; the question must still be waiting"
14890 );
14891 }
14892 }
14893
14894 #[test]
14895 fn settle_questions_never_touches_an_already_answered_question() {
14896 ask_test_home();
14897 let store = ask::Questions::open();
14898 let mut runner = runner_at(RunStatus::Failed);
14899 let run = runner.state.id.clone();
14900 let mut q = ask_open_question(&store, &run);
14901 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14902 .unwrap();
14903 store.put(&mut q).unwrap();
14904
14905 runner.settle_questions();
14910 runner.settle_questions();
14911
14912 let back = store.get(&q.id).unwrap();
14913 assert_eq!(
14914 back.status,
14915 ask::QuestionStatus::Answered,
14916 "a real answer is a decision on record, never overwritten by a sweep"
14917 );
14918 }
14919
14920 #[tokio::test]
14931 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14932 crate::run::pin_test_home();
14933 let tmp = tempfile::tempdir().expect("tempdir");
14934 let repo = tmp.path().join("repo");
14935 std::fs::create_dir_all(&repo).unwrap();
14936 init_repo(&repo);
14937
14938 let mut config = Config::default();
14939 config.graph.worktree_root = Some(tmp.path().join("wt"));
14940
14941 let mut state = RunState::new(
14942 repo.clone(),
14943 "main".to_owned(),
14944 "deadbeef".to_owned(),
14945 "task".to_owned(),
14946 config,
14947 );
14948 let root = state.worktree_root();
14949 let wt_a = root.join("cand-A");
14950 let wt_b = root.join("cand-B");
14951 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14952 .await
14953 .expect("worktree A");
14954 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14955 .await
14956 .expect("worktree B");
14957
14958 state.candidates = vec![
14959 Candidate {
14960 index: 0,
14961 label: 'A',
14962 agent: "alpha".to_owned(),
14963 branch: "magi/x/A".to_owned(),
14964 worktree: wt_a.clone(),
14965 summary: String::new(),
14966 stat: String::new(),
14967 files: 0,
14968 commits: 0,
14969 empty: false,
14970 failed: None,
14971 verified_noop: None,
14972 duration_ms: 0,
14973 folded: false,
14974 },
14975 Candidate {
14976 index: 1,
14977 label: 'B',
14978 agent: "beta".to_owned(),
14979 branch: "magi/x/B".to_owned(),
14980 worktree: wt_b.clone(),
14981 summary: String::new(),
14982 stat: String::new(),
14983 files: 0,
14984 commits: 0,
14985 empty: false,
14986 failed: None,
14987 verified_noop: None,
14988 duration_ms: 0,
14989 folded: false,
14990 },
14991 ];
14992 state.tally = Some(Tally {
14993 first_choice: BTreeMap::from([('A', 1)]),
14994 borda: BTreeMap::new(),
14995 winner: 'A',
14996 rankings: 1,
14997 unanimous_initial: true,
14998 deliberated: false,
14999 changed_votes: 0,
15000 unanimous_final: true,
15001 tie_break: None,
15002 judges: 1,
15003 present: 1,
15004 quorum: 1,
15005 met_quorum: true,
15006 uncontested: None,
15007 });
15008 state.status = RunStatus::Ready;
15009
15010 fold_run(&mut state, false, &crate::run::home())
15011 .await
15012 .expect("fold_run");
15013
15014 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15015 assert!(
15016 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15017 "the unmerged winner's branch survives"
15018 );
15019 assert!(
15020 !state.candidates[0].folded,
15021 "the winner is not marked folded"
15022 );
15023
15024 assert!(!wt_b.exists(), "the loser's worktree is removed");
15025 assert!(
15026 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15027 "the loser's branch is removed"
15028 );
15029 assert!(state.candidates[1].folded, "the loser is marked folded");
15030 }
15031
15032 #[tokio::test]
15033 #[cfg(unix)]
15034 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15035 use std::os::fd::AsRawFd as _;
15036 use std::time::Duration;
15037
15038 let tmp = tempfile::tempdir().unwrap();
15039 let repo = tmp.path().join("repo");
15040 std::fs::create_dir_all(&repo).unwrap();
15041 init_repo(&repo);
15042 let common = repo.join(".git");
15043 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15044 let home = tmp.path().join("home");
15045 let mut config = Config::default();
15046 config.coordination = crate::config::test_coordination();
15047 config.graph.worktree_root = Some(tmp.path().join("wt"));
15048 config.graph.timeout_resource_wait = 1;
15049 let mut state = RunState::new(
15050 repo.clone(),
15051 "main".into(),
15052 "deadbeef".into(),
15053 "task".into(),
15054 config,
15055 );
15056 let root = state.worktree_root();
15057 let candidate = root.join("cand-A");
15058 let reviewer = root.join("review-1");
15059 let branch = state.branch_for('A');
15060 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15061 .await
15062 .unwrap();
15063 git::worktree_add_detached(&repo, &reviewer, "main")
15064 .await
15065 .unwrap();
15066 git::acquire_worktree_config(&repo).await.unwrap();
15067 state.enabled_worktree_config = true;
15068 state.status = RunStatus::Ready;
15069 state.candidates.push(Candidate {
15070 index: 0,
15071 label: 'A',
15072 agent: "implement".into(),
15073 branch: branch.clone(),
15074 worktree: candidate.clone(),
15075 summary: String::new(),
15076 stat: String::new(),
15077 files: 0,
15078 commits: 0,
15079 empty: false,
15080 failed: None,
15081 verified_noop: None,
15082 duration_ms: 0,
15083 folded: false,
15084 });
15085 state.save_under(&home).unwrap();
15086 let config_before = std::fs::read(common.join("config")).unwrap();
15087 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15088 .await
15089 .unwrap();
15090 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15093 assert_eq!(
15094 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15095 0
15096 );
15097 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15098 assert!(
15099 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15100 .await
15101 .is_err()
15102 );
15103 assert!(candidate.exists());
15104 assert!(reviewer.exists());
15105 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15106 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15107 assert_eq!(
15108 git::git(&repo, &["worktree", "list", "--porcelain"])
15109 .await
15110 .unwrap(),
15111 worktrees_before
15112 );
15113 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15114 .await
15115 .unwrap()
15116 .unwrap_err();
15117 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15118 drop(fold);
15119 assert!(candidate.exists() && reviewer.exists());
15120 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15121 assert_eq!(
15122 state.resource_block.as_ref().unwrap().kind,
15123 "repository coordination"
15124 );
15125 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15126 drop(external);
15127 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15128 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15129 .await
15130 .unwrap()
15131 .unwrap();
15132 drop(fold);
15133 assert!(removed.contains(&branch));
15134 assert!(!root.exists());
15135 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15136 assert!(state.candidates[0].folded);
15137 assert!(!state.enabled_worktree_config);
15138 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15139 let enabled = git::git_raw(
15140 &repo,
15141 &["config", "--local", "--get", "extensions.worktreeConfig"],
15142 )
15143 .await
15144 .unwrap();
15145 assert!(!enabled.ok(), "the last config reference was released");
15146 }
15147
15148 #[tokio::test]
15151 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15152 let tmp = tempfile::tempdir().expect("tempdir");
15153 let repo = tmp.path().join("repo");
15154 std::fs::create_dir_all(&repo).unwrap();
15155 init_repo(&repo);
15156 let home = tmp.path().join("home");
15157
15158 let mut config = Config::default();
15159 config.graph.worktree_root = Some(tmp.path().join("wt"));
15160 let mut state = RunState::new(
15161 repo.clone(),
15162 "main".to_owned(),
15163 "deadbeef".to_owned(),
15164 "task".to_owned(),
15165 config,
15166 );
15167 git::git(&repo, &["branch", "magi/x/A", "main"])
15169 .await
15170 .expect("branch");
15171 state.candidates = vec![Candidate {
15172 index: 0,
15173 label: 'A',
15174 agent: "alpha".to_owned(),
15175 branch: "magi/x/A".to_owned(),
15176 worktree: state.worktree_root().join("cand-A"),
15177 summary: String::new(),
15178 stat: String::new(),
15179 files: 0,
15180 commits: 0,
15181 empty: false,
15182 failed: None,
15183 verified_noop: None,
15184 duration_ms: 0,
15185 folded: true,
15186 }];
15187 state.released_to = Some("20260901-000000-new1".to_owned());
15188 state.released_branches = vec!["magi/x/A".to_owned()];
15189
15190 fold_run(&mut state, true, &home).await.expect("fold_run");
15191
15192 assert!(
15193 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15194 "the handed-over branch survives a fold"
15195 );
15196 }
15197
15198 #[tokio::test]
15202 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15203 let tmp = tempfile::tempdir().expect("tempdir");
15204 let repo = tmp.path().join("repo");
15205 std::fs::create_dir_all(&repo).unwrap();
15206 init_repo(&repo);
15207 let run = |args: &[&str]| {
15208 let out = std::process::Command::new("git")
15209 .quiet()
15210 .args(args)
15211 .current_dir(&repo)
15212 .output()
15213 .expect("spawn git");
15214 assert!(out.status.success(), "git {args:?}");
15215 };
15216 run(&["branch", "magi/x/A"]);
15217 run(&["checkout", "-q", "-b", "magi/x/B"]);
15218 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15219 run(&["add", "-A"]);
15220 run(&["commit", "-q", "-m", "work"]);
15221 run(&["checkout", "-q", "main"]);
15222
15223 let probe = RunState::new(
15227 repo.clone(),
15228 "main".to_owned(),
15229 "deadbeef".to_owned(),
15230 "task".to_owned(),
15231 Config::default(),
15232 );
15233 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15234 let bare = tmp.path().join("origin.git");
15235 let out = std::process::Command::new("git")
15236 .quiet()
15237 .args(["init", "-q", "--bare"])
15238 .arg(&bare)
15239 .output()
15240 .expect("spawn git");
15241 assert!(out.status.success(), "git init --bare");
15242 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15243 run(&["push", "-q", "origin", "main"]);
15244
15245 let mut state = RunState::new(
15246 repo.clone(),
15247 "main".to_owned(),
15248 "deadbeef".to_owned(),
15249 "task".to_owned(),
15250 Config::default(),
15251 );
15252 state.seeds = vec![refs::Seed {
15253 token: "magi/27b2/A".to_owned(),
15254 kind: refs::SeedKind::Unresolved,
15255 sha: String::new(),
15256 branch: true,
15257 detail: "no branch or commit named magi/27b2/A".to_owned(),
15258 }];
15259
15260 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15261 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15262 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15263 let detail = empty_candidate_detail(&state, "main");
15264 assert!(detail.starts_with("empty candidate"), "{detail}");
15265 assert!(detail.contains("magi/27b2/A"), "{detail}");
15266 }
15267
15268 #[tokio::test]
15277 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15278 let tmp = tempfile::tempdir().expect("tempdir");
15279 let repo = tmp.path().join("repo");
15280 std::fs::create_dir_all(&repo).unwrap();
15281 init_repo(&repo);
15282
15283 let mut config = Config::default();
15284 config.merge.mode = MergeMode::Local;
15285
15286 let mut state = RunState::new(
15287 repo.clone(),
15288 "main".to_owned(),
15289 "deadbeef".to_owned(),
15290 "task".to_owned(),
15291 config,
15292 );
15293 state.candidates = vec![Candidate {
15294 index: 0,
15295 label: 'A',
15296 agent: "alpha".to_owned(),
15297 branch: "does-not-exist".to_owned(),
15298 worktree: repo.clone(),
15299 summary: String::new(),
15300 stat: String::new(),
15301 files: 0,
15302 commits: 0,
15303 empty: false,
15304 failed: None,
15305 verified_noop: None,
15306 duration_ms: 0,
15307 folded: false,
15308 }];
15309 state.tally = Some(Tally {
15310 first_choice: BTreeMap::from([('A', 1)]),
15311 borda: BTreeMap::new(),
15312 winner: 'A',
15313 rankings: 1,
15314 unanimous_initial: true,
15315 deliberated: false,
15316 changed_votes: 0,
15317 unanimous_final: true,
15318 tie_break: None,
15319 judges: 0,
15320 present: 0,
15321 quorum: 0,
15322 met_quorum: true,
15323 uncontested: Some("only candidate A produced a change".to_owned()),
15324 });
15325 state.reviews = vec![ReviewRound {
15326 round: 1,
15327 head: "deadbeef".to_owned(),
15328 verified_head: None,
15329 verified_at: None,
15330 reviews: Vec::new(),
15331 e2e: Vec::new(),
15332 fix: None,
15333 blocking: 0,
15334 answered: 0,
15335 expected: 0,
15336 clean: true,
15337 verify_retried: false,
15338 e2e_deferred: false,
15339 e2e_defer_reason: None,
15340 progressed: false,
15341 vote_split: false,
15342 reconsideration: Vec::new(),
15343 verdict: None,
15344 }];
15345 state.gate = vec![CommandOutcome {
15346 command: "test".to_owned(),
15347 code: Some(0),
15348 output_tail: String::new(),
15349 duration_ms: 0,
15350 resource_blocked: false,
15351 }];
15352 state.gate_ran = true;
15353 state.status = RunStatus::Ready;
15358 state.merge = Some(MergeOutcome {
15359 mode: MergeMode::Local,
15360 ok: false,
15361 detail: "already concluded".to_owned(),
15362 empty: false,
15363 });
15364
15365 let mut runner = Runner {
15366 state,
15367 roles: ResolvedRoles {
15368 implementers: Vec::new(),
15369 judges: Vec::new(),
15370 reviewers: Vec::new(),
15371 fixer: None,
15372 conductor: conductor(),
15373 implementer_roster: Vec::new(),
15374 judge_roster: Vec::new(),
15375 reviewer_roster: Vec::new(),
15376 },
15377 sem: Arc::new(Semaphore::new(1)),
15378 pause: Pause::new(),
15379 interrupt: Pause::new(),
15380 };
15381
15382 runner.merge().await.expect("merge");
15383
15384 assert_eq!(
15385 runner.state.status,
15386 RunStatus::Ready,
15387 "a concluded run's status must not change on reentry"
15388 );
15389 assert_eq!(
15390 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15391 Some("already concluded"),
15392 "merge must not run again once the node already recorded an outcome"
15393 );
15394 }
15395
15396 #[tokio::test]
15405 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15406 let tmp = tempfile::tempdir().expect("tempdir");
15407 let repo = tmp.path().join("repo");
15408 std::fs::create_dir_all(&repo).unwrap();
15409 init_repo(&repo);
15410
15411 let mut config = Config::default();
15412 config.merge.mode = MergeMode::Local;
15413
15414 let mut state = RunState::new(
15415 repo.clone(),
15416 "main".to_owned(),
15417 "deadbeef".to_owned(),
15418 "task".to_owned(),
15419 config,
15420 );
15421 state.candidates = vec![Candidate {
15422 index: 0,
15423 label: 'A',
15424 agent: "alpha".to_owned(),
15425 branch: "does-not-exist".to_owned(),
15426 worktree: repo.clone(),
15427 summary: String::new(),
15428 stat: String::new(),
15429 files: 0,
15430 commits: 0,
15431 empty: false,
15432 failed: None,
15433 verified_noop: None,
15434 duration_ms: 0,
15435 folded: false,
15436 }];
15437 state.tally = Some(Tally {
15438 first_choice: BTreeMap::from([('A', 1)]),
15439 borda: BTreeMap::new(),
15440 winner: 'A',
15441 rankings: 1,
15442 unanimous_initial: true,
15443 deliberated: false,
15444 changed_votes: 0,
15445 unanimous_final: true,
15446 tie_break: None,
15447 judges: 0,
15448 present: 0,
15449 quorum: 0,
15450 met_quorum: true,
15451 uncontested: Some("only candidate A produced a change".to_owned()),
15452 });
15453 state.reviews = vec![ReviewRound {
15454 round: 1,
15455 head: "deadbeef".to_owned(),
15456 verified_head: None,
15457 verified_at: None,
15458 reviews: Vec::new(),
15459 e2e: Vec::new(),
15460 fix: None,
15461 blocking: 0,
15462 answered: 0,
15463 expected: 0,
15464 clean: true,
15465 verify_retried: false,
15466 e2e_deferred: false,
15467 e2e_defer_reason: None,
15468 progressed: false,
15469 vote_split: false,
15470 reconsideration: Vec::new(),
15471 verdict: None,
15472 }];
15473 state.gate = Vec::new();
15475 state.gate_ran = false;
15476 state.status = RunStatus::Gating;
15477
15478 let mut runner = Runner {
15479 state,
15480 roles: ResolvedRoles {
15481 implementers: Vec::new(),
15482 judges: Vec::new(),
15483 reviewers: Vec::new(),
15484 fixer: None,
15485 conductor: conductor(),
15486 implementer_roster: Vec::new(),
15487 judge_roster: Vec::new(),
15488 reviewer_roster: Vec::new(),
15489 },
15490 sem: Arc::new(Semaphore::new(1)),
15491 pause: Pause::new(),
15492 interrupt: Pause::new(),
15493 };
15494
15495 runner.merge().await.expect("merge");
15496
15497 assert!(
15498 runner.state.merge.is_none(),
15499 "an empty gate must never be read as a passing one: {:?}",
15500 runner.state.merge
15501 );
15502 }
15503
15504 #[tokio::test]
15511 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15512 let tmp = tempfile::tempdir().expect("tempdir");
15513 let repo = tmp.path().join("repo");
15514 std::fs::create_dir_all(&repo).unwrap();
15515 init_repo(&repo);
15516
15517 let config = Config::default();
15519
15520 let mut state = RunState::new(
15521 repo.clone(),
15522 "main".to_owned(),
15523 "deadbeef".to_owned(),
15524 "task".to_owned(),
15525 config,
15526 );
15527 state.candidates = vec![Candidate {
15528 index: 0,
15529 label: 'A',
15530 agent: "alpha".to_owned(),
15531 branch: "does-not-exist".to_owned(),
15532 worktree: repo.clone(),
15533 summary: String::new(),
15534 stat: String::new(),
15535 files: 0,
15536 commits: 0,
15537 empty: false,
15538 failed: None,
15539 verified_noop: None,
15540 duration_ms: 0,
15541 folded: false,
15542 }];
15543 state.tally = Some(Tally {
15544 first_choice: BTreeMap::from([('A', 1)]),
15545 borda: BTreeMap::new(),
15546 winner: 'A',
15547 rankings: 1,
15548 unanimous_initial: true,
15549 deliberated: false,
15550 changed_votes: 0,
15551 unanimous_final: true,
15552 tie_break: None,
15553 judges: 0,
15554 present: 0,
15555 quorum: 0,
15556 met_quorum: true,
15557 uncontested: Some("only candidate A produced a change".to_owned()),
15558 });
15559 state.reviews = vec![ReviewRound {
15560 round: 1,
15561 head: "deadbeef".to_owned(),
15562 verified_head: None,
15563 verified_at: None,
15564 reviews: Vec::new(),
15565 e2e: Vec::new(),
15566 fix: None,
15567 blocking: 0,
15568 answered: 0,
15569 expected: 0,
15570 clean: true,
15571 verify_retried: false,
15572 e2e_deferred: false,
15573 e2e_defer_reason: None,
15574 progressed: false,
15575 vote_split: false,
15576 reconsideration: Vec::new(),
15577 verdict: None,
15578 }];
15579
15580 let mut runner = Runner {
15581 state,
15582 roles: ResolvedRoles {
15583 implementers: Vec::new(),
15584 judges: Vec::new(),
15585 reviewers: Vec::new(),
15586 fixer: None,
15587 conductor: conductor(),
15588 implementer_roster: Vec::new(),
15589 judge_roster: Vec::new(),
15590 reviewer_roster: Vec::new(),
15591 },
15592 sem: Arc::new(Semaphore::new(1)),
15593 pause: Pause::new(),
15594 interrupt: Pause::new(),
15595 };
15596
15597 runner.gate().await.expect("gate");
15598 assert!(
15599 runner.state.gate_ran,
15600 "zero configured commands is still a real attempt, not an unrun gate"
15601 );
15602 assert!(runner.state.gate.is_empty());
15603 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15604 assert_ne!(
15605 runner.state.status,
15606 RunStatus::Blocked,
15607 "a gate with nothing to check must not read as failed"
15608 );
15609
15610 runner.merge().await.expect("merge");
15611 assert_eq!(
15612 runner.state.status,
15613 RunStatus::Ready,
15614 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15615 );
15616 }
15617
15618 #[tokio::test]
15629 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15630 crate::run::pin_test_home();
15631 let home = crate::run::home();
15632
15633 let tmp = tempfile::tempdir().expect("tempdir");
15634 let repo = tmp.path().join("repo");
15635 std::fs::create_dir_all(&repo).unwrap();
15636 init_repo(&repo);
15637 let cache_dir = tmp.path().join("target");
15640
15641 let mut config = Config::default();
15642 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15643 config.graph.timeout_verify = Some(2);
15646 config.graph.timeout_resource_wait = 2;
15647
15648 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15649 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15650 .expect("no io error acquiring directly")
15651 {
15652 crate::cache::AcquireOutcome::Acquired(g) => g,
15653 crate::cache::AcquireOutcome::Busy(b) => {
15654 panic!("expected the direct acquire to win the lease first: {b:?}")
15655 }
15656 };
15657
15658 let mut state = RunState::new(
15659 repo.clone(),
15660 "main".to_owned(),
15661 "deadbeef".to_owned(),
15662 "task".to_owned(),
15663 config,
15664 );
15665 state.candidates = vec![Candidate {
15666 index: 0,
15667 label: 'A',
15668 agent: "alpha".to_owned(),
15669 branch: "does-not-exist".to_owned(),
15670 worktree: repo.clone(),
15671 summary: String::new(),
15672 stat: String::new(),
15673 files: 0,
15674 commits: 0,
15675 empty: false,
15676 failed: None,
15677 verified_noop: None,
15678 duration_ms: 0,
15679 folded: false,
15680 }];
15681 state.tally = Some(Tally {
15682 first_choice: BTreeMap::from([('A', 1)]),
15683 borda: BTreeMap::new(),
15684 winner: 'A',
15685 rankings: 1,
15686 unanimous_initial: true,
15687 deliberated: false,
15688 changed_votes: 0,
15689 unanimous_final: true,
15690 tie_break: None,
15691 judges: 0,
15692 present: 0,
15693 quorum: 0,
15694 met_quorum: true,
15695 uncontested: Some("only candidate A produced a change".to_owned()),
15696 });
15697 state.reviews = vec![ReviewRound {
15698 round: 1,
15699 head: "deadbeef".to_owned(),
15700 verified_head: None,
15701 verified_at: None,
15702 reviews: Vec::new(),
15703 e2e: Vec::new(),
15704 fix: None,
15705 blocking: 0,
15706 answered: 0,
15707 expected: 0,
15708 clean: true,
15709 verify_retried: false,
15710 e2e_deferred: false,
15711 e2e_defer_reason: None,
15712 progressed: false,
15713 vote_split: false,
15714 reconsideration: Vec::new(),
15715 verdict: None,
15716 }];
15717
15718 let mut runner = Runner {
15719 state,
15720 roles: ResolvedRoles {
15721 implementers: Vec::new(),
15722 judges: Vec::new(),
15723 reviewers: Vec::new(),
15724 fixer: None,
15725 conductor: conductor(),
15726 implementer_roster: Vec::new(),
15727 judge_roster: Vec::new(),
15728 reviewer_roster: Vec::new(),
15729 },
15730 sem: Arc::new(Semaphore::new(1)),
15731 pause: Pause::new(),
15732 interrupt: Pause::new(),
15733 };
15734
15735 let started = std::time::Instant::now();
15736 runner.gate().await.expect("gate");
15737 assert!(
15738 started.elapsed() < Duration::from_secs(1),
15739 "a gate with nothing to run must never wait on a lease it never needed"
15740 );
15741 assert!(
15742 runner.state.gate_ran,
15743 "zero commands is still a real, immediate attempt"
15744 );
15745 assert!(runner.state.gate.is_empty());
15746 assert_ne!(
15747 runner.state.status,
15748 RunStatus::Blocked,
15749 "must not read as resource-blocked on a lease it never asked for"
15750 );
15751 }
15752
15753 #[cfg(unix)]
15754 #[tokio::test]
15755 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15756 crate::run::pin_test_home();
15757 let tmp = tempfile::tempdir().unwrap();
15758 let mut config = Config::default();
15759 config.disk.min_free_bytes = 0;
15760 let mut state = RunState::new(
15761 tmp.path().to_owned(),
15762 "main".into(),
15763 "base".into(),
15764 "verification evidence".into(),
15765 config,
15766 );
15767 let (outcomes, pids) = run_commands(
15768 &mut state,
15769 "gate",
15770 "gate",
15771 1,
15772 &["sh".into(), "-c".into()],
15773 &[
15774 "printf before-timeout; exec sleep 30".into(),
15775 "kill -TERM $$".into(),
15776 "exit 7".into(),
15777 "printf success".into(),
15778 ],
15779 tmp.path(),
15780 Duration::from_secs(1),
15781 )
15782 .await;
15783 wait_for_timed_out_children_to_die(&pids).await;
15784 assert_eq!(
15785 pids.len(),
15786 1,
15787 "unleased timeout retains its death-check PID"
15788 );
15789 assert_eq!(outcomes.len(), 4);
15790 assert_eq!(outcomes[0].code, None);
15791 assert!(outcomes[0].output_tail.contains("before-timeout"));
15792 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15793 assert_eq!(outcomes[1].code, None);
15794 assert!(outcomes[1].output_tail.contains("command terminated:"));
15795 assert!(!outcomes[1].output_tail.contains("timed out"));
15796 assert_eq!(outcomes[2].code, Some(7));
15797 assert!(!outcomes[2].output_tail.contains("timed out"));
15798 assert_eq!(outcomes[3].code, Some(0));
15799 assert_eq!(outcomes[3].output_tail, "success");
15800 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15801 }
15802
15803 #[tokio::test]
15813 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15814 crate::run::pin_test_home();
15815
15816 let tmp = tempfile::tempdir().expect("tempdir");
15817 let repo = tmp.path().join("repo");
15818 std::fs::create_dir_all(&repo).unwrap();
15819 init_repo(&repo);
15820
15821 let mut config = Config::default();
15822 config.verify.gate = vec![
15823 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15824 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15825 .to_owned(),
15826 ];
15827
15828 let mut state = RunState::new(
15829 repo.clone(),
15830 "main".to_owned(),
15831 "deadbeef".to_owned(),
15832 "task".to_owned(),
15833 config,
15834 );
15835 let run_id = state.id.clone();
15836 state.candidates = vec![Candidate {
15837 index: 0,
15838 label: 'A',
15839 agent: "alpha".to_owned(),
15840 branch: "does-not-exist".to_owned(),
15841 worktree: repo.clone(),
15842 summary: String::new(),
15843 stat: String::new(),
15844 files: 0,
15845 commits: 0,
15846 empty: false,
15847 failed: None,
15848 verified_noop: None,
15849 duration_ms: 0,
15850 folded: false,
15851 }];
15852 state.tally = Some(Tally {
15853 first_choice: BTreeMap::from([('A', 1)]),
15854 borda: BTreeMap::new(),
15855 winner: 'A',
15856 rankings: 1,
15857 unanimous_initial: true,
15858 deliberated: false,
15859 changed_votes: 0,
15860 unanimous_final: true,
15861 tie_break: None,
15862 judges: 0,
15863 present: 0,
15864 quorum: 0,
15865 met_quorum: true,
15866 uncontested: Some("only candidate A produced a change".to_owned()),
15867 });
15868 state.reviews = vec![ReviewRound {
15869 round: 1,
15870 head: "deadbeef".to_owned(),
15871 verified_head: None,
15872 verified_at: None,
15873 reviews: Vec::new(),
15874 e2e: Vec::new(),
15875 fix: None,
15876 blocking: 0,
15877 answered: 0,
15878 expected: 0,
15879 clean: true,
15880 verify_retried: false,
15881 e2e_deferred: false,
15882 e2e_defer_reason: None,
15883 progressed: false,
15884 vote_split: false,
15885 reconsideration: Vec::new(),
15886 verdict: None,
15887 }];
15888
15889 let mut runner = Runner {
15890 state,
15891 roles: ResolvedRoles {
15892 implementers: Vec::new(),
15893 judges: Vec::new(),
15894 reviewers: Vec::new(),
15895 fixer: None,
15896 conductor: conductor(),
15897 implementer_roster: Vec::new(),
15898 judge_roster: Vec::new(),
15899 reviewer_roster: Vec::new(),
15900 },
15901 sem: Arc::new(Semaphore::new(1)),
15902 pause: Pause::new(),
15903 interrupt: Pause::new(),
15904 };
15905
15906 let started_marker = repo.join("started.marker");
15907 let release_marker = repo.join("release.marker");
15908 let poller = tokio::spawn(async move {
15909 for _ in 0..100 {
15914 if started_marker.exists()
15915 && let Ok(s) = crate::run::RunState::load(&run_id)
15916 && let Some(a) = s.active.get("gate")
15917 {
15918 std::fs::write(&release_marker, b"go").expect("release marker");
15919 return Some(a.clone());
15920 }
15921 tokio::time::sleep(Duration::from_millis(50)).await;
15922 }
15923 None
15924 });
15925
15926 runner.gate().await.expect("gate");
15927 let captured = poller.await.expect("poller task");
15928 let captured = captured.expect(
15929 "the poller never saw a `gate` task entry in run.json while the command was \
15930 still blocked on its own release marker",
15931 );
15932
15933 assert_eq!(captured.task.as_deref(), Some("gate"));
15934 assert_eq!(captured.node, "gate");
15935 assert_eq!(captured.index, Some(1));
15936 assert_eq!(captured.total, Some(1));
15937 assert!(
15938 captured
15939 .command
15940 .as_deref()
15941 .is_some_and(|c| c.contains("started.marker")),
15942 "{captured:?}"
15943 );
15944
15945 assert!(
15946 runner.state.active.is_empty(),
15947 "the entry must be cleared once the command actually finished: {:?}",
15948 runner.state.active
15949 );
15950 assert!(runner.state.gate_ran);
15951 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15952 }
15953
15954 #[tokio::test]
15957 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15958 use crate::verdict::{Finding, ReviewVote, Severity};
15959 crate::run::pin_test_home();
15960 let tmp = tempfile::tempdir().expect("tempdir");
15961 let repo = tmp.path().join("repo");
15962 std::fs::create_dir_all(&repo).unwrap();
15963 init_repo(&repo);
15964
15965 for (severity, vote, expect) in [
15966 (Severity::Major, ReviewVote::Reject, true),
15967 (Severity::Minor, ReviewVote::Reject, false),
15968 (Severity::Major, ReviewVote::Approve, false),
15969 ] {
15970 let mut round = review_round(false, 1, 1, 1, false, true);
15971 round.reviews = vec![ReviewRecord {
15972 reviewer: 1,
15973 agent: "alpha".to_owned(),
15974 summary: String::new(),
15975 findings: vec![Finding {
15976 id: "R1-1-1".to_owned(),
15977 severity,
15978 file: None,
15979 line: None,
15980 title: "t".to_owned(),
15981 detail: String::new(),
15982 }],
15983 vote: Some(vote),
15984 failed: None,
15985 duration_ms: 0,
15986 attempts: 0,
15987 }];
15988 let mut state = RunState::new(
15989 repo.clone(),
15990 "main".to_owned(),
15991 "deadbeef".to_owned(),
15992 "task".to_owned(),
15993 Config::default(),
15994 );
15995 state.reviews = vec![round];
15996 let mut runner = Runner {
15997 state,
15998 roles: ResolvedRoles {
15999 implementers: Vec::new(),
16000 judges: Vec::new(),
16001 reviewers: Vec::new(),
16002 fixer: None,
16003 conductor: conductor(),
16004 implementer_roster: Vec::new(),
16005 judge_roster: Vec::new(),
16006 reviewer_roster: Vec::new(),
16007 },
16008 sem: Arc::new(Semaphore::new(1)),
16009 pause: Pause::new(),
16010 interrupt: Pause::new(),
16011 };
16012 let shell = runner.state.config.shell();
16013 runner
16014 .stop_reviewing("round budget spent", &shell, &repo)
16015 .await
16016 .expect("stop_reviewing");
16017 assert_eq!(runner.state.status, RunStatus::Gating);
16018 assert_eq!(
16019 runner.state.contested_handoff.is_some(),
16020 expect,
16021 "{severity:?} + {vote:?}"
16022 );
16023 }
16024 }
16025
16026 #[tokio::test]
16039 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16040 crate::run::pin_test_home();
16041 let home = crate::run::home();
16042
16043 let tmp = tempfile::tempdir().expect("tempdir");
16044 let repo = tmp.path().join("repo");
16045 std::fs::create_dir_all(&repo).unwrap();
16046 init_repo(&repo);
16047 let head = crate::git::rev_parse(&repo, "HEAD")
16048 .await
16049 .expect("rev-parse");
16050 let cache_dir = tmp.path().join("target");
16053
16054 let mut config = Config::default();
16055 config.verify.e2e = vec![format!(
16056 "CARGO_TARGET_DIR='{}' test -f README.md",
16057 cache_dir.display()
16058 )];
16059 config.graph.review_rounds = 1;
16060 config.graph.timeout_verify = Some(2);
16063 config.graph.timeout_resource_wait = 2;
16064
16065 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16066 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16067 .expect("no io error acquiring directly")
16068 {
16069 crate::cache::AcquireOutcome::Acquired(g) => g,
16070 crate::cache::AcquireOutcome::Busy(b) => {
16071 panic!("expected the direct acquire to win the lease first: {b:?}")
16072 }
16073 };
16074
16075 let mut state = RunState::new(
16076 repo.clone(),
16077 "main".to_owned(),
16078 head.clone(),
16079 "task".to_owned(),
16080 config,
16081 );
16082 state.candidates = vec![Candidate {
16083 index: 0,
16084 label: 'A',
16085 agent: "alpha".to_owned(),
16086 branch: "does-not-exist".to_owned(),
16087 worktree: repo.clone(),
16088 summary: String::new(),
16089 stat: String::new(),
16090 files: 0,
16091 commits: 0,
16092 empty: false,
16093 failed: None,
16094 verified_noop: None,
16095 duration_ms: 0,
16096 folded: false,
16097 }];
16098 state.tally = Some(Tally {
16099 first_choice: BTreeMap::from([('A', 1)]),
16100 borda: BTreeMap::new(),
16101 winner: 'A',
16102 rankings: 1,
16103 unanimous_initial: true,
16104 deliberated: false,
16105 changed_votes: 0,
16106 unanimous_final: true,
16107 tie_break: None,
16108 judges: 0,
16109 present: 0,
16110 quorum: 0,
16111 met_quorum: true,
16112 uncontested: Some("only candidate A produced a change".to_owned()),
16113 });
16114 state.reviews = vec![ReviewRound {
16118 round: 1,
16119 head: head.clone(),
16120 verified_head: None,
16121 verified_at: None,
16122 reviews: Vec::new(),
16123 e2e: Vec::new(),
16124 fix: None,
16125 blocking: 1,
16126 answered: 1,
16127 expected: 1,
16128 clean: false,
16129 verify_retried: false,
16130 e2e_deferred: true,
16131 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16132 progressed: false,
16133 vote_split: false,
16134 reconsideration: Vec::new(),
16135 verdict: None,
16136 }];
16137
16138 let mut runner = Runner {
16139 state,
16140 roles: ResolvedRoles {
16141 implementers: Vec::new(),
16142 judges: Vec::new(),
16143 reviewers: Vec::new(),
16144 fixer: None,
16145 conductor: conductor(),
16146 implementer_roster: Vec::new(),
16147 judge_roster: Vec::new(),
16148 reviewer_roster: Vec::new(),
16149 },
16150 sem: Arc::new(Semaphore::new(1)),
16151 pause: Pause::new(),
16152 interrupt: Pause::new(),
16153 };
16154
16155 let shell = runner.state.config.shell();
16156 runner
16157 .stop_reviewing("round budget spent", &shell, &repo)
16158 .await
16159 .expect("stop_reviewing");
16160
16161 let last = runner.state.reviews.last().expect("round record");
16162 assert_eq!(
16163 last.e2e_status(),
16164 E2eStatus::ResourceBlocked,
16165 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16166 failed: {last:?}"
16167 );
16168 assert_eq!(
16169 last.verified_head.as_deref(),
16170 Some(head.as_str()),
16171 "which commit this attempt targeted is known even though nothing finished checking \
16172 it"
16173 );
16174 let first_attempt_at = last
16175 .verified_at
16176 .expect("when this attempt ran is known too");
16177 assert_ne!(
16178 runner.state.status,
16179 RunStatus::Blocked,
16180 "contention is evidence about the machine, not the patch — it must not settle the \
16181 run as blocked: {:?}",
16182 runner.state.status
16183 );
16184 assert!(
16185 !runner
16186 .state
16187 .events
16188 .iter()
16189 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16190 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16191 runner.state.events
16192 );
16193
16194 runner
16199 .stop_reviewing("round budget spent", &shell, &repo)
16200 .await
16201 .expect("stop_reviewing retry");
16202 assert_eq!(
16203 runner.state.reviews.len(),
16204 1,
16205 "no new round was started: {:?}",
16206 runner.state.reviews
16207 );
16208 let last = runner.state.reviews.last().expect("round record");
16209 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16210 assert!(
16211 last.verified_at.expect("still known") > first_attempt_at,
16212 "a second reentry must be a fresh attempt, not a stale copy of the first"
16213 );
16214 assert_ne!(runner.state.status, RunStatus::Blocked);
16215
16216 held.release();
16217 }
16218
16219 #[tokio::test]
16231 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16232 crate::run::pin_test_home();
16233 let home = crate::run::home();
16234
16235 let tmp = tempfile::tempdir().expect("tempdir");
16236 let repo = tmp.path().join("repo");
16237 std::fs::create_dir_all(&repo).unwrap();
16238 init_repo(&repo);
16239 let head = crate::git::rev_parse(&repo, "HEAD")
16240 .await
16241 .expect("rev-parse");
16242 let cache_dir = tmp.path().join("target");
16243
16244 let mut config = Config::default();
16245 config.verify.e2e = vec![format!(
16246 "CARGO_TARGET_DIR='{}' test -f README.md",
16247 cache_dir.display()
16248 )];
16249 config.graph.review_rounds = 1;
16250 config.graph.timeout_verify = Some(2);
16251 config.graph.timeout_resource_wait = 2;
16252
16253 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16254 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16255 .expect("no io error acquiring directly")
16256 {
16257 crate::cache::AcquireOutcome::Acquired(g) => g,
16258 crate::cache::AcquireOutcome::Busy(b) => {
16259 panic!("expected the direct acquire to win the lease first: {b:?}")
16260 }
16261 };
16262
16263 let mut state = RunState::new(
16264 repo.clone(),
16265 "main".to_owned(),
16266 head.clone(),
16267 "task".to_owned(),
16268 config,
16269 );
16270 state.candidates = vec![Candidate {
16271 index: 0,
16272 label: 'A',
16273 agent: "alpha".to_owned(),
16274 branch: "does-not-exist".to_owned(),
16275 worktree: repo.clone(),
16276 summary: String::new(),
16277 stat: String::new(),
16278 files: 0,
16279 commits: 0,
16280 empty: false,
16281 failed: None,
16282 verified_noop: None,
16283 duration_ms: 0,
16284 folded: false,
16285 }];
16286 state.tally = Some(Tally {
16287 first_choice: BTreeMap::from([('A', 1)]),
16288 borda: BTreeMap::new(),
16289 winner: 'A',
16290 rankings: 1,
16291 unanimous_initial: true,
16292 deliberated: false,
16293 changed_votes: 0,
16294 unanimous_final: true,
16295 tie_break: None,
16296 judges: 0,
16297 present: 0,
16298 quorum: 0,
16299 met_quorum: true,
16300 uncontested: Some("only candidate A produced a change".to_owned()),
16301 });
16302 state.reviews = vec![ReviewRound {
16306 round: 1,
16307 head: head.clone(),
16308 verified_head: Some(head.clone()),
16309 verified_at: Some(jiff::Timestamp::now()),
16310 reviews: Vec::new(),
16311 e2e: vec![CommandOutcome {
16312 command: format!(
16313 "CARGO_TARGET_DIR='{}' test -f README.md",
16314 cache_dir.display()
16315 ),
16316 code: None,
16317 output_tail: "waiting for the shared build cache".to_owned(),
16318 duration_ms: 0,
16319 resource_blocked: true,
16320 }],
16321 fix: None,
16322 blocking: 1,
16323 answered: 1,
16324 expected: 1,
16325 clean: false,
16326 verify_retried: false,
16327 e2e_deferred: false,
16328 e2e_defer_reason: None,
16329 progressed: false,
16330 vote_split: false,
16331 reconsideration: Vec::new(),
16332 verdict: None,
16333 }];
16334
16335 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16336 let mut runner = Runner {
16337 state,
16338 roles: ResolvedRoles {
16339 implementers: Vec::new(),
16340 judges: Vec::new(),
16341 reviewers: Vec::new(),
16342 fixer: None,
16343 conductor: conductor(),
16344 implementer_roster: Vec::new(),
16345 judge_roster: Vec::new(),
16346 reviewer_roster: Vec::new(),
16347 },
16348 sem: Arc::new(Semaphore::new(1)),
16349 pause: Pause::new(),
16350 interrupt: Pause::new(),
16351 };
16352
16353 runner.review_loop().await.expect("review_loop");
16358
16359 assert_eq!(
16360 runner.state.reviews.len(),
16361 1,
16362 "no new round was started on top of the unresolved one: {:?}",
16363 runner.state.reviews
16364 );
16365 let last = &runner.state.reviews[0];
16366 assert_eq!(
16367 last.e2e_status(),
16368 E2eStatus::ResourceBlocked,
16369 "still contended: {last:?}"
16370 );
16371 assert!(
16372 last.verified_at.expect("still known") > first_attempt_at,
16373 "review_loop must have actually retried the check, not left it exactly as found"
16374 );
16375 assert_ne!(
16376 runner.state.status,
16377 RunStatus::Blocked,
16378 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16379 runner.state.status
16380 );
16381
16382 held.release();
16383 }
16384
16385 #[tokio::test]
16386 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16387 crate::run::pin_test_home();
16388 let tmp = tempfile::tempdir().expect("tempdir");
16389 let repo = tmp.path().join("repo");
16390 std::fs::create_dir_all(&repo).unwrap();
16391 init_repo(&repo);
16392
16393 let mut config = Config::default();
16394 config.merge.mode = MergeMode::Pr;
16395 config.graph.land = true;
16396 config.graph.land_approval = false;
16397
16398 let mut state = RunState::new(
16399 repo.clone(),
16400 "main".to_owned(),
16401 "deadbeef".to_owned(),
16402 "task".to_owned(),
16403 config,
16404 );
16405 state.candidates = vec![Candidate {
16406 index: 0,
16407 label: 'A',
16408 agent: "alpha".to_owned(),
16409 branch: "does-not-exist".to_owned(),
16410 worktree: repo.clone(),
16411 summary: String::new(),
16412 stat: String::new(),
16413 files: 0,
16414 commits: 0,
16415 empty: false,
16416 failed: None,
16417 verified_noop: None,
16418 duration_ms: 0,
16419 folded: false,
16420 }];
16421 state.tally = Some(Tally {
16422 first_choice: BTreeMap::from([('A', 1)]),
16423 borda: BTreeMap::new(),
16424 winner: 'A',
16425 rankings: 1,
16426 unanimous_initial: true,
16427 deliberated: false,
16428 changed_votes: 0,
16429 unanimous_final: true,
16430 tie_break: None,
16431 judges: 0,
16432 present: 0,
16433 quorum: 0,
16434 met_quorum: true,
16435 uncontested: Some("only candidate A produced a change".to_owned()),
16436 });
16437 state.reviews = vec![ReviewRound {
16438 round: 1,
16439 head: "deadbeef".to_owned(),
16440 verified_head: None,
16441 verified_at: None,
16442 reviews: Vec::new(),
16443 e2e: Vec::new(),
16444 fix: None,
16445 blocking: 0,
16446 answered: 0,
16447 expected: 0,
16448 clean: true,
16449 verify_retried: false,
16450 e2e_deferred: false,
16451 e2e_defer_reason: None,
16452 progressed: false,
16453 vote_split: false,
16454 reconsideration: Vec::new(),
16455 verdict: None,
16456 }];
16457 state.gate = vec![CommandOutcome {
16458 command: "test".to_owned(),
16459 code: Some(0),
16460 output_tail: String::new(),
16461 duration_ms: 0,
16462 resource_blocked: false,
16463 }];
16464 state.gate_ran = true;
16465 state.status = RunStatus::Landing;
16469 state.merge = Some(MergeOutcome {
16470 mode: MergeMode::Pr,
16471 ok: true,
16472 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16473 empty: false,
16474 });
16475
16476 ask_test_home();
16480 state.save().expect("save landing run");
16483 let store = ask::Questions::open();
16484 let q = ask_open_question(&store, &state.id);
16485
16486 let mut runner = Runner {
16487 state,
16488 roles: ResolvedRoles {
16489 implementers: Vec::new(),
16490 judges: Vec::new(),
16491 reviewers: Vec::new(),
16492 fixer: None,
16493 conductor: conductor(),
16494 implementer_roster: Vec::new(),
16495 judge_roster: Vec::new(),
16496 reviewer_roster: Vec::new(),
16497 },
16498 sem: Arc::new(Semaphore::new(1)),
16499 pause: Pause::new(),
16500 interrupt: Pause::new(),
16501 };
16502
16503 runner.execute().await.expect("execute");
16508
16509 assert_eq!(
16510 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16511 Some("https://example.invalid/x/y/pull/1"),
16512 "reentry must not push again or open a second pull request over the \
16513 one `land` is already watching"
16514 );
16515 assert_ne!(
16516 runner.state.status,
16517 RunStatus::Landing,
16518 "land could not actually reach the fake pull request, so it must \
16519 have given up rather than left the run silently parked forever"
16520 );
16521 assert_eq!(runner.state.status, RunStatus::Blocked);
16525 assert!(
16526 store.get(&q.id).unwrap().status.open(),
16527 "Blocked is still alive; settle_questions must have been a no-op here"
16528 );
16529 }
16530
16531 fn state_with_round(round: ReviewRound) -> RunState {
16532 let mut s = RunState::new(
16533 PathBuf::from("/repo"),
16534 "main".to_owned(),
16535 "abc1234".to_owned(),
16536 "add retries".to_owned(),
16537 Config::default(),
16538 );
16539 s.reviews = vec![round];
16540 s
16541 }
16542
16543 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16544 crate::verdict::Finding {
16545 id: id.to_owned(),
16546 severity,
16547 file: None,
16548 line: None,
16549 title: title.to_owned(),
16550 detail: String::new(),
16551 }
16552 }
16553
16554 #[test]
16555 fn pr_body_names_open_findings_and_declined_ones() {
16556 let round = ReviewRound {
16557 round: 2,
16558 head: "deadbee".to_owned(),
16559 verified_head: None,
16560 verified_at: None,
16561 reviews: vec![ReviewRecord {
16562 attempts: 0,
16563 reviewer: 1,
16564 agent: "alpha".to_owned(),
16565 summary: String::new(),
16566 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16567 vote: None,
16568 failed: None,
16569 duration_ms: 0,
16570 }],
16571 e2e: vec![CommandOutcome {
16572 command: "cargo test".to_owned(),
16573 code: Some(0),
16574 output_tail: String::new(),
16575 duration_ms: 0,
16576 resource_blocked: false,
16577 }],
16578 verify_retried: false,
16579 e2e_deferred: false,
16580 e2e_defer_reason: None,
16581 fix: Some(FixRecord {
16582 agent: "alpha".to_owned(),
16583 addressed: Vec::new(),
16584 rejected: vec![crate::verdict::Rejection {
16585 id: "R1-1-1".to_owned(),
16586 why: "not reachable from any caller".to_owned(),
16587 }],
16588 notes: String::new(),
16589 committed: true,
16590 failed: None,
16591 duration_ms: 0,
16592 continuation: None,
16593 }),
16594 blocking: 0,
16595 answered: 1,
16596 expected: 1,
16597 clean: false,
16598 progressed: true,
16599 vote_split: false,
16600 reconsideration: Vec::new(),
16601 verdict: None,
16602 };
16603 let state = state_with_round(round);
16604 let body = pr_message(&state, 'A').body;
16605
16606 assert!(body.contains("add retries"), "the task must still be there");
16607 assert!(body.contains("R2-1-1"), "{body}");
16608 assert!(!body.contains("unused import"), "{body}");
16609 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16610 assert!(
16611 !body.contains("not reachable from any caller"),
16612 "the reason it was declined: {body}"
16613 );
16614 }
16615
16616 #[test]
16617 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16618 let round = ReviewRound {
16619 round: 1,
16620 head: "deadbee".to_owned(),
16621 verified_head: None,
16622 verified_at: None,
16623 reviews: vec![ReviewRecord {
16624 attempts: 0,
16625 reviewer: 1,
16626 agent: "alpha".to_owned(),
16627 summary: String::new(),
16628 findings: Vec::new(),
16629 vote: None,
16630 failed: None,
16631 duration_ms: 0,
16632 }],
16633 e2e: Vec::new(),
16634 verify_retried: false,
16635 e2e_deferred: false,
16636 e2e_defer_reason: None,
16637 fix: None,
16638 blocking: 0,
16639 answered: 1,
16640 expected: 1,
16641 clean: true,
16642 progressed: false,
16643 vote_split: false,
16644 reconsideration: Vec::new(),
16645 verdict: None,
16646 };
16647 let state = state_with_round(round);
16648 let body = pr_message(&state, 'A').body;
16649 assert!(!body.contains("Open review findings"), "{body}");
16650 assert!(!body.contains("Declined"), "{body}");
16651 }
16652
16653 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16654 let mut state = RunState::new(
16655 PathBuf::from("/repo"),
16656 "main".to_owned(),
16657 "abc1234".to_owned(),
16658 instruction.to_owned(),
16659 Config::default(),
16660 );
16661 state.candidates.push(Candidate {
16662 index: 0,
16663 label: 'A',
16664 agent: "alpha".to_owned(),
16665 branch: "magi/x/A".to_owned(),
16666 worktree: PathBuf::from("/wt"),
16667 summary: summary.to_owned(),
16668 stat: String::new(),
16669 files: 1,
16670 commits: 1,
16671 empty: false,
16672 failed: None,
16673 verified_noop: None,
16674 folded: false,
16675 duration_ms: 0,
16676 });
16677 state
16678 }
16679
16680 fn review_state(subjects: &[&str]) -> RunState {
16681 let mut state = state_with_summary(
16682 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16683 "",
16684 );
16685 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16686 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16687 state
16688 }
16689
16690 #[tokio::test]
16693 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16694 ask_test_home();
16695 let tmp = tempfile::tempdir().unwrap();
16696 let repo = tmp.path().join("repo");
16697 std::fs::create_dir_all(&repo).unwrap();
16698 init_repo(&repo);
16699 let origin = tmp.path().join("origin.git");
16700 let g = |dir: &Path, args: &[&str]| {
16701 let out = std::process::Command::new("git")
16702 .args(args)
16703 .current_dir(dir)
16704 .quiet()
16705 .output()
16706 .expect("spawn git");
16707 assert!(
16708 out.status.success(),
16709 "git {args:?}: {}",
16710 String::from_utf8_lossy(&out.stderr)
16711 );
16712 };
16713 g(
16714 tmp.path(),
16715 &[
16716 "clone",
16717 "--bare",
16718 "-q",
16719 repo.to_str().unwrap(),
16720 origin.to_str().unwrap(),
16721 ],
16722 );
16723 g(
16724 &repo,
16725 &["remote", "add", "origin", origin.to_str().unwrap()],
16726 );
16727 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16728
16729 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16731 g(&repo, &["add", "-A"]);
16732 g(
16733 &repo,
16734 &["commit", "-q", "-m", "chore(deps): update a crate"],
16735 );
16736 g(&repo, &["push", "-q", "origin", "main"]);
16737 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16738 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16739 g(&repo, &["add", "-A"]);
16740 g(
16741 &repo,
16742 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16743 );
16744 g(&repo, &["checkout", "-q", "main"]);
16745
16746 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16747 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16748 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16750 assert_eq!(fallback, c1);
16751
16752 let mut state = review_state(&[
16753 "chore(deps): update a crate",
16754 "fix(daemon): apply a chosen action",
16755 ]);
16756 state.repo = repo.clone();
16757 state.base_branch = "main".to_owned();
16758 state.base_commit = c1;
16759 refresh_reviewed_commits(&mut state, "feat/own").await;
16760 assert_eq!(
16761 state.reviewed_commits,
16762 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16763 );
16764 assert_eq!(
16765 review_title(&state).as_deref(),
16766 Some("fix(daemon): apply a chosen action")
16767 );
16768 assert_eq!(
16769 leaked_subjects(&state, "feat/own").await,
16770 Some(vec!["chore(deps): update a crate".to_owned()])
16771 );
16772 state.config.merge.remote = "nowhere".to_owned();
16775 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16776 }
16777
16778 struct SummaryFixture {
16782 _tmp: tempfile::TempDir,
16783 repo: PathBuf,
16784 reply: PathBuf,
16785 runner: Runner,
16786 }
16787
16788 fn summary_fixture() -> SummaryFixture {
16789 ask_test_home();
16790 let tmp = tempfile::tempdir().unwrap();
16791 let repo = tmp.path().join("repo");
16792 std::fs::create_dir_all(&repo).unwrap();
16793 init_repo(&repo);
16794 let origin = tmp.path().join("origin.git");
16795 let g = |args: &[&str]| {
16796 let out = std::process::Command::new("git")
16797 .args(args)
16798 .current_dir(&repo)
16799 .quiet()
16800 .output()
16801 .expect("spawn git");
16802 assert!(
16803 out.status.success(),
16804 "git {args:?}: {}",
16805 String::from_utf8_lossy(&out.stderr)
16806 );
16807 };
16808 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16809 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16810 g(&["push", "-q", "origin", "main"]);
16811 g(&["checkout", "-q", "-b", "feat/own"]);
16812 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16813 g(&["add", "-A"]);
16814 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16815 let reply = tmp.path().join("reply.txt");
16816 std::fs::write(
16817 &reply,
16818 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16819 )
16820 .unwrap();
16821 let mut runner = runner_at(RunStatus::Gating);
16822 let mut state = review_state(&["feat(own): add the own file"]);
16823 state.repo = repo.clone();
16824 state.base_branch = "main".to_owned();
16825 state.candidates[0].branch = "feat/own".to_owned();
16826 state.candidates[0].worktree = repo.clone();
16827 let mut author = spec("alpha");
16828 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16829 author
16830 .env
16831 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16832 state.config.agents = vec![author];
16833 runner.state = state;
16834 SummaryFixture {
16835 _tmp: tmp,
16836 repo,
16837 reply,
16838 runner,
16839 }
16840 }
16841
16842 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16843 let winner = f.runner.state.candidates[0].clone();
16844 f.runner
16845 .guarded_pr_message(&winner, None, true)
16846 .await
16847 .unwrap()
16848 .expect("an English description never parks")
16849 }
16850
16851 fn clean_round_with_minor_findings() -> ReviewRound {
16852 let mut round = passing_round(2, None);
16853 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16854 round.reviews = vec![ReviewRecord {
16855 attempts: 0,
16856 reviewer: 1,
16857 agent: "alpha".into(),
16858 summary: "内部レビュー".into(),
16859 findings: vec![
16860 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16861 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16862 ],
16863 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16864 failed: None,
16865 duration_ms: 0,
16866 }];
16867 round
16868 }
16869
16870 #[tokio::test]
16871 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16872 for review_only in [false, true] {
16873 let mut f = summary_fixture();
16874 if !review_only {
16875 f.runner.state.candidates[0].agent = "alpha".into();
16876 }
16877 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16878 assert!(f.runner.state.reviews[0].clean);
16879 assert!(f.runner.state.reviews[0].fix.is_none());
16880 assert!(f.runner.state.open_findings().is_empty());
16881 assert_eq!(
16882 crate::pr_outcome::required_finding_ids(&f.runner.state),
16883 ["R2-1-1", "R2-1-2"]
16884 );
16885 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16886 let evidence = crate::pr_outcome::summary_evidence(
16887 &f.runner.state,
16888 crate::pr_outcome::FinalHead::At(&head),
16889 );
16890 assert!(
16891 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16892 );
16893 let omitted = posted(&mut f).await;
16895 assert!(omitted.body.contains("No description"));
16896 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16897 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16898 assert!(f.runner.state.change_summary.is_none());
16899 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16900 let described = posted(&mut f).await;
16901 assert!(described.body.contains("improve documentation for callers"));
16902 assert!(described.body.contains("remaining edge case"));
16903 assert!(described.body.contains("## Open review findings"));
16904 assert!(!described.body.contains("未修正"));
16905 }
16906 }
16907
16908 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16909 let mut f = summary_fixture();
16910 if !review_only {
16911 f.runner.state.candidates[0].agent = "alpha".into();
16912 }
16913 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16914 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16915 fixed
16916 .fix
16917 .as_mut()
16918 .unwrap()
16919 .rejected
16920 .push(crate::verdict::Rejection {
16921 id: "R1-1-1".into(),
16922 why: "呼び出されないコードのため辞退".into(),
16923 });
16924 let mut clean = clean_round_with_minor_findings();
16925 clean.verified_head = Some(head.clone());
16926 f.runner.state.reviews = vec![fixed, clean];
16927 ok_gate(&mut f.runner.state, Some(&head));
16928 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16929 f.runner.state.config.agents.push(scripted("beta", reply));
16930 f.runner.state.config.roles.language_judge =
16931 Some(crate::config::AgentChoice::One("beta".into()));
16932 f
16933 }
16934
16935 #[tokio::test]
16936 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16937 for review_only in [false, true] {
16938 let mut f = rewrite_fixture_with_findings(review_only,
16939 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16940 let m = posted(&mut f).await;
16941 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16942 let expected =
16943 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16944 assert!(m.body.contains(&expected));
16945 assert!(m.body.contains("## Review rounds"));
16946 assert!(
16947 m.body
16948 .contains("Gate (`verify.gate`): passed on the final commit")
16949 );
16950 assert!(
16951 m.body
16952 .contains("Tests (`verify.e2e`): passed on the final commit")
16953 );
16954 let c = f.runner.state.change_summary.as_ref().unwrap();
16955 assert!(
16956 m.body
16957 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16958 );
16959 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16960 assert!(crate::pr_outcome::covers_required_findings(
16961 &f.runner.state,
16962 &m.body
16963 ));
16964 assert!(m.body.contains("documentation remains incomplete"));
16965 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16966 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16967 assert!(f.runner.state.github_text.is_none());
16968 }
16969 }
16970
16971 #[tokio::test]
16972 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16973 for review_only in [false, true] {
16974 for reply in [
16975 "TITLE: fix: public description\nA short English description.",
16976 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16977 ] {
16978 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16979 let winner = f.runner.state.candidates[0].clone();
16980 let posted = f
16981 .runner
16982 .guarded_pr_message(&winner, None, true)
16983 .await
16984 .unwrap();
16985 assert!(posted.is_none());
16986 assert!(f.runner.state.parked);
16987 let withheld = f.runner.state.github_text.as_ref().unwrap();
16988 assert!(
16989 withheld.rewrite.is_none(),
16990 "an incomplete rewrite must not be selectable"
16991 );
16992 let q = ask::Questions::open()
16993 .get(withheld.question.as_ref().unwrap())
16994 .unwrap();
16995 assert!(
16996 !q.choices
16997 .contains(&crate::github_text::USE_REWRITE.to_owned())
16998 );
16999 assert!(
17000 q.detail.contains("R1-1-1")
17001 && q.detail.contains("R2-1-1")
17002 && q.detail.contains("R2-1-2")
17003 );
17004 }
17005 }
17006 }
17007
17008 #[tokio::test]
17009 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17010 for review_only in [false, true] {
17011 let mut f = summary_fixture();
17012 if !review_only {
17013 f.runner.state.candidates[0].agent = "alpha".into();
17014 }
17015 f.runner.state.instruction = "内部タスクの説明".into();
17016 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17017 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17018 f.runner.state.reviews = vec![
17019 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17020 round_with_notes(2, Some("二回目の内部修正報告です。")),
17021 passing_round(3, Some(&head)),
17022 ];
17023 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17024 ok_gate(&mut f.runner.state, Some(&head));
17025 let captured = f.repo.join("public-prompt.txt");
17026 let author = &mut f.runner.state.config.agents[0];
17027 author.command = vec![
17028 "sh".into(),
17029 "-c".into(),
17030 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17031 ];
17032 author.env.insert(
17033 "PROMPT_FILE".into(),
17034 captured.to_string_lossy().into_owned(),
17035 );
17036 let m = posted(&mut f).await;
17037 assert!(m.body.contains("Adds `own.txt`"));
17038 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17039 assert!(m.body.contains("R1-1-1"));
17040 assert!(
17041 m.body
17042 .contains("Gate (`verify.gate`): passed on the final commit")
17043 );
17044 assert!(
17045 m.body
17046 .contains("Tests (`verify.e2e`): passed on the final commit")
17047 );
17048 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17049 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17050 let prompt = std::fs::read_to_string(captured).unwrap();
17051 assert!(prompt.contains(&format!(
17052 "git diff {} {head}",
17053 f.runner.state.change_summary.as_ref().unwrap().base
17054 )));
17055 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17056 assert_eq!(
17057 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17058 "二回目の内部修正報告です。"
17059 );
17060 }
17061 }
17062
17063 #[tokio::test]
17064 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17065 for review_only in [false, true] {
17066 let mut f = summary_fixture();
17067 if !review_only {
17068 f.runner.state.candidates[0].agent = "alpha".into();
17069 }
17070 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17071 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17072 std::fs::write(&f.reply, "no usable description").unwrap();
17073 let failed = posted(&mut f).await;
17074 assert!(
17075 failed
17076 .body
17077 .contains("No description of what this change does could be generated")
17078 );
17079 assert!(failed.body.is_ascii());
17080 assert!(f.runner.state.github_text.is_none());
17081 assert!(f.runner.state.change_summary.is_none());
17082 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17083 let evidence = crate::pr_outcome::summary_evidence(
17084 &f.runner.state,
17085 crate::pr_outcome::FinalHead::At(&head),
17086 );
17087 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17088 assert!(
17089 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17090 .exists()
17091 );
17092 f.runner.state =
17093 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17094 std::fs::write(
17095 &f.reply,
17096 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17097 )
17098 .unwrap();
17099 let retried = posted(&mut f).await;
17100 assert!(retried.body.contains("Adds the committed own file."));
17101 assert_eq!(
17102 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17103 "日本語の修正報告"
17104 );
17105 }
17106 }
17107
17108 #[tokio::test]
17109 async fn same_head_verification_and_findings_invalidate_public_prose() {
17110 let mut f = summary_fixture();
17111 posted(&mut f).await;
17112 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17113 let original = f
17114 .runner
17115 .state
17116 .change_summary
17117 .as_ref()
17118 .unwrap()
17119 .evidence
17120 .clone();
17121 ok_gate(&mut f.runner.state, Some(&head));
17122 std::fs::write(
17123 &f.reply,
17124 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17125 )
17126 .unwrap();
17127 let verified = posted(&mut f).await;
17128 assert!(verified.body.contains("now verified"));
17129 assert_ne!(
17130 f.runner.state.change_summary.as_ref().unwrap().evidence,
17131 original
17132 );
17133 let mut round = round_with_notes(1, Some("内部報告"));
17134 round.reviews.push(ReviewRecord {
17135 attempts: 0,
17136 reviewer: 1,
17137 agent: "alpha".into(),
17138 summary: "内部レビュー".into(),
17139 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17140 vote: None,
17141 failed: None,
17142 duration_ms: 0,
17143 });
17144 round
17145 .fix
17146 .as_mut()
17147 .unwrap()
17148 .rejected
17149 .push(crate::verdict::Rejection {
17150 id: "R1-1-1".into(),
17151 why: "呼び出し元から到達できない".into(),
17152 });
17153 f.runner.state.reviews = vec![round];
17154 let omitted = posted(&mut f).await;
17156 assert!(omitted.body.contains("No description"));
17157 assert!(omitted.body.contains("R1-1-1"));
17158 assert!(!omitted.body.contains("到達"));
17159 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17160 let described = posted(&mut f).await;
17161 assert!(described.body.contains("unreachable from callers"));
17162 assert!(described.body.contains("R1-1-1"));
17163 assert!(!described.body.contains("到達"));
17164 let evidence = f
17165 .runner
17166 .state
17167 .change_summary
17168 .as_ref()
17169 .unwrap()
17170 .evidence
17171 .clone();
17172 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17173 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17174 assert!(posted(&mut f).await.body.contains("rationale updated"));
17175 assert_ne!(
17176 f.runner.state.change_summary.as_ref().unwrap().evidence,
17177 evidence
17178 );
17179 f.runner
17181 .state
17182 .change_summary
17183 .as_mut()
17184 .unwrap()
17185 .evidence
17186 .clear();
17187 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17188 assert!(posted(&mut f).await.body.contains("current specification"));
17189 }
17190
17191 #[tokio::test]
17192 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17193 let mut f = summary_fixture();
17194 let m = posted(&mut f).await;
17195 assert_eq!(m.title, "feat(own): add the own file");
17196 assert!(
17197 m.body
17198 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17199 "{}",
17200 m.body
17201 );
17202 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17203 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17204 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17205 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17206 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17207 let kept = f.runner.state.change_summary.as_ref().unwrap();
17208 assert_eq!(
17209 (kept.head.as_str(), kept.base.as_str()),
17210 (head.as_str(), base.as_str())
17211 );
17212 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17213 }
17214
17215 #[tokio::test]
17216 async fn review_only_summary_receives_the_configured_github_language() {
17217 let mut f = summary_fixture();
17218 f.runner.state.config.graph.github_language = "ja".into();
17219 let captured = f.repo.join("summary-prompt.txt");
17220 let author = &mut f.runner.state.config.agents[0];
17221 author.command = vec![
17222 "sh".into(),
17223 "-c".into(),
17224 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17225 ];
17226 author.env.insert(
17227 "PROMPT_FILE".into(),
17228 captured.to_string_lossy().into_owned(),
17229 );
17230 std::fs::write(
17231 &f.reply,
17232 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17233 )
17234 .unwrap();
17235 let m = posted(&mut f).await;
17236 let prompt = std::fs::read_to_string(captured).unwrap();
17237 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17238 assert!(!prompt.contains("in English"));
17239 assert_eq!(m.title, "feat(own): ファイルを追加");
17240 assert!(m.body.contains("own.txt を追加します。"));
17241 }
17242
17243 #[tokio::test]
17244 async fn review_fixes_come_after_the_whole_change_summary() {
17245 let mut f = summary_fixture();
17246 f.runner.state.reviews = vec![
17247 round_with_notes(1, Some("handled the PATH case")),
17248 round_with_notes(2, None),
17249 ];
17250 let m = posted(&mut f).await;
17251 let summary = m.body.find("## Summary").expect("summary");
17252 let fixes = m.body.find("## Review rounds").expect("review rounds");
17253 assert!(summary < fixes, "{}", m.body);
17254 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17255 assert!(m.body.contains("## Final verification"), "{}", m.body);
17256 }
17257
17258 #[tokio::test]
17259 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17260 let mut f = summary_fixture();
17261 let first = posted(&mut f).await;
17262 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17263 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17265 let again = posted(&mut f).await;
17266 assert_eq!((again.title, again.body), (first.title, first.body));
17267
17268 let g = |args: &[&str]| {
17269 let out = std::process::Command::new("git")
17270 .args(args)
17271 .current_dir(&f.repo)
17272 .quiet()
17273 .output()
17274 .expect("spawn git");
17275 assert!(out.status.success(), "git {args:?}");
17276 };
17277 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17278 g(&["add", "-A"]);
17279 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17280 let second = posted(&mut f).await;
17281 let kept = f.runner.state.change_summary.clone().unwrap();
17282 assert_ne!(kept.head, first_head);
17283 assert!(second.body.contains("Not read yet."), "{}", second.body);
17284 assert!(!second.body.contains("so the first summary is read"));
17285 assert_eq!(second.title, "feat(own): other");
17286 }
17287
17288 #[tokio::test]
17289 async fn an_unavailable_summary_is_stated_and_never_invented() {
17290 let mut f = summary_fixture();
17291 std::fs::write(&f.reply, "no title line here").unwrap();
17292 let m = posted(&mut f).await;
17293 assert!(
17294 m.body
17295 .contains("No description of what this change does could be generated"),
17296 "{}",
17297 m.body
17298 );
17299 assert!(m.body.contains("internal reports remain in local records"));
17300 assert!(f.runner.state.change_summary.is_none());
17301 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17302 let winner = f.runner.state.candidates[0].clone();
17304 let preview = f
17305 .runner
17306 .guarded_pr_message(&winner, None, false)
17307 .await
17308 .unwrap()
17309 .unwrap();
17310 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17311 }
17312
17313 #[test]
17314 fn pr_message_review_single_commit_uses_its_subject() {
17315 let state = review_state(&["feat(nats): per-role user"]);
17316 let m = pr_message(&state, 'A');
17317 assert_eq!(m.title, "feat(nats): per-role user");
17318 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17319 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17320 }
17321
17322 #[test]
17323 fn pr_message_review_multi_commit_takes_the_oldest() {
17324 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17325 let m = pr_message(&state, 'A');
17326 assert_eq!(m.title, "feat: the change");
17327 for s in ["feat: the change", "fix: typo", "fix: again"] {
17328 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17329 }
17330 }
17331
17332 #[test]
17333 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17334 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17335 let mut state = review_state(&[first, "fix: later fixup"]);
17336 state.candidates[0].branch = "機能/ログイン".to_owned();
17337 let m = pr_message(&state, 'A');
17338 assert!(
17339 m.title.starts_with("chore: land candidate A of run"),
17340 "{}",
17341 m.title
17342 );
17343 }
17344 }
17345
17346 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17347 BranchFacts {
17348 commits: commits
17349 .iter()
17350 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17351 .collect(),
17352 stat: stat.to_owned(),
17353 }
17354 }
17355
17356 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17357 let mut r = review_round(true, 0, 1, 1, true, true);
17358 r.round = round;
17359 r.fix = notes.map(|n| FixRecord {
17360 agent: "fixer".to_owned(),
17361 addressed: Vec::new(),
17362 rejected: Vec::new(),
17363 notes: n.to_owned(),
17364 committed: true,
17365 failed: None,
17366 duration_ms: 0,
17367 continuation: None,
17368 });
17369 r
17370 }
17371
17372 #[test]
17373 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17374 let state = review_state(&["ignored"]);
17375 let f = facts(
17376 &[
17377 (
17378 "fix(login): resolve PATH on macOS",
17379 "Login shells skip rc files.",
17380 ),
17381 ("fix: address review", ""),
17382 ],
17383 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17384 );
17385 let m = pr_message_with(&state, 'A', Some(&f));
17386 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17387 assert!(
17388 m.body
17389 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17390 "{}",
17391 m.body
17392 );
17393 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17394 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17395 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17396 for banned in [
17397 "Review the work already",
17398 "no task statement",
17399 "Original task",
17400 ] {
17401 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17402 }
17403 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17404 }
17405
17406 #[test]
17407 fn pr_message_review_truncates_a_huge_first_commit_body() {
17408 let state = review_state(&["ignored"]);
17409 let f = facts(
17410 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17411 "s",
17412 );
17413 let m = pr_message_with(&state, 'A', Some(&f));
17414 assert!(m.body.len() < 30_000, "{}", m.body.len());
17415 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17416 assert!(
17417 m.body.contains("- ... 1 more commit(s)"),
17418 "{}",
17419 m.body.len()
17420 );
17421 assert!(m.body.ends_with("magi:candidate-a\n"));
17422 }
17423
17424 #[test]
17425 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17426 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17427 assert_eq!(m.title, "feat: x");
17428 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17429 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17430 assert!(!m.body.contains("no task statement"), "{}", m.body);
17431 }
17432
17433 #[test]
17434 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17435 let mut state = review_state(&["日本語の件名"]);
17436 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17437 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17438 state.candidates[0].branch = "機能/ログイン".to_owned();
17439 assert!(
17440 pr_message(&state, 'A')
17441 .title
17442 .starts_with("chore: land candidate A")
17443 );
17444 }
17445
17446 #[test]
17447 fn pr_message_review_fixes_survive_a_clean_final_round() {
17448 let mut state = review_state(&["feat: x"]);
17449 state.reviews = vec![
17450 round_with_notes(1, Some("handled the PATH case")),
17451 round_with_notes(2, None),
17452 ];
17453 let body = pr_message(&state, 'A').body;
17454 assert!(
17455 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17456 "{body}"
17457 );
17458 assert!(body.contains("### Round 1"), "{body}");
17459
17460 state.reviews = vec![
17461 round_with_notes(1, Some("first fix")),
17462 round_with_notes(2, Some("")),
17463 round_with_notes(3, Some("second fix")),
17464 round_with_notes(4, None),
17465 ];
17466 let body = pr_message(&state, 'A').body;
17467 assert!(
17468 body.contains("### Round 1") && !body.contains("first fix"),
17469 "{body}"
17470 );
17471 assert!(
17472 body.contains("### Round 3") && !body.contains("second fix"),
17473 "{body}"
17474 );
17475 assert!(body.contains("### Round 2"), "{body}");
17476 }
17477
17478 const FINAL: &str = "abcdef1234567890";
17479
17480 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17481 let mut r = round_with_notes(round, None);
17482 r.verified_head = verified.map(str::to_owned);
17483 r
17484 }
17485
17486 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17487 state.gate = vec![CommandOutcome {
17488 command: "gate".to_owned(),
17489 code: Some(0),
17490 output_tail: String::new(),
17491 duration_ms: 0,
17492 resource_blocked: false,
17493 }];
17494 state.gate_ran = true;
17495 state.gate_head = head.map(str::to_owned);
17496 }
17497
17498 fn body_at(state: &RunState) -> String {
17499 pr_message_raw(
17500 state,
17501 'A',
17502 None,
17503 crate::pr_outcome::FinalHead::At(FINAL),
17504 ChangeNote::NotAsked,
17505 )
17506 .body
17507 }
17508
17509 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17510
17511 #[test]
17512 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17513 let mut state = review_state(&["feat: x"]);
17514 state.reviews = vec![
17515 round_with_notes(1, Some(SANDBOXED)),
17516 passing_round(2, Some(FINAL)),
17517 ];
17518 ok_gate(&mut state, Some(FINAL));
17519 let body = body_at(&state);
17520 assert!(!body.contains("Handled the PATH case."), "{body}");
17521 assert!(!body.contains("could not commit"), "{body}");
17522 assert!(!body.contains("could not run"), "{body}");
17523 assert!(
17524 body.contains("The fix is committed on the branch"),
17525 "{body}"
17526 );
17527 assert!(!body.contains("sandbox"), "{body}");
17528 assert!(
17529 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17530 "{body}"
17531 );
17532 assert!(
17533 body.contains("Gate (`verify.gate`): passed on the final commit"),
17534 "{body}"
17535 );
17536 assert!(
17538 state.reviews[0]
17539 .fix
17540 .as_ref()
17541 .unwrap()
17542 .notes
17543 .contains("could not commit")
17544 );
17545 }
17546
17547 #[test]
17548 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17549 let mut state = review_state(&["feat: x"]);
17550 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17551 r1.fix.as_mut().unwrap().committed = false;
17552 let mut r2 = passing_round(2, Some(FINAL));
17553 r2.e2e[0].code = Some(1);
17554 state.reviews = vec![r1, r2];
17555 let body = body_at(&state);
17556 assert!(!body.contains("could not commit"), "{body}");
17557 assert!(!body.contains("could not run"), "{body}");
17558 assert!(body.contains("This round produced no commit"), "{body}");
17559 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17560 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17561 assert!(!body.contains("passed"), "{body}");
17562 }
17563
17564 #[test]
17565 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17566 let mut state = review_state(&["feat: x"]);
17567 state.reviews = vec![
17568 round_with_notes(1, Some(SANDBOXED)),
17569 passing_round(2, Some("0123456789abcdef")),
17570 ];
17571 ok_gate(&mut state, Some("0123456789abcdef"));
17572 let body = body_at(&state);
17573 assert!(!body.contains("HTTP tests could not run"), "{body}");
17574 assert!(!body.contains("could not commit"), "{body}");
17576 assert!(
17577 body.contains("passed on an earlier commit (`0123456`)"),
17578 "{body}"
17579 );
17580 assert!(!body.contains("passed on the final commit"), "{body}");
17581
17582 ok_gate(&mut state, None);
17584 let body = body_at(&state);
17585 assert!(
17586 body.contains("passed on a commit that was not recorded"),
17587 "{body}"
17588 );
17589 }
17590
17591 #[test]
17592 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17593 let state = state_with_summary(
17594 "add retries to the client",
17595 "TITLE: feat: retries\n\nDid it.",
17596 );
17597 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17598 assert_eq!(m.title, "feat: retries");
17599 assert!(
17600 m.body.contains("<summary>Original task</summary>"),
17601 "{}",
17602 m.body
17603 );
17604 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17605 assert!(
17606 m.body
17607 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17608 "{}",
17609 m.body
17610 );
17611 }
17612
17613 #[test]
17614 fn pr_message_review_bounds_a_long_english_subject() {
17615 let long = format!("feat: {}", "word ".repeat(100));
17616 let m = pr_message(&review_state(&[&long]), 'A');
17617 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17618 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17619 }
17620
17621 #[test]
17622 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17623 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17624 let m = pr_message(&state, 'A');
17625 assert_eq!(m.title, "add retries");
17626 assert!(m.body.contains("<summary>Original task</summary>"));
17627 assert!(!m.body.contains("Commits under review"));
17628 assert_eq!(landing_subject_source(&state), state.instruction);
17629 }
17630
17631 #[test]
17632 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17633 let state = review_state(&["feat: the change", "fix: typo"]);
17634 let source = landing_subject_source(&state);
17635 assert_eq!(land::merge_subject("", &source), "feat: the change");
17636 assert_eq!(
17637 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17638 "feat: the change"
17639 );
17640 assert_eq!(
17642 land::merge_subject("feat: renamed by hand", &source),
17643 "feat: renamed by hand"
17644 );
17645 let blank = review_state(&["日本語"]);
17646 assert!(
17647 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17648 );
17649 }
17650
17651 #[test]
17652 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17653 let state = review_state(&["feat: the change"]);
17654 let source = landing_subject_source(&state);
17655 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17656 assert_eq!(
17657 land::merge_subject(landing_title(&state, old), &source),
17658 "feat: the change"
17659 );
17660 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17661 let task = state_with_summary("add retries", "");
17662 assert_eq!(landing_title(&task, old), old);
17663 }
17664
17665 #[test]
17666 fn pr_message_describes_the_change_not_the_task() {
17667 let state = state_with_summary(
17668 "今回やってほしいこと: results projector を直す",
17669 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17670 );
17671 let m = pr_message(&state, 'A');
17672 assert_eq!(m.title, "fix(web): batch the runs list reads");
17673 assert!(
17674 m.body.starts_with("## Summary\n\n- reads run.json once"),
17675 "{}",
17676 m.body
17677 );
17678 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17679 let task_at = m.body.find("今回やってほしいこと").unwrap();
17680 let details_at = m.body.find("<details>").unwrap();
17681 assert!(
17682 details_at < task_at,
17683 "the task lives inside <details>: {}",
17684 m.body
17685 );
17686 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17687 assert!(m.body.contains("magi:candidate-a"));
17688 }
17689
17690 #[test]
17691 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17692 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17693 let m = pr_message(&state, 'A');
17694 assert_eq!(m.title, "add retries");
17695 assert!(
17696 m.body.contains("## Summary\n\n- did some things"),
17697 "{}",
17698 m.body
17699 );
17700
17701 let none = RunState::new(
17702 PathBuf::from("/repo"),
17703 "main".to_owned(),
17704 "abc1234".to_owned(),
17705 "add retries".to_owned(),
17706 Config::default(),
17707 );
17708 let m = pr_message(&none, 'A');
17709 assert_eq!(m.title, "add retries");
17710 assert!(!m.body.contains("## Summary"), "{}", m.body);
17711 }
17712
17713 #[test]
17714 fn pr_message_refuses_the_candidate_commit_subject() {
17715 for bad in [
17716 "TITLE: magi: candidate A (uncommitted work)",
17717 "TITLE: chore: stuff (uncommitted work)",
17718 "TITLE: ",
17719 ] {
17720 let state = state_with_summary("add retries", bad);
17721 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17722 }
17723 }
17724
17725 #[test]
17726 fn pr_message_bounds_a_very_long_task_and_title() {
17727 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17728 let state = state_with_summary(&long, "- nothing");
17729 let m = pr_message(&state, 'A');
17730 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17731 assert!(!m.title.contains('\n'));
17732
17733 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17734 let m = pr_message(&state, 'A');
17735 assert!(m.title.starts_with("feat: "));
17736 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17737 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17738 }
17739
17740 fn long_title_of(instruction: &str) -> String {
17741 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17742 }
17743
17744 #[test]
17745 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17746 let first = "Make the landing path keep a readable title for long tasks";
17747 let line = format!(
17748 "{first}. {}",
17749 "And then keep going with more words ".repeat(20)
17750 );
17751 let t = long_title_of(&line);
17752 assert_eq!(t, first);
17753 assert!(!t.starts_with("chore: land"));
17754 }
17755
17756 #[test]
17757 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17758 let line = "word ".repeat(200);
17759 let t = long_title_of(&line);
17760 assert!(t.ends_with("word..."), "{t}");
17761 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17762 }
17763
17764 #[test]
17765 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17766 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17767 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17768 }
17769 }
17770
17771 #[test]
17772 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17773 let mut state = state_with_summary("再試行を追加する", "- nothing");
17774 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17775 state.config.graph.github_language = "ja".into();
17776 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17777 }
17778
17779 #[test]
17780 fn pr_message_title_limit_is_exact() {
17781 let at = "a".repeat(PR_TITLE_MAX);
17782 assert_eq!(long_title_of(&at), at);
17783 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17784 assert!(over.ends_with("..."), "{over}");
17785 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17786 }
17787
17788 #[test]
17789 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17790 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17791 let t = long_title_of(&line);
17792 assert!(t.ends_with("..."), "{t}");
17793 assert!(t.is_ascii(), "{t}");
17794 }
17795
17796 #[test]
17797 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17798 let line = format!(
17799 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17800 "plus more filler words ".repeat(20)
17801 );
17802 let t = long_title_of(&line);
17803 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17804 }
17805
17806 #[test]
17807 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17808 let line = format!("\n\n# {}", "title words ".repeat(40));
17809 let state = state_with_summary(&line, "- nothing");
17810 let m = pr_message(&state, 'A');
17811 assert!(m.title.starts_with("title words"), "{}", m.title);
17812 assert_eq!(
17813 land::merge_subject(&m.title, &landing_subject_source(&state)),
17814 m.title
17815 );
17816 assert_eq!(
17818 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17819 "feat: renamed by hand"
17820 );
17821 }
17822
17823 #[tokio::test]
17824 async fn github_text_rewrite_is_bounded_and_falls_back() {
17825 crate::run::pin_test_home();
17826 for (reply, accepted) in [
17827 (
17828 "TITLE: fix: retries\nAdd retries for failed requests.",
17829 true,
17830 ),
17831 (
17832 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17833 false,
17834 ),
17835 (
17836 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17837 true,
17838 ),
17839 (
17840 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17841 true,
17842 ),
17843 (
17844 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17845 true,
17846 ),
17847 (
17848 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17849 false,
17850 ),
17851 ("TITLE: fix: retries\n", false),
17852 ] {
17853 let dir = tempfile::tempdir().unwrap();
17854 let mut runner = runner_at(RunStatus::Gating);
17855 runner.state = state_with_summary(
17856 "add retries",
17857 "TITLE: fix: retries\n日本語の説明を書きます。",
17858 );
17859 runner.state.repo = dir.path().to_owned();
17860 runner.state.candidates[0].worktree = dir.path().to_owned();
17861 let mut author = spec("alpha");
17862 author.command = vec![
17863 "sh".into(),
17864 "-c".into(),
17865 "printf '%s' \"$REWRITE_REPLY\"".into(),
17866 ];
17867 author.env.insert("REWRITE_REPLY".into(), reply.into());
17868 runner.state.config.agents = vec![author];
17869 seed_public_summary(&mut runner);
17870 let winner = runner.state.candidates[0].clone();
17871 let first = runner
17872 .guarded_pr_message(&winner, None, true)
17873 .await
17874 .unwrap();
17875 let message = if accepted {
17876 first.expect("a passing rewrite never asks")
17877 } else {
17878 assert!(first.is_none());
17880 assert!(runner.state.parked);
17881 let store = ask::Questions::open();
17882 let asked: Vec<_> = store
17883 .list()
17884 .into_iter()
17885 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17886 .collect();
17887 assert_eq!(asked.len(), 1);
17888 assert!(!asked[0].detail.contains("secret"));
17889 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17890 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17891 assert_eq!(
17892 asked[0]
17893 .choices
17894 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17895 rewrite.is_some()
17896 );
17897 store
17900 .update(&asked[0].id, |q| {
17901 q.abandon("test");
17902 Ok(())
17903 })
17904 .unwrap();
17905 let again = runner
17906 .guarded_pr_message(&winner, None, true)
17907 .await
17908 .unwrap()
17909 .expect("an abandoned question falls back");
17910 let n = store
17911 .list()
17912 .into_iter()
17913 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17914 .count();
17915 assert_eq!(n, 1, "asked once per rejected text");
17916 again
17917 };
17918 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17919 assert!(!message.body.contains("secret"));
17920 assert!(!message.body.contains("prefix"));
17921 assert!(!message.title.contains("secret"));
17922 assert_eq!(
17923 message.body.contains("Add retries for failed requests."),
17924 accepted
17925 );
17926 assert_eq!(
17927 runner.state.seats["github-text-rewrite"].turns, 1,
17928 "only one rewrite invocation"
17929 );
17930 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17931 if accepted {
17932 assert!(runner.state.github_text.is_none());
17933 assert!(!runner.state.parked);
17934 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17935 assert!(message.body.contains("## Final verification"));
17936 assert!(message.body.contains("Evidence: final diff"));
17937 assert!(
17938 message
17939 .body
17940 .contains(&format!("magi:run/{}", runner.state.id))
17941 );
17942 }
17943 if !accepted {
17944 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17945 assert!(
17946 message
17947 .body
17948 .contains(&format!("magi:run/{}", runner.state.id))
17949 );
17950 }
17951 }
17952 }
17953
17954 fn scripted(id: &str, reply: &str) -> AgentSpec {
17955 let mut a = spec(id);
17956 a.command = vec![
17957 "sh".into(),
17958 "-c".into(),
17959 "printf '%s' \"$REWRITE_REPLY\"".into(),
17960 ];
17961 a.env.insert("REWRITE_REPLY".into(), reply.into());
17962 a
17963 }
17964
17965 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17966 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17967
17968 fn seed_public_summary(runner: &mut Runner) {
17970 let repo = &runner.state.repo;
17971 let g = |args: &[&str]| {
17972 let out = std::process::Command::new("git")
17973 .args(args)
17974 .current_dir(repo)
17975 .quiet()
17976 .output()
17977 .unwrap();
17978 assert!(out.status.success(), "git {args:?}");
17979 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17980 };
17981 g(&["init", "-q", "-b", "main"]);
17982 g(&[
17983 "-c",
17984 "user.name=Test",
17985 "-c",
17986 "user.email=test@example.invalid",
17987 "commit",
17988 "-q",
17989 "--allow-empty",
17990 "-m",
17991 "feat: fixture",
17992 ]);
17993 let head = g(&["rev-parse", "HEAD"]);
17994 g(&["update-ref", "refs/remotes/origin/main", &head]);
17995 let evidence = crate::pr_outcome::summary_evidence(
17996 &runner.state,
17997 crate::pr_outcome::FinalHead::At(&head),
17998 );
17999 let generated = &runner.state.candidates[0].summary;
18000 runner.state.change_summary = Some(crate::run::ChangeSummary {
18001 base: head.clone(),
18002 head,
18003 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18004 title: summary_title(generated).unwrap(),
18005 body: summary_without_title(generated),
18006 });
18007 }
18008
18009 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18010 let mut runner = runner_at(RunStatus::Gating);
18011 runner.state = state_with_summary("add retries", JA_SUMMARY);
18012 if review_only {
18013 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18014 }
18015 runner.state.repo = dir.to_owned();
18016 runner.state.candidates[0].worktree = dir.to_owned();
18017 runner.state.config.agents = agents;
18018 seed_public_summary(&mut runner);
18019 runner
18020 }
18021
18022 fn gate_questions(runner: &Runner) -> usize {
18023 ask::Questions::open()
18024 .list()
18025 .into_iter()
18026 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18027 .count()
18028 }
18029
18030 #[tokio::test]
18031 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18032 crate::run::pin_test_home();
18033 let dir = tempfile::tempdir().unwrap();
18034 let mut runner = rewrite_runner(
18035 dir.path(),
18036 true,
18037 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18038 );
18039 runner.state.config.roles.language_judge =
18040 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18041 let winner = runner.state.candidates[0].clone();
18042 let msg = runner
18043 .guarded_pr_message(&winner, None, true)
18044 .await
18045 .unwrap()
18046 .expect("an English rewrite posts without asking");
18047 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18048 assert!(msg.body.contains("<details>kept</details>"));
18049 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18050 assert_eq!(gate_questions(&runner), 0);
18051 assert!(runner.state.github_text.is_none());
18052 assert!(!runner.state.seats.contains_key("impl-A"));
18053 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18054 }
18055
18056 #[tokio::test]
18057 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18058 crate::run::pin_test_home();
18059 let dir = tempfile::tempdir().unwrap();
18060 let mut runner = rewrite_runner(
18061 dir.path(),
18062 false,
18063 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18064 );
18065 runner.state.config.roles.language_judge =
18066 Some(crate::config::AgentChoice::One("beta".into()));
18067 runner.state.seats.insert(
18068 "impl-A".into(),
18069 SeatState::new("impl-A", "alpha", runner.state.seed),
18070 );
18071 let winner = runner.state.candidates[0].clone();
18072 let msg = runner
18073 .guarded_pr_message(&winner, None, true)
18074 .await
18075 .unwrap()
18076 .expect("rewritten");
18077 assert!(msg.body.contains("<details>kept</details>"));
18078 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18079 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18080 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18081 }
18082
18083 #[tokio::test]
18084 async fn rewrite_reuses_a_resumable_implementer_session() {
18085 crate::run::pin_test_home();
18086 let dir = tempfile::tempdir().unwrap();
18087 let mut runner = rewrite_runner(
18088 dir.path(),
18089 false,
18090 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18091 );
18092 runner.state.config.roles.language_judge =
18093 Some(crate::config::AgentChoice::One("beta".into()));
18094 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18095 seat.turns = 1;
18096 runner.state.seats.insert("impl-A".into(), seat);
18097 let winner = runner.state.candidates[0].clone();
18098 runner
18099 .guarded_pr_message(&winner, None, true)
18100 .await
18101 .unwrap()
18102 .expect("rewritten");
18103 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18104 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18105 }
18106
18107 #[tokio::test]
18108 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18109 crate::run::pin_test_home();
18110 let dir = tempfile::tempdir().unwrap();
18111 let mut broken = spec("bad");
18112 broken.command = vec!["false".into()];
18113 let mut runner = rewrite_runner(
18114 dir.path(),
18115 true,
18116 vec![
18117 broken,
18118 scripted("noformat", "Add retries."),
18119 scripted("good", EN_REWRITE),
18120 ],
18121 );
18122 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18123 "bad".into(),
18124 "noformat".into(),
18125 "good".into(),
18126 ]));
18127 let winner = runner.state.candidates[0].clone();
18128 let msg = runner
18129 .guarded_pr_message(&winner, None, true)
18130 .await
18131 .unwrap()
18132 .expect("the third agent rewrites");
18133 assert!(msg.body.contains("`src/net.rs`"));
18134 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18135 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18136 }
18137
18138 #[tokio::test]
18139 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18140 crate::run::pin_test_home();
18141 for (agents, role, saved) in [
18143 (
18144 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18145 Some("beta"),
18146 true,
18147 ),
18148 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18149 (Vec::new(), None, false),
18150 ] {
18151 let dir = tempfile::tempdir().unwrap();
18152 let mut runner = rewrite_runner(dir.path(), true, agents);
18153 runner.state.config.roles.language_judge =
18154 role.map(|r| crate::config::AgentChoice::One(r.into()));
18155 let winner = runner.state.candidates[0].clone();
18156 let first = runner
18157 .guarded_pr_message(&winner, None, true)
18158 .await
18159 .unwrap();
18160 assert!(first.is_none(), "withheld text parks the run");
18161 assert!(runner.state.parked);
18162 assert_eq!(gate_questions(&runner), 1);
18163 let g = runner.state.github_text.as_ref().unwrap();
18164 assert_eq!(g.rewrite.is_some(), saved);
18165 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18166 }
18167 }
18168
18169 #[tokio::test]
18170 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18171 crate::run::pin_test_home();
18172 let dir = tempfile::tempdir().unwrap();
18173 let mut runner = runner_at(RunStatus::Gating);
18174 runner.state =
18175 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18176 runner.state.repo = dir.path().to_owned();
18177 runner.state.candidates[0].worktree = dir.path().to_owned();
18178 let mut author = spec("alpha");
18179 author.command = vec![
18180 "sh".into(),
18181 "-c".into(),
18182 "printf '%s' \"$REWRITE_REPLY\"".into(),
18183 ];
18184 author.env.insert(
18185 "REWRITE_REPLY".into(),
18186 format!(
18187 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18188 "English sentence. ".repeat(600)
18189 ),
18190 );
18191 runner.state.config.agents = vec![author];
18192 seed_public_summary(&mut runner);
18193 let winner = runner.state.candidates[0].clone();
18194 assert!(
18195 runner
18196 .guarded_pr_message(&winner, None, true)
18197 .await
18198 .unwrap()
18199 .is_none()
18200 );
18201 let record = runner.state.github_text.clone().unwrap();
18202 let rewrite = record.rewrite.clone().unwrap();
18203 assert_eq!(rewrite.failed, ["body-language"]);
18204 assert!(
18205 rewrite
18206 .body
18207 .ends_with(&format!("magi:run/{}", runner.state.id))
18208 );
18209 let store = ask::Questions::open();
18210 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18211 assert!(
18212 q.choices
18213 .contains(&crate::github_text::USE_REWRITE.to_owned())
18214 );
18215 for label in [
18216 "ORIGINAL",
18217 "FALLBACK",
18218 "REWRITE",
18219 "Rules still failed: body-language",
18220 "truncated",
18221 ] {
18222 assert!(q.detail.contains(label), "{label}");
18223 }
18224 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18225 &runner.state,
18226 &crate::github_text::artifact_name(&record.fingerprint),
18227 ))
18228 .unwrap();
18229 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18230 runner.state =
18232 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18233 store
18234 .update(&q.id, |q| {
18235 q.say("まだ日本語")?;
18236 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18237 })
18238 .unwrap();
18239 assert!(
18240 runner
18241 .guarded_pr_message(&winner, None, true)
18242 .await
18243 .unwrap()
18244 .is_none()
18245 );
18246 let retry = runner.state.github_text.as_ref().unwrap();
18247 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18248 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18249 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18250 assert!(
18251 q.posting_comparison
18252 .as_ref()
18253 .unwrap()
18254 .matches_detail(&q.detail)
18255 );
18256 store
18257 .update(&q.id, |q| {
18258 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18259 })
18260 .unwrap();
18261 let message = runner
18262 .guarded_pr_message(&winner, None, true)
18263 .await
18264 .unwrap()
18265 .unwrap();
18266 assert_eq!(message.title, rewrite.title);
18267 assert_eq!(message.body, rewrite.body);
18268 assert_eq!(
18269 runner.state.github_text.as_ref().unwrap().choice,
18270 Some(GithubTextChoice::Rewrite)
18271 );
18272 let again = runner
18273 .guarded_pr_message(&winner, None, true)
18274 .await
18275 .unwrap()
18276 .unwrap();
18277 assert_eq!(again.title, message.title);
18278 assert_eq!(again.body, message.body);
18279 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18280 assert!(
18281 !runner
18282 .state
18283 .events
18284 .iter()
18285 .any(|e| e.message.contains(&rewrite.title)
18286 || e.message.contains("English sentence."))
18287 );
18288 }
18289
18290 #[tokio::test]
18291 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18292 crate::run::pin_test_home();
18293 let dir = tempfile::tempdir().unwrap();
18294 let mut runner = runner_at(RunStatus::Gating);
18295 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18296 runner.state.repo = dir.path().to_owned();
18297 runner.state.candidates[0].worktree = dir.path().to_owned();
18298 seed_public_summary(&mut runner);
18299 let winner = runner.state.candidates[0].clone();
18300 assert!(
18301 runner
18302 .guarded_pr_message(&winner, None, true)
18303 .await
18304 .unwrap()
18305 .is_none()
18306 );
18307 let record = runner.state.github_text.as_ref().unwrap();
18308 assert!(record.rewrite.is_none());
18309 let mut old = serde_json::to_value(record).unwrap();
18311 old.as_object_mut().unwrap().remove("rewrite");
18312 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18313 assert!(old.rewrite.is_none());
18314 let id = record.question.clone().unwrap();
18315 ask::Questions::open()
18316 .update(&id, |q| {
18317 q.choices.push(crate::github_text::USE_REWRITE.into());
18318 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18319 })
18320 .unwrap();
18321 let message = runner
18322 .guarded_pr_message(&winner, None, true)
18323 .await
18324 .unwrap()
18325 .unwrap();
18326 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18327 assert_eq!(
18328 runner.state.github_text.as_ref().unwrap().choice,
18329 Some(GithubTextChoice::Fallback)
18330 );
18331 }
18332
18333 #[tokio::test]
18334 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18335 crate::run::pin_test_home();
18336 let dir = tempfile::tempdir().unwrap();
18337 let mut runner = runner_at(RunStatus::Gating);
18338 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18339 runner.state.repo = dir.path().to_owned();
18340 runner.state.candidates[0].worktree = dir.path().to_owned();
18341 seed_public_summary(&mut runner);
18342 let winner = runner.state.candidates[0].clone();
18343 assert!(
18344 runner
18345 .guarded_pr_message(&winner, None, true)
18346 .await
18347 .unwrap()
18348 .is_none()
18349 );
18350 let store = ask::Questions::open();
18351 let find = |run: &str| {
18352 store
18353 .list()
18354 .into_iter()
18355 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18356 .unwrap()
18357 };
18358 let run = runner.state.id.clone();
18359 let q = find(&run);
18360 let fp = runner
18361 .state
18362 .github_text
18363 .as_ref()
18364 .unwrap()
18365 .fingerprint
18366 .clone();
18367 let name = crate::github_text::artifact_name(&fp);
18368 let path = crate::run::artifact_path(&runner.state, &name);
18369 assert!(q.detail.contains(&path.display().to_string()));
18370 assert!(q.detail.contains("title-language: title"));
18371 let saved = std::fs::read_to_string(&path).expect("artifact written");
18372 assert!(saved.contains("日本語のタイトル"));
18373 assert!(q.cwd.is_none());
18374 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18375 let say = |id: &str, text: &str, choice: &str| {
18377 store
18378 .update(id, |q| {
18379 q.thread.push(ask::Turn {
18380 who: ask::Who::Operator,
18381 body: text.to_owned(),
18382 at: jiff::Timestamp::now(),
18383 note: None,
18384 });
18385 q.answer(ask::Answer::Choice(choice.to_owned()))
18386 })
18387 .unwrap();
18388 };
18389 say(&q.id, "まだ日本語", "use my text");
18390 assert!(
18391 runner
18392 .guarded_pr_message(&winner, None, true)
18393 .await
18394 .unwrap()
18395 .is_none()
18396 );
18397 let second = find(&run);
18398 assert_ne!(second.id, q.id);
18399 assert!(second.detail.contains("did not pass"));
18400 assert!(second.detail.contains("日本語のタイトル"));
18403 assert!(!second.detail.contains("まだ日本語"));
18404 assert!(second.detail.contains(&path.display().to_string()));
18405 say(&second.id, "fix: retry failed requests", "use my text");
18406 let message = runner
18407 .guarded_pr_message(&winner, None, true)
18408 .await
18409 .unwrap()
18410 .expect("a vetted replacement is posted");
18411 assert_eq!(message.title, "fix: retry failed requests");
18412 assert_eq!(
18413 runner
18414 .state
18415 .github_text
18416 .as_ref()
18417 .unwrap()
18418 .chosen_title
18419 .as_deref(),
18420 Some("fix: retry failed requests")
18421 );
18422 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18423 let again = runner
18425 .guarded_pr_message(&winner, None, true)
18426 .await
18427 .unwrap()
18428 .unwrap();
18429 assert_eq!(again.title, message.title);
18430 assert_eq!(
18431 store
18432 .list()
18433 .into_iter()
18434 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18435 .count(),
18436 2
18437 );
18438 }
18439
18440 #[test]
18441 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18442 let mut state = state_with_summary(
18446 "add retries",
18447 "TITLE: fix(web): batch reads\n- reads run.json once",
18448 );
18449 state.config.graph.language = "ja".to_owned();
18450 let m = pr_message(&state, 'A');
18451 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18452 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18453
18454 let task = "今回やってほしいこと: results projector を直す";
18457 let mut state = state_with_summary(task, "- no title line");
18458 state.config.graph.language = "ja".to_owned();
18459 let m = pr_message(&state, 'A');
18460 assert_eq!(
18461 m.title,
18462 format!("chore: land candidate A of run {}", state.id)
18463 );
18464 assert!(
18465 m.body.contains(&format!(
18466 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18467 )),
18468 "{}",
18469 m.body
18470 );
18471 }
18472
18473 #[test]
18474 fn pr_message_scrubs_home_paths_and_addresses() {
18475 let state = state_with_summary(
18476 "fix it in /Users/someone/src/x",
18477 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18478 );
18479 let m = pr_message(&state, 'A');
18480 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18481 assert!(!m.body.contains(leak), "{}", m.body);
18482 }
18483 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18484 }
18485
18486 #[test]
18487 fn pr_message_survives_a_task_that_closes_details() {
18488 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18489 let m = pr_message(&state, 'A');
18490 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18491 }
18492
18493 #[test]
18494 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18495 let cmd = manual_merge_command(
18496 MergeStyle::Squash,
18497 Path::new("/repo"),
18498 "b",
18499 "fix: \"quoted\" $(x) `y`\n\nbody",
18500 );
18501 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18502 }
18503
18504 #[test]
18505 fn manual_merge_command_matches_the_configured_style() {
18506 let repo = Path::new("/repo");
18507 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18508
18509 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18510 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18511
18512 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18513 assert_eq!(
18514 squash,
18515 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18516 \"Merge magi run 0832 (candidate A)\""
18517 );
18518
18519 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18520 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18521 }
18522
18523 #[test]
18524 fn a_nudge_gets_a_quarter_of_the_budget() {
18525 assert_eq!(retry_budget(secs(1200), true), secs(300));
18527 assert_eq!(retry_budget(secs(3600), true), secs(900));
18528 }
18529
18530 #[test]
18531 fn a_resent_prompt_keeps_the_whole_budget() {
18532 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18535 assert_eq!(retry_budget(secs(60), false), secs(60));
18536 }
18537
18538 #[test]
18539 fn the_floor_never_exceeds_the_original_budget() {
18540 assert_eq!(retry_budget(secs(60), true), secs(60));
18544 assert_eq!(retry_budget(secs(480), true), secs(120));
18545 assert_eq!(retry_budget(secs(0), true), secs(0));
18546 }
18547
18548 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18549 agent::CommandEvidence {
18550 id: "item1".to_owned(),
18551 description: "cargo test".to_owned(),
18552 exit_code,
18553 result_summary: String::new(),
18554 source: "codex".to_owned(),
18555 }
18556 }
18557
18558 #[test]
18559 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18560 assert!(!has_unconfirmed_command(&[]));
18564 }
18565
18566 #[test]
18567 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18568 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18572 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18573 assert!(!has_unconfirmed_command(&[
18574 evidence(Some(0)),
18575 evidence(Some(101))
18576 ]));
18577 }
18578
18579 #[test]
18580 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18581 assert!(has_unconfirmed_command(&[
18582 evidence(Some(0)),
18583 evidence(None)
18584 ]));
18585 }
18586
18587 #[test]
18588 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18589 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18590 assert_eq!(
18591 verified_noop_claim(true, &[], text).as_deref(),
18592 Some("already fixed by b32cfc4, on main.")
18593 );
18594 }
18595
18596 #[test]
18597 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18598 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18601 assert!(verified_noop_claim(false, &[], text).is_none());
18602 }
18603
18604 #[test]
18605 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18606 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18607 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18608 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18610 }
18611
18612 #[test]
18613 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18614 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18615 }
18616
18617 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18620 runner.state.candidates = shape
18621 .iter()
18622 .enumerate()
18623 .map(|(i, &(empty, verified))| Candidate {
18624 index: i,
18625 label: (b'A' + i as u8) as char,
18626 agent: "sonnet".to_owned(),
18627 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18628 worktree: PathBuf::from(format!("/wt/{i}")),
18629 summary: String::new(),
18630 stat: String::new(),
18631 files: 0,
18632 commits: 0,
18633 empty,
18634 failed: None,
18635 verified_noop: verified.map(str::to_owned),
18636 duration_ms: 0,
18637 folded: false,
18638 })
18639 .collect();
18640 }
18641
18642 #[test]
18643 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18644 ask_test_home();
18645 let mut runner = runner_at(RunStatus::Implementing);
18646 set_candidates(
18647 &mut runner,
18648 &[
18649 (true, Some("already on main at b32cfc4")),
18650 (true, Some("same fix, see the existing test")),
18651 ],
18652 );
18653
18654 runner
18655 .after_implement()
18656 .expect("a verified no-op is not an error");
18657
18658 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18659 }
18660
18661 #[test]
18662 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18663 ask_test_home();
18664 let mut runner = runner_at(RunStatus::Implementing);
18665 set_candidates(
18669 &mut runner,
18670 &[(true, Some("already on main at b32cfc4")), (true, None)],
18671 );
18672
18673 let err = runner
18674 .after_implement()
18675 .expect_err("an unverified empty candidate must still fail the run");
18676
18677 assert!(
18678 err.to_string().contains("no candidate produced a change"),
18679 "{err}"
18680 );
18681 assert_eq!(runner.state.status, RunStatus::Failed);
18682 }
18683
18684 #[test]
18685 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18686 ask_test_home();
18687 let mut runner = runner_at(RunStatus::Implementing);
18688 set_candidates(&mut runner, &[(true, None), (true, None)]);
18689
18690 let err = runner
18691 .after_implement()
18692 .expect_err("no candidate declared anything; this is an ordinary failure");
18693
18694 assert!(
18695 err.to_string().contains("no candidate produced a change"),
18696 "{err}"
18697 );
18698 assert_eq!(runner.state.status, RunStatus::Failed);
18699 }
18700}
18701
18702#[derive(Debug)]
18705struct GraftRestoreFailed(String);
18706
18707impl std::fmt::Display for GraftRestoreFailed {
18708 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18709 write!(
18710 f,
18711 "could not restore the winner to its pre-graft commit: {}",
18712 self.0
18713 )
18714 }
18715}
18716
18717impl std::error::Error for GraftRestoreFailed {}
18718
18719#[cfg(test)]
18723mod resource_wait_tests {
18724 use super::*;
18725 use std::collections::BTreeMap;
18726
18727 fn sh(id: &str, script: &str) -> AgentSpec {
18728 AgentSpec {
18729 id: id.to_owned(),
18730 kind: crate::config::AgentKind::Command,
18731 model: None,
18732 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18733 extra_args: Vec::new(),
18734 env: BTreeMap::new(),
18735 prompt_delivery: None,
18736 }
18737 }
18738
18739 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18740 SeatJob {
18741 seat: SeatState::new("impl-A", &spec.id, 1),
18742 spec,
18743 cwd: dir.to_path_buf(),
18744 prompt: "do it".to_owned(),
18745 timeout,
18746 allow_write: true,
18747 sessions: false,
18748 artifacts: dir.join("art"),
18749 stem: "impl-A".to_owned(),
18750 handover: None,
18751 }
18752 }
18753
18754 fn state(resource_wait_secs: u64) -> RunState {
18755 let mut config = Config::default();
18756 config.graph.timeout_resource_wait = resource_wait_secs;
18757 RunState::new(
18758 PathBuf::from("/repo"),
18759 "main".to_owned(),
18760 "abc1234".to_owned(),
18761 "task".to_owned(),
18762 config,
18763 )
18764 }
18765
18766 fn hold(cache: &Path) -> crate::cache::Guard {
18767 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18768 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18769 crate::cache::AcquireOutcome::Acquired(g) => g,
18770 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18771 }
18772 }
18773
18774 async fn run(
18775 jobs: Vec<SeatJob>,
18776 sem: Arc<Semaphore>,
18777 cache: Option<&Path>,
18778 st: &mut RunState,
18779 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18780 let prompts = Prompts::default();
18781 let ctx = WaveCtx {
18782 run: "r",
18783 node: "implement",
18784 prompts: &prompts,
18785 cache,
18786 round: None,
18787 carry_seats: false,
18788 pauses: None,
18789 };
18790 wave(jobs, sem, &ctx, st, 0).await
18791 }
18792
18793 #[tokio::test]
18794 async fn verification_cache_wait_preserves_the_command_budget() {
18795 crate::run::pin_test_home();
18796 let tmp = tempfile::tempdir().unwrap();
18797 let cache = tmp.path().join("target");
18798 let guard = hold(&cache);
18799 let release = tokio::spawn(async move {
18800 tokio::time::sleep(Duration::from_millis(100)).await;
18801 guard.release();
18802 });
18803 let mut st = state(10);
18804 let budget = Duration::from_millis(50);
18805 let wait = repository_wait(&st, None);
18806 let (outcomes, _) = with_cache_lease(
18807 &mut st,
18808 Some(&cache),
18809 "gate",
18810 "gate",
18811 tmp.path(),
18812 "abc1234",
18813 budget,
18814 "test gate",
18815 &wait,
18816 |_, available| async move {
18817 assert_eq!(
18818 available, budget,
18819 "contention must not shorten verification"
18820 );
18821 (Vec::new(), false, Vec::new())
18822 },
18823 )
18824 .await
18825 .unwrap();
18826 release.await.unwrap();
18827 assert!(outcomes.is_empty(), "{outcomes:?}");
18828 assert!(
18829 st.events
18830 .iter()
18831 .any(|event| event.message.contains("waiting for"))
18832 );
18833 assert!(st.resource_wait.is_none());
18834 hold(&cache).release();
18836 }
18837
18838 #[tokio::test]
18839 async fn verification_cache_wait_expires_without_starting_commands() {
18840 crate::run::pin_test_home();
18841 let tmp = tempfile::tempdir().unwrap();
18842 let cache = tmp.path().join("target");
18843 let _held = hold(&cache);
18844 let mut st = state(0);
18845 let wait = repository_wait(&st, None);
18846 let (outcomes, _) = with_cache_lease(
18847 &mut st,
18848 Some(&cache),
18849 "gate",
18850 "gate",
18851 tmp.path(),
18852 "abc1234",
18853 Duration::from_secs(60),
18854 "test gate",
18855 &wait,
18856 |_, _| async { panic!("commands must not start without the cache lease") },
18857 )
18858 .await
18859 .unwrap();
18860 assert_eq!(outcomes.len(), 1);
18861 assert!(outcomes[0].resource_blocked);
18862 assert!(st.resource_wait.is_none());
18863 }
18864
18865 #[tokio::test]
18866 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18867 crate::run::pin_test_home();
18868 let tmp = tempfile::tempdir().unwrap();
18869 let cache = tmp.path().join("target-a");
18870 let guard = hold(&cache);
18871 tokio::spawn(async move {
18873 tokio::time::sleep(Duration::from_millis(1500)).await;
18874 guard.release();
18875 });
18876 let mut st = state(60);
18877 let results = run(
18878 vec![job(
18879 sh("alpha", "echo ok"),
18880 tmp.path(),
18881 Duration::from_secs(1),
18882 )],
18883 Arc::new(Semaphore::new(1)),
18884 Some(&cache),
18885 &mut st,
18886 )
18887 .await;
18888 assert!(
18889 matches!(results[0].2, AgentOutcome::Ok(_)),
18890 "the wait must not be charged to the agent"
18891 );
18892 assert!(st.resource_block.is_none());
18893 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18894 assert!(st.resource_wait.is_none() && st.active.is_empty());
18895 }
18896
18897 #[tokio::test]
18898 #[cfg(unix)]
18899 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18900 use std::os::fd::AsRawFd as _;
18901 crate::run::pin_test_home();
18902 let tmp = tempfile::tempdir().unwrap();
18903 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18904 let common = tmp.path().join(".git");
18905 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18906 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18907 assert_eq!(
18908 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18909 0
18910 );
18911 let mut st = state(1);
18912 st.config.coordination = crate::config::test_coordination();
18913 let mut reader = job(
18914 sh("reader", "echo must-not-run"),
18915 tmp.path(),
18916 Duration::from_secs(30),
18917 );
18918 reader.allow_write = false;
18919 let results = run(
18920 vec![reader.clone()],
18921 Arc::new(Semaphore::new(1)),
18922 None,
18923 &mut st,
18924 )
18925 .await;
18926 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18927 assert_eq!(results[0].1.turns, 0);
18928 assert!(FailClass::of(&results[0].2).is_none());
18929 assert_eq!(
18930 st.resource_block.as_ref().unwrap().kind,
18931 "repository coordination"
18932 );
18933 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18934 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18935 drop(external);
18936 let editor = crate::repo_lock::SeatLease::for_checkout(
18937 tmp.path(),
18938 true,
18939 crate::config::test_coordination().as_ref(),
18940 )
18941 .await
18942 .unwrap();
18943 let pause = Pause::new();
18944 let interrupt = Pause::new();
18945 let prompts = Prompts::default();
18946 let ctx = WaveCtx {
18947 run: "r",
18948 node: "review",
18949 prompts: &prompts,
18950 cache: None,
18951 round: None,
18952 carry_seats: false,
18953 pauses: Some((&pause, &interrupt)),
18954 };
18955 let (results, ()) = tokio::join!(
18956 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18957 async {
18958 tokio::time::sleep(Duration::from_millis(50)).await;
18959 pause.park();
18960 }
18961 );
18962 assert!(
18963 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18964 );
18965 assert_eq!(results[0].1.turns, 0);
18966 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18967 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18968 drop(editor);
18969 }
18970
18971 #[tokio::test]
18972 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18973 crate::run::pin_test_home();
18974 let tmp = tempfile::tempdir().unwrap();
18975 let cache = tmp.path().join("target-b");
18976 let _held = hold(&cache);
18977 let mut st = state(1);
18978 let results = run(
18979 vec![job(
18980 sh("alpha", "echo ok"),
18981 tmp.path(),
18982 Duration::from_secs(1),
18983 )],
18984 Arc::new(Semaphore::new(1)),
18985 Some(&cache),
18986 &mut st,
18987 )
18988 .await;
18989 let (_, seat, out) = &results[0];
18990 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18991 assert!(wave_blocked(&results));
18992 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18993 assert_eq!(seat.turns, 0, "nothing was asked");
18994 assert!(
18995 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18996 "no agent was spawned"
18997 );
18998 assert!(FailClass::of(out).is_none(), "not a failure class");
18999 let block = st.resource_block.as_ref().expect("recorded for the resume");
19000 assert_eq!(block.kind, "shared build cache");
19001 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19002 assert!(st.active.is_empty());
19003 }
19004
19005 #[tokio::test]
19006 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19007 crate::run::pin_test_home();
19008 let tmp = tempfile::tempdir().unwrap();
19009 let mut st = state(1);
19010 let results = run(
19011 vec![job(
19012 sh("alpha", "echo ok"),
19013 tmp.path(),
19014 Duration::from_secs(1),
19015 )],
19016 Arc::new(Semaphore::new(0)),
19017 None,
19018 &mut st,
19019 )
19020 .await;
19021 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19022 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19023 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19024 }
19025
19026 #[tokio::test]
19027 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19028 crate::run::pin_test_home();
19029 let tmp = tempfile::tempdir().unwrap();
19030 let mut st = state(60);
19031 let results = run(
19032 vec![job(
19033 sh("alpha", "sleep 30"),
19034 tmp.path(),
19035 Duration::from_secs(1),
19036 )],
19037 Arc::new(Semaphore::new(1)),
19038 None,
19039 &mut st,
19040 )
19041 .await;
19042 let out = &results[0].2;
19043 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19044 let class = FailClass::of(out).expect("a failure");
19045 assert_eq!(class, FailClass::Timeout);
19046 assert!(should_hand_over(None, &class), "existing handover policy");
19047 assert!(st.resource_block.is_none());
19048 }
19049}