1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
188}
189
190#[derive(Debug, Clone, Default)]
217pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
218
219impl Pause {
220 #[must_use]
222 pub fn new() -> Self {
223 Self::default()
224 }
225
226 pub fn park(&self) {
228 self.0.store(true, Ordering::SeqCst);
229 }
230
231 pub fn park_because(&self, reason: impl Into<String>) {
237 self.park_for(ParkCause::Interrupt {
238 reason: reason.into(),
239 });
240 }
241
242 pub fn park_for(&self, cause: ParkCause) {
245 let mut reason_guard = self
246 .1
247 .lock()
248 .unwrap_or_else(std::sync::PoisonError::into_inner);
249 if reason_guard.is_none() {
250 *reason_guard = Some(cause);
251 }
252 drop(reason_guard);
253 self.park();
254 }
255
256 #[must_use]
258 pub fn parked(&self) -> bool {
259 self.0.load(Ordering::SeqCst)
260 }
261
262 #[must_use]
264 pub fn reason(&self) -> Option<String> {
265 self.cause().map(|c| c.label())
266 }
267
268 #[must_use]
270 pub fn cause(&self) -> Option<ParkCause> {
271 self.1
272 .lock()
273 .unwrap_or_else(std::sync::PoisonError::into_inner)
274 .clone()
275 }
276}
277
278pub struct Runner {
280 pub state: RunState,
282 roles: ResolvedRoles,
283 sem: Arc<Semaphore>,
284 pause: Pause,
288 interrupt: Pause,
294}
295
296async fn review_base(
301 repo: &Path,
302 remote: &str,
303 base_branch: &str,
304 base_commit: &str,
305 branch: &str,
306) -> String {
307 review_base_checked(repo, remote, base_branch, base_commit, branch)
308 .await
309 .0
310}
311
312async fn review_base_checked(
317 repo: &Path,
318 remote: &str,
319 base_branch: &str,
320 base_commit: &str,
321 branch: &str,
322) -> (String, bool) {
323 let tracking = format!("{remote}/{base_branch}");
324 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
325 if git::rev_exists(repo, &tracking).await
326 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
327 && !mb.is_empty()
328 {
329 return (mb, fresh);
330 }
331 let mb = git::merge_base(repo, base_commit, branch)
332 .await
333 .ok()
334 .filter(|mb| !mb.is_empty())
335 .unwrap_or_else(|| base_commit.to_owned());
336 (mb, false)
337}
338
339pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
343 if !is_review_run(state) {
344 return;
345 }
346 let base = review_base(
347 &state.repo,
348 &state.config.merge.remote,
349 &state.base_branch,
350 &state.base_commit,
351 branch,
352 )
353 .await;
354 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
355 && !subjects.is_empty()
356 && state.reviewed_commits.as_ref() != Some(&subjects)
357 {
358 state.reviewed_commits = Some(subjects);
359 state.save().ok();
360 }
361}
362
363async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
367 let (base, trusted) = review_base_checked(
368 &state.repo,
369 &state.config.merge.remote,
370 &state.base_branch,
371 &state.base_commit,
372 branch,
373 )
374 .await;
375 if !trusted {
376 return None;
377 }
378 git::subjects(&state.repo, &state.base_commit, &base)
379 .await
380 .ok()
381}
382
383pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
387 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
388 if is_leak(computed) {
389 return false;
390 }
391 let cur = current.trim();
392 cur.is_empty()
393 || cur.starts_with(REVIEW_PROMPT_OPENING)
394 || cur.starts_with("chore: land candidate")
395 || cur.starts_with("magi: candidate")
396 || is_leak(cur)
397}
398
399async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
428 let tracking = format!("{remote}/{branch}");
429 let fetched = git::fetch(repo, remote, branch).await;
430 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
431 let local_exists = git::branch_exists(repo, branch).await?;
432 if !fresh {
433 if !local_exists {
434 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
435 }
436 tracing::warn!(
437 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
438 );
439 return Ok(());
440 }
441 let remote_sha = git::rev_parse(repo, &tracking).await?;
442 if !local_exists {
443 git::git(repo, &["branch", branch, &tracking]).await?;
444 return Ok(());
445 }
446 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
447 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
448 return Ok(());
449 }
450 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
451 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
457 .await?
458 {
459 crate::reconcile::Reconciliation::Pushed => {
460 tracing::warn!(
461 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
462 short(&local_sha),
463 short(&remote_sha)
464 );
465 return Ok(());
466 }
467 crate::reconcile::Reconciliation::Placeholder => {}
468 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
469 }
470 }
471 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
472 if !out.ok() {
473 bail!(
474 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
475 short(&local_sha),
476 short(&remote_sha),
477 out.stderr
478 );
479 }
480 tracing::warn!(
481 "local `{branch}` was stale: fast-forwarded {} -> {}",
482 short(&local_sha),
483 short(&remote_sha)
484 );
485 Ok(())
486}
487
488async fn resolve_base(
489 repo: &Path,
490 base_branch: &str,
491 remote: &str,
492 coordination: Option<&crate::config::Coordination>,
493) -> Result<crate::run::FetchedBase> {
494 let _lease = crate::repo_lock::SeatLease::operation_with(
495 repo,
496 coordination,
497 &crate::repo_lock::WaitPolicy::current(),
498 )
499 .await?;
500 let tracking = format!("{remote}/{base_branch}");
501 let fetched = git::fetch(repo, remote, base_branch).await;
502 if let Ok(out) = &fetched
503 && out.ok()
504 && git::rev_exists(repo, &tracking).await
505 {
506 return Ok(crate::run::FetchedBase {
507 remote: remote.to_owned(),
508 branch: base_branch.to_owned(),
509 commit: git::rev_parse(repo, &tracking).await?,
510 fetched_at: jiff::Timestamp::now(),
511 });
512 }
513 let why = match &fetched {
514 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
515 Ok(_) => format!("{remote} has no {base_branch}"),
516 Err(e) => e.to_string(),
517 };
518 bail!(
521 "cannot read {tracking} ({why}); refusing to branch off the local \
522 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
523 base / remote in magi.toml. This is launcher preparation: check the \
524 launcher's Git administration access, network and repository coordination \
525 before retrying; an unattended seat cannot approve those permissions"
526 )
527}
528
529struct FixClaim {
546 path: PathBuf,
547}
548
549impl FixClaim {
550 fn acquire(dir: &Path) -> Result<Self> {
551 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
552 let path = dir.join("fix.lock");
553 match Self::create(&path) {
554 Ok(claim) => Ok(claim),
555 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
556 if Self::reclaim_if_dead(&path) {
557 Self::create(&path).with_context(|| format!("lock {}", path.display()))
558 } else {
559 bail!(
560 "another repair or `magi fix` is already running for this run ({} exists)",
561 path.display()
562 )
563 }
564 }
565 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
566 }
567 }
568
569 fn create(path: &Path) -> std::io::Result<Self> {
570 let mut f = std::fs::OpenOptions::new()
571 .write(true)
572 .create_new(true)
573 .open(path)?;
574 use std::io::Write as _;
575 writeln!(f, "{}", std::process::id())?;
577 Ok(Self {
578 path: path.to_owned(),
579 })
580 }
581
582 fn reclaim_if_dead(path: &Path) -> bool {
586 let dead = std::fs::read_to_string(path)
587 .ok()
588 .and_then(|body| body.trim().parse::<u32>().ok())
589 .is_some_and(|pid| !crate::proc::pid_alive(pid));
590 dead && std::fs::remove_file(path).is_ok()
591 }
592}
593
594impl Drop for FixClaim {
595 fn drop(&mut self) {
596 let _ = std::fs::remove_file(&self.path);
597 }
598}
599
600struct DriverExecution<'a> {
603 runner: &'a mut Runner,
604 driver: crate::driver::DriverGuard,
605}
606
607impl Drop for DriverExecution<'_> {
608 fn drop(&mut self) {
609 self.driver.release_local(&mut self.runner.state);
610 }
611}
612
613impl Runner {
614 pub async fn start(
616 repo: &Path,
617 instruction: String,
618 config: Config,
619 origin: Origin,
620 ) -> Result<Self> {
621 Self::start_naming(repo, instruction, "", config, origin).await
622 }
623
624 pub async fn start_naming(
628 repo: &Path,
629 instruction: String,
630 also_scan: &str,
631 config: Config,
632 origin: Origin,
633 ) -> Result<Self> {
634 let repo = git::toplevel(repo).await?;
635 let missing = agent::missing_programs(&config.agents);
636 if !missing.is_empty() {
637 bail!(
638 "these agent programs are not on PATH: {}. Fix the roster in \
639 magi.toml or install them.",
640 missing.join(", ")
641 );
642 }
643 let base_branch =
644 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
645 .await?;
646 let fetched_base = resolve_base(
647 &repo,
648 &base_branch,
649 &config.merge.remote,
650 config.coordination.as_ref(),
651 )
652 .await?;
653 let base_commit = fetched_base.commit.clone();
654 let roles = config.resolve_roles()?;
655 let max_parallel = config.graph.max_parallel.max(1);
656 let seeds = refs::resolve(
659 &repo,
660 &base_commit,
661 &config.merge.remote,
662 &format!("{also_scan}\n{instruction}"),
663 )
664 .await;
665 refs::plan(&repo, &seeds).await?;
666 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
667 state.fetched_base = Some(fetched_base);
671 state.origin = Some(origin);
672 for seed in &seeds {
673 state.event(
674 "seed",
675 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
676 );
677 }
678 state.seeds = seeds;
679 state.event("start", format!("run {} created", state.id));
680 state.save()?;
681 Ok(Self {
682 state,
683 roles,
684 sem: Arc::new(Semaphore::new(max_parallel)),
685 pause: Pause::new(),
686 interrupt: Pause::new(),
687 })
688 }
689
690 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
704 Self::review_taking_over(repo, branch, config, None, origin).await
705 }
706
707 pub async fn review_taking_over(
714 repo: &Path,
715 branch: &str,
716 config: Config,
717 takeover: Option<crate::handover::Takeover>,
718 origin: Origin,
719 ) -> Result<Self> {
720 let repo = git::toplevel(repo).await?;
721 let missing = agent::missing_programs(&config.agents);
722 if !missing.is_empty() {
723 bail!(
724 "these agent programs are not on PATH: {}. Fix the roster in \
725 magi.toml or install them.",
726 missing.join(", ")
727 );
728 }
729 let base_branch =
730 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
731 .await?;
732 if base_branch == branch {
733 bail!("`{branch}` is the base branch; there is nothing to review against");
734 }
735 let fetched_base = resolve_base(
736 &repo,
737 &base_branch,
738 &config.merge.remote,
739 config.coordination.as_ref(),
740 )
741 .await?;
742 let base_commit = fetched_base.commit.clone();
743
744 let roles = config.resolve_roles()?;
745 let max_parallel = config.graph.max_parallel.max(1);
746 let mut state = RunState::new(
747 repo.clone(),
748 base_branch,
749 base_commit.clone(),
750 String::new(),
751 config,
752 );
753 state.fetched_base = Some(fetched_base);
754 state.origin = Some(origin);
755
756 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
763 earlier: Vec::new(),
764 home: crate::run::home(),
765 choice: None,
766 task_instruction: None,
767 });
768 let wait = repository_wait(&state, None);
769 let coordination_config = state.config.coordination.clone();
770 let coordination =
771 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
772 .await?;
773 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
774 if let Some(released) = &released {
775 state.event(
776 "release",
777 format!(
778 "took `{branch}` over from run {}: its worktree was released: {}",
779 crate::run::short_of(&released.old_id),
780 released.audit
781 ),
782 );
783 }
784 if let Some(choice) = takeover.choice.as_ref()
788 && let Err(e) =
789 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
790 .await
791 {
792 if let Some(released) = &released {
793 released.restore(&repo, branch).await;
794 }
795 return Err(e.context("applying the owner's answer about the diverged branch"));
796 }
797 for id in takeover.earlier.iter().rev() {
803 if let Ok(old) = RunState::load_under(id, &takeover.home)
804 && let Some(head) = old.pr_head.clone()
805 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
806 {
807 state.pr_head = Some(head);
808 break;
809 }
810 }
811 drop(coordination);
812 let opened = Self::open_review(
813 &repo,
814 branch,
815 state,
816 roles,
817 max_parallel,
818 base_commit,
819 &takeover,
820 )
821 .await;
822 if opened.is_err()
823 && let Some(released) = &released
824 {
825 let _coordination = crate::repo_lock::SeatLease::operation_with(
826 &repo,
827 coordination_config.as_ref(),
828 &wait,
829 )
830 .await?;
831 released.restore(&repo, branch).await;
832 }
833 opened
834 }
835
836 pub async fn recover_pr(
838 repo: &Path,
839 task: &str,
840 selection: crate::pr_recovery::Selection,
841 mut config: Config,
842 ) -> Result<Self> {
843 selection.configure(&mut config)?;
844 selection.check_task(task, &selection.instruction)?;
845 let base_branch =
846 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
847 .await?;
848 let fetched_base = resolve_base(
849 repo,
850 &base_branch,
851 &config.merge.remote,
852 config.coordination.as_ref(),
853 )
854 .await?;
855 let base_commit = fetched_base.commit.clone();
856 selection
857 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
858 .await?;
859 Box::pin(selection.verify_source(repo)).await?;
860 if let Some(proof) = &selection.source_proof {
861 anyhow::ensure!(
862 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
863 "source rebase base is outside the fetched landing base"
864 );
865 }
866 let roles = config.resolve_roles()?;
867 let max_parallel = config.graph.max_parallel.max(1);
868 let mut state = RunState::new(
869 repo.to_path_buf(),
870 base_branch,
871 base_commit.clone(),
872 selection.instruction.clone(),
873 config,
874 );
875 state.fetched_base = Some(fetched_base);
876 state.origin = Some(crate::run::Origin::queue(task));
877 state.pr_head = Some(selection.branch.clone());
878 state.pr_recovery = Some(selection.clone());
879 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
880 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
881 }
882
883 async fn open_selected(
884 repo: &Path,
885 state: RunState,
886 roles: ResolvedRoles,
887 max_parallel: usize,
888 base_commit: String,
889 selection: crate::pr_recovery::Selection,
890 ) -> Result<Self> {
891 let mut state = state;
892 let branch = state.branch_for('A');
893 let wait = repository_wait(&state, None);
894 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
895 git::git(repo, &["branch", &branch, selection.source()]).await?;
896 drop(coordination);
897 state.event(
898 "recovery",
899 format!(
900 "selected implementation {} for {} (public lease {})",
901 selection.source(),
902 selection.url,
903 selection.head
904 ),
905 );
906 let takeover = crate::handover::Takeover {
907 earlier: Vec::new(),
908 home: crate::run::home(),
909 choice: None,
910 task_instruction: Some(selection.instruction.clone()),
911 };
912 state.save()?;
913 let id = state.id.clone();
914 let opened = Self::open_review(
915 repo,
916 &branch,
917 state,
918 roles,
919 max_parallel,
920 base_commit,
921 &takeover,
922 )
923 .await;
924 match opened {
925 Ok(runner) => Ok(runner),
926 Err(e) => {
927 if let Ok(mut partial) = RunState::load(&id) {
928 partial.status = RunStatus::Blocked;
929 partial.driver_exited = true;
930 partial.event(
931 "recovery",
932 format!("startup failed; retained source and candidate {branch}: {e:#}"),
933 );
934 partial.save()?;
935 }
936 Err(crate::pr_recovery::StartupFailed {
937 run: id,
938 cause: format!("{e:#}"),
939 }
940 .into())
941 }
942 }
943 }
944
945 async fn inherit(
950 repo: &Path,
951 branch: &str,
952 base_commit: &str,
953 takeover: &crate::handover::Takeover,
954 ) -> Option<crate::recovery::RecoveryRecord> {
955 use crate::recovery::{Relation, assess};
956 let newest = takeover.earlier.last()?;
957 let mut chosen = None;
958 for id in takeover.earlier.iter().rev() {
959 if let Ok(old) = RunState::load_under(id, &takeover.home)
960 && !old.reviews.is_empty()
961 && (old.winner().is_some_and(|w| w.branch == branch)
962 || old.pr_head.as_deref() == Some(branch))
963 {
964 chosen = Some(old);
965 break;
966 }
967 }
968 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
969 .await
970 .unwrap_or_default();
971 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
972 Some(last) if last.head == tip => Relation::Same,
973 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
974 _ => Relation::Diverged,
975 };
976 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
977 Some(assess(from, chosen.as_ref(), base_commit, relation))
978 }
979
980 async fn open_review(
983 repo: &Path,
984 branch: &str,
985 mut state: RunState,
986 roles: ResolvedRoles,
987 max_parallel: usize,
988 base_commit: String,
989 takeover: &crate::handover::Takeover,
990 ) -> Result<Self> {
991 let wait = repository_wait(&state, None);
992 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
993 if state.pr_recovery.is_none() {
994 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
995 }
996 let start = review_base(
999 repo,
1000 &state.config.merge.remote,
1001 &state.base_branch,
1002 &base_commit,
1003 branch,
1004 )
1005 .await;
1006 let log = git::log_oneline(repo, &start, branch)
1007 .await
1008 .unwrap_or_default();
1009 let log_text = if log.trim().is_empty() {
1010 "(no commit messages)"
1011 } else {
1012 log.trim()
1013 };
1014 let mut instruction = format!(
1015 "Review the work already on branch `{branch}`. There is no task \
1016 statement: what the change claims to do is whatever its commits \
1017 say.\n\n{log_text}"
1018 );
1019 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1023 if let Some(task) = takeover
1024 .task_instruction
1025 .as_deref()
1026 .filter(|t| !t.trim().is_empty())
1027 && rec.reviewed_head.is_some()
1028 {
1029 instruction = format!(
1030 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1031 task.trim()
1032 );
1033 rec.criteria_inherited = true;
1034 }
1035 state.event("recovery", rec.summary());
1036 state.recovery = Some(rec);
1037 }
1038 if let Some(selected) = &state.pr_recovery {
1039 instruction = selected.instruction.clone();
1040 }
1041 state.task_instruction = takeover.task_instruction.clone();
1042 state.instruction = instruction;
1043 state.reviewed_commits = Some(
1044 git::subjects(repo, &start, branch)
1045 .await
1046 .unwrap_or_default(),
1047 );
1048
1049 let worktree = state.worktree_root().join("under-review");
1052 if let Some(parent) = worktree.parent() {
1053 tokio::fs::create_dir_all(parent).await.ok();
1054 }
1055 let path = worktree.to_string_lossy().to_string();
1056 git::git(repo, &["worktree", "add", &path, branch])
1057 .await
1058 .with_context(|| {
1059 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1060 })?;
1061
1062 let setup = match ensure_setup_config(&mut state, repo).await {
1063 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1064 Err(e) => Err(e),
1065 };
1066 if let Err(e) = setup {
1067 git::worktree_remove_clean(repo, &worktree).await.ok();
1068 return Err(e);
1069 }
1070 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1071 .await
1072 .unwrap_or(0);
1073 if commits == 0 {
1074 git::worktree_remove_clean(repo, &worktree).await.ok();
1075 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1076 }
1077 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1078 .await
1079 .map(|f| f.len())
1080 .unwrap_or(0);
1081 if files == 0
1082 && let (Ok(head_tree), Ok(base_tree)) = (
1083 git::tree_of(&worktree, "HEAD").await,
1084 git::tree_of(&worktree, &base_commit).await,
1085 )
1086 && head_tree == base_tree
1087 {
1088 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1089 git::worktree_remove_clean(repo, &worktree).await.ok();
1090 bail!(
1091 "`{branch}` at {} has a tree identical to base {}; this usually means \
1092 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1093 against `{}/{branch}`) rather than an empty change",
1094 short(&head),
1095 short(&base_commit),
1096 state.config.merge.remote
1097 );
1098 }
1099 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1100 .await
1101 .unwrap_or_default();
1102
1103 state.candidates.push(Candidate {
1104 index: 0,
1105 label: 'A',
1106 agent: EXISTING_BRANCH.to_owned(),
1109 branch: branch.to_owned(),
1110 worktree,
1111 summary: String::new(),
1112 stat,
1113 files,
1114 commits,
1115 empty: false,
1116 failed: None,
1117 verified_noop: None,
1118 duration_ms: 0,
1119 folded: false,
1120 });
1121 state.tally = Some(Tally {
1122 first_choice: BTreeMap::from([('A', 0)]),
1123 borda: BTreeMap::new(),
1124 winner: 'A',
1125 rankings: 0,
1126 unanimous_initial: false,
1127 deliberated: false,
1128 changed_votes: 0,
1129 unanimous_final: false,
1130 tie_break: None,
1131 judges: 0,
1135 present: 0,
1136 quorum: 0,
1137 met_quorum: true,
1138 uncontested: Some("review-only run: nothing competed".to_owned()),
1139 });
1140 state.status = RunStatus::Reviewing;
1141 state.prep_complete = Some(true);
1142 state.event(
1143 "start",
1144 format!(
1145 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1146 state.id
1147 ),
1148 );
1149 state.save()?;
1150 Ok(Self {
1151 state,
1152 roles,
1153 sem: Arc::new(Semaphore::new(max_parallel)),
1154 pause: Pause::new(),
1155 interrupt: Pause::new(),
1156 })
1157 }
1158
1159 pub fn resume(id: &str) -> Result<Self> {
1161 let state = RunState::load(id)?;
1162 crate::pr_head::ensure_owner(&state)?;
1163 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1164 bail!(
1165 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1166 );
1167 }
1168 if let Some(to) = &state.released_to {
1169 bail!(
1170 "run {} cannot be resumed: its worktree was released to run {}",
1171 state.short(),
1172 crate::run::short_of(to)
1173 );
1174 }
1175 let roles = state.config.resolve_roles()?;
1176 if let Some(id) = &state.config.graph.selected_implementer
1177 && state.candidates.iter().any(|c| c.agent != *id)
1178 {
1179 bail!(
1180 "replacement implementer pin does not match the saved candidates; task must stay held"
1181 );
1182 }
1183 let max_parallel = state.config.graph.max_parallel.max(1);
1184 Ok(Self {
1185 state,
1186 roles,
1187 sem: Arc::new(Semaphore::new(max_parallel)),
1188 pause: Pause::new(),
1189 interrupt: Pause::new(),
1190 })
1191 }
1192
1193 pub async fn execute(&mut self) -> Result<()> {
1200 if let Some(selection) = &self.state.pr_recovery
1201 && (selection.source_proof.is_some() || selection.source_run.is_some())
1202 {
1203 Box::pin(selection.verify_source(&self.state.repo)).await?;
1204 }
1205 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1208 || self.state.land_approval.is_some()
1209 || self.state.merge_decision.is_some()
1210 || !self.state.merge_repairs.is_empty()
1211 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1212 {
1213 let claim = FixClaim::acquire(&self.state.dir())?;
1214 let disk = RunState::load(&self.state.id)?;
1215 let checkpoint = |s: &RunState| {
1216 (
1217 s.merge_repairs.len(),
1218 s.merge_repairs
1219 .last()
1220 .map(|r| (r.question.clone(), r.phase)),
1221 s.land_approval
1222 .as_ref()
1223 .map(|a| (a.question.clone(), a.head.clone())),
1224 )
1225 };
1226 if checkpoint(&disk) != checkpoint(&self.state) {
1227 bail!(
1228 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1229 );
1230 }
1231 Some(claim)
1232 } else {
1233 None
1234 };
1235 let mut execution = self.claim_driver()?;
1236 let runner = &mut *execution.runner;
1237 let wait = runner.repository_wait();
1238 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1241 let result = match result {
1242 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1243 record_repository_block(&mut runner.state, &e);
1244 runner.park_here()?;
1245 Ok(())
1246 }
1247 other => other,
1248 };
1249 execution.driver.finish(&mut runner.state)?;
1250 let ended = if result.is_err() {
1251 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1252 } else {
1253 crate::notices::run_ended(&runner.state)
1254 };
1255 if let Some(notice) = ended {
1256 crate::notices::raise(notice);
1257 }
1258 result
1259 }
1260
1261 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1262 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1263 }
1264
1265 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1266 let wait = self.repository_wait();
1267 coordinate_operation(&mut self.state, cwd, &wait).await
1268 }
1269
1270 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1272 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1273 Ok(owner) => owner,
1274 Err(error) => {
1275 let detail = format!("worktree maintenance or ownership update: {error:#}");
1276 self.state.resource_block = Some(crate::run::ResourceBlock {
1280 node: "start".into(),
1281 kind: "worktree maintenance".into(),
1282 detail: detail.clone(),
1283 at: Timestamp::now(),
1284 });
1285 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1286 }
1287 };
1288 let disk = RunState::load(&self.state.id)?;
1289 crate::pr_head::ensure_owner(&disk)?;
1290 if disk.liveness(false) == crate::run::Liveness::Live {
1291 bail!("run already has a live driver; wait for its execution to end");
1292 }
1293 if let Some(to) = &disk.released_to {
1294 bail!(
1295 "run {} cannot continue: its worktree was released to run {}",
1296 self.state.short(),
1297 crate::run::short_of(to)
1298 );
1299 }
1300 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1301 let mut execution = DriverExecution {
1302 runner: self,
1303 driver,
1304 };
1305 let _owner = owner;
1308 let state = &mut execution.runner.state;
1309 state.parked = false;
1314 state.resource_wait = None;
1315 state.resource_block = None;
1316 state.clear_active();
1323 let pid = std::process::id();
1338 state.driver_pid = Some(pid);
1339 state.driver_started_at = crate::proc::process_started_at(pid);
1340 state.driver_exited = false;
1341 state.begin_pass();
1342 execution.driver.bind(state);
1343 state.save()?;
1344 execution.driver.record(state)?;
1345 Ok(execution)
1346 }
1347
1348 async fn execute_graph(&mut self) -> Result<()> {
1349 self.state.configure_recovery_resume()?;
1350 self.persist_candidates().await?;
1353 if self
1354 .state
1355 .merge_decision
1356 .as_ref()
1357 .is_some_and(|decision| !decision.closed)
1358 && self.state.status != RunStatus::Landing
1359 {
1360 if crate::merge_decision::resumable_hold(&self.state) {
1361 self.state.merge = Some(MergeOutcome {
1362 mode: MergeMode::Pr,
1363 ok: false,
1364 detail: self
1365 .state
1366 .merge_decision
1367 .as_ref()
1368 .unwrap()
1369 .target
1370 .pr
1371 .clone(),
1372 empty: false,
1373 });
1374 self.run_land().await?;
1375 self.settle_questions();
1376 return Ok(());
1377 }
1378 bail!(
1379 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1380 );
1381 }
1382 if self.state.config.graph.selected_implementer.is_some()
1383 && self.state.status == RunStatus::Blocked
1384 && self.state.tally.is_none()
1385 && self.state.candidate_preservation.is_empty()
1386 && self.state.interrupted_implementations.is_empty()
1387 {
1388 bail!(
1389 "selected implementation is held; explicitly release the owning task to retry it"
1390 );
1391 }
1392
1393 if self.state.merge_repairs.last().is_some_and(|r| {
1394 matches!(
1395 r.phase,
1396 crate::run::MergeRepairPhase::Requested
1397 | crate::run::MergeRepairPhase::Fixing
1398 | crate::run::MergeRepairPhase::Reviewing
1399 )
1400 }) {
1401 if self.repair_merge_review().await? {
1402 self.run_land().await?;
1403 }
1404 self.settle_questions();
1405 return Ok(());
1406 }
1407 if self.state.pr_publication_pending {
1410 self.merge().await?;
1411 if self.resource_halt()? {
1412 return Ok(());
1413 }
1414 self.state.save()?;
1415 self.settle_questions();
1416 return Ok(());
1417 }
1418 if self.state.status == RunStatus::Stalled {
1431 if self.recover_stall().await? {
1432 self.finish_after_tally().await?;
1433 } else {
1434 self.state.save()?;
1436 }
1437 return Ok(());
1438 }
1439 if self.state.status == RunStatus::Landing {
1449 self.run_land().await?;
1450 self.settle_questions();
1455 return Ok(());
1456 }
1457 if self.state.github_text.is_some()
1462 && self.state.merge.is_none()
1463 && self.state.status == RunStatus::Gating
1464 {
1465 self.merge().await?;
1466 if self.resource_halt()? {
1467 return Ok(());
1468 }
1469 self.state.save()?;
1470 return Ok(());
1471 }
1472 if let Err(e) = self.prep().await {
1473 if e.is::<crate::repo_lock::ResourceBlocked>() {
1474 return Err(e);
1475 }
1476 self.state.status = RunStatus::Blocked;
1477 self.state
1478 .prep_failure
1479 .get_or_insert_with(|| format!("{e:#}"));
1480 self.state
1481 .event("prep", format!("preparation stopped: {e:#}"));
1482 self.state.save()?;
1483 return Err(e);
1484 }
1485 if self.park_here()? {
1486 return Ok(());
1487 }
1488 self.advise().await?;
1489 if self.park_here()? {
1490 return Ok(());
1491 }
1492 self.implement().await?;
1493 if self.park_here()? {
1494 return Ok(());
1495 }
1496 if self.state.status == RunStatus::VerifiedNoop {
1500 return Ok(());
1501 }
1502 self.judge().await?;
1503 if self.park_here()? {
1504 return Ok(());
1505 }
1506 self.deliberate().await?;
1507 if self.park_here()? {
1508 return Ok(());
1509 }
1510 self.vote().await?;
1511 if self.park_here()? {
1512 return Ok(());
1513 }
1514 self.tally()?;
1515 if self.state.status == RunStatus::Stalled {
1520 self.state.save()?;
1524 return Ok(());
1525 }
1526 self.finish_after_tally().await?;
1527 Ok(())
1528 }
1529
1530 fn stop_for_resource(&mut self) -> Result<()> {
1535 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1537 {
1538 self.state.status = RunStatus::Blocked;
1539 }
1540 self.state.save()
1541 }
1542
1543 fn resource_halt(&mut self) -> Result<bool> {
1545 if self.state.resource_block.is_none() {
1546 return Ok(false);
1547 }
1548 self.stop_for_resource()?;
1549 Ok(true)
1550 }
1551
1552 fn park_here(&mut self) -> Result<bool> {
1559 let parking = self.pause.parked() || self.interrupt.parked();
1560 if !parking && self.state.resource_block.is_some() {
1561 self.stop_for_resource()?;
1562 return Ok(true);
1563 }
1564 if !self.pause.parked() && !self.interrupt.parked() {
1569 return Ok(false);
1570 }
1571 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1572 let why = match cause.as_ref().map(ParkCause::label) {
1573 Some(reason) => format!(
1574 "parked after `{}` ({reason}) — resume to carry on from here",
1575 self.state.status.as_str()
1576 ),
1577 None => format!(
1578 "parked after `{}` — resume to carry on from here",
1579 self.state.status.as_str()
1580 ),
1581 };
1582 self.state.event("park", why);
1583 self.state
1584 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1585 self.state.parked = true;
1586 self.state.save()?;
1587 Ok(true)
1588 }
1589
1590 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1592 if self.state.prep_failure.is_none() {
1593 return Ok(());
1594 }
1595 let failed_at = self
1596 .state
1597 .events
1598 .iter()
1599 .rev()
1600 .find(|event| event.node == "prep")
1601 .map_or(self.state.updated_at, |event| event.at);
1602 anyhow::ensure!(
1603 approval_at > failed_at,
1604 "preparation retry requires a new explicit release after the failure"
1605 );
1606 self.state.prep_failure = None;
1607 self.state.prep_complete = Some(false);
1608 self.state
1609 .event("prep-retry", "operator approved retry of preparation");
1610 Ok(())
1611 }
1612
1613 pub fn on_pause(&mut self, pause: Pause) {
1615 self.pause = pause;
1616 }
1617
1618 pub fn watch_interrupt(&mut self, pause: Pause) {
1624 self.interrupt = pause;
1625 }
1626
1627 fn settle_questions(&mut self) {
1647 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1648 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1649 }
1650 }
1651
1652 async fn finish_after_tally(&mut self) -> Result<()> {
1655 self.graft().await?;
1658 if self.resource_halt()? {
1659 return Ok(());
1660 }
1661 self.fold_losers().await?;
1662 self.sync_to_base().await?;
1667 if self.state.status == RunStatus::AlreadyInBase {
1668 return Ok(());
1669 }
1670 self.review_loop().await?;
1671 if self.resource_halt()? {
1672 return Ok(());
1673 }
1674 self.sync_to_base().await?;
1675 if self.state.status == RunStatus::AlreadyInBase {
1676 return Ok(());
1677 }
1678 self.gate().await?;
1679 self.merge().await?;
1680 if self.resource_halt()? {
1681 return Ok(());
1682 }
1683 self.state.save()?;
1684 Ok(())
1685 }
1686
1687 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1694 let result = match ensure_setup_config(&mut self.state, repo).await {
1695 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1696 Err(e) => Err(e),
1697 };
1698 if let Err(e) = result {
1699 git::worktree_remove_clean(repo, wt).await.ok();
1700 if in_prep {
1701 self.rollback_prep(repo).await;
1702 }
1703 self.state.status = RunStatus::Blocked;
1704 self.state.event(
1705 "setup",
1706 format!("worktree setup failed in {}: {e:#}", wt.display()),
1707 );
1708 self.state.save()?;
1709 return Err(e);
1710 }
1711 Ok(())
1712 }
1713
1714 async fn rollback_prep(&mut self, repo: &Path) {
1716 let root = self.state.worktree_root();
1717 for c in std::mem::take(&mut self.state.candidates) {
1718 if matches!(
1719 git::worktree_remove_clean(repo, &c.worktree).await,
1720 Ok(true)
1721 ) {
1722 git::branch_delete(repo, &c.branch).await.ok();
1723 } else {
1724 self.state.candidates.push(c);
1725 }
1726 }
1727 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1728 for j in 1..=seats {
1729 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1730 .await
1731 .ok();
1732 }
1733 for k in 1..=self.state.config.graph.advisors {
1734 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1735 .await
1736 .ok();
1737 }
1738 }
1739
1740 async fn prep(&mut self) -> Result<()> {
1741 if let Some(reason) = &self.state.prep_failure {
1742 bail!(
1743 "candidate preparation previously failed: {reason}; correct the source before retrying"
1744 );
1745 }
1746 if self
1747 .state
1748 .prep_complete
1749 .unwrap_or(!self.state.candidates.is_empty())
1750 {
1751 return Ok(());
1752 }
1753 let repo = self.state.repo.clone();
1754 let _coordination = self.coordinate(&repo).await?;
1757 self.rollback_prep(&repo).await;
1758 if !self.state.candidates.is_empty() {
1759 bail!(
1760 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1761 );
1762 }
1763 self.state.prep_complete = Some(false);
1764 self.state.status = RunStatus::Prep;
1765 self.state.save()?;
1766 let repo = self.state.repo.clone();
1767 let base = self.state.base_commit.clone();
1768 let plan = refs::plan(&repo, &self.state.seeds).await?;
1769 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1770 let root = self.state.worktree_root();
1771 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1772
1773 let hooks_dir = self.state.dir().join("hooks");
1776 if self.state.config.blind.commit_msg_hook {
1777 std::fs::create_dir_all(&hooks_dir)
1778 .with_context(|| format!("create {}", hooks_dir.display()))?;
1779 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1780 let path = hooks_dir.join("commit-msg");
1781 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1782 make_executable(&path)?;
1783 git::acquire_worktree_config(&repo).await?;
1791 self.state.enabled_worktree_config = true;
1792 }
1793
1794 ensure_setup_config(&mut self.state, &repo).await?;
1797
1798 for (index, (spec, label)) in self
1799 .roles
1800 .implementers
1801 .clone()
1802 .into_iter()
1803 .zip(labels)
1804 .enumerate()
1805 {
1806 let branch = self.state.branch_for(label);
1807 let worktree = root.join(format!("cand-{label}"));
1808 self.state.event(
1812 "prep-intent",
1813 serde_json::json!({
1814 "branch": branch,
1815 "worktree": worktree,
1816 "start": start,
1817 "picks": plan.picks,
1818 })
1819 .to_string(),
1820 );
1821 self.state.save()?;
1822 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1823 if self.state.config.blind.commit_msg_hook {
1824 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1825 }
1826 git::local_exclude(&worktree, "/.magi/").await?;
1827 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1828 git::worktree_remove_clean(&repo, &worktree).await.ok();
1831 git::branch_delete(&repo, &branch).await.ok();
1832 self.rollback_prep(&repo).await;
1833 self.state.status = RunStatus::Blocked;
1834 self.state
1835 .event("prep", format!("worktree setup failed: {e:#}"));
1836 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1837 self.state.save()?;
1838 return Err(e);
1839 }
1840 self.state.candidates.push(Candidate {
1841 index,
1842 label,
1843 agent: spec.id.clone(),
1844 branch,
1845 worktree,
1846 summary: String::new(),
1847 stat: String::new(),
1848 files: 0,
1849 commits: 0,
1850 empty: false,
1851 failed: None,
1852 verified_noop: None,
1853 duration_ms: 0,
1854 folded: false,
1855 });
1856 self.state.save()?;
1857 for pick in &plan.picks {
1858 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1859 {
1860 self.state.status = RunStatus::Blocked;
1861 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1862 self.state
1863 .event("prep", format!("cannot apply referenced commit: {e}"));
1864 self.state.save()?;
1865 return Err(e);
1866 }
1867 }
1868 }
1869
1870 for j in 1..=self.roles.judges.len() {
1871 let wt = root.join(format!("judge-{j}"));
1872 if !wt.exists() {
1873 git::worktree_add_detached(&repo, &wt, &base).await?;
1874 self.setup_seat_worktree(&repo, &wt, true).await?;
1875 }
1876 }
1877
1878 if self.state.config.graph.advise {
1886 for k in 1..=self.state.config.graph.advisors {
1887 let wt = root.join(format!("advisor-{k}"));
1888 if !wt.exists() {
1889 git::worktree_add_detached(&repo, &wt, &base).await?;
1890 self.setup_seat_worktree(&repo, &wt, true).await?;
1891 }
1892 }
1893 }
1894
1895 let authors: Vec<&str> = self
1900 .roles
1901 .implementers
1902 .iter()
1903 .map(|a| a.id.as_str())
1904 .collect();
1905 let overlap: Vec<String> = self
1906 .roles
1907 .judges
1908 .iter()
1909 .enumerate()
1910 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1911 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1912 .collect();
1913 if !overlap.is_empty() {
1914 let note = format!(
1915 "{} also authored a candidate; blind, but the panel is less \
1916 independent than {} distinct agents would be",
1917 overlap.join(", "),
1918 self.roles.judges.len()
1919 );
1920 self.state.event("prep", note);
1921 }
1922
1923 self.state.event(
1924 "prep",
1925 format!(
1926 "{} candidates, {} judges, base {} ({})",
1927 self.state.candidates.len(),
1928 self.roles.judges.len(),
1929 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1930 self.state.base_branch
1931 ),
1932 );
1933 self.state.prep_complete = Some(true);
1934 self.state.status = RunStatus::Implementing;
1935 self.state.save()?;
1936 Ok(())
1937 }
1938
1939 async fn advise(&mut self) -> Result<()> {
1972 let implement_untouched = self
1973 .state
1974 .candidates
1975 .iter()
1976 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1977 if !self.state.config.graph.advise || self.state.advise_attempted {
1978 return Ok(());
1979 }
1980 if !implement_untouched {
1981 self.state.event(
1982 "advise",
1983 "skipping the design-deliberation stage: at least one \
1984 candidate already shows implementation progress, so this \
1985 run is past the point the stage exists to run before"
1986 .to_owned(),
1987 );
1988 self.state.advise_attempted = true;
1989 self.state.save()?;
1990 return Ok(());
1991 }
1992 let run_id = self.state.id.clone();
1993 let prompts = self.state.config.prompts.clone();
1994 let instruction = self.state.instruction.clone();
1995 let language = self.state.config.graph.language.clone();
1996 let root = self.state.worktree_root();
1997 let n = self.state.config.graph.advisors;
1998 let where_recorded = self.state.dir().join("run.json");
1999
2000 let seats = match self.state.config.advisors() {
2001 Ok(seats) if !seats.is_empty() => seats,
2002 Ok(_) => {
2003 self.state.event(
2004 "advise",
2005 format!(
2006 "[graph] advisors is 0; skipping the design-deliberation \
2007 stage and continuing without a synthesis brief (see {})",
2008 where_recorded.display()
2009 ),
2010 );
2011 self.state.advise_attempted = true;
2012 self.state.save()?;
2013 return Ok(());
2014 }
2015 Err(e) => {
2016 self.state.event(
2017 "advise",
2018 format!(
2019 "could not resolve advisor seats ({e:#}); continuing \
2020 without a design-deliberation brief (see {})",
2021 where_recorded.display()
2022 ),
2023 );
2024 self.state.advise_attempted = true;
2025 self.state.save()?;
2026 return Ok(());
2027 }
2028 };
2029
2030 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2031 let artifacts = agent::artifacts_dir(&self.state.dir());
2032 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2033
2034 let mut jobs = Vec::new();
2035 for (i, spec) in seats.iter().cloned().enumerate() {
2036 let seat_key = format!("advisor-{}", i + 1);
2037 let seat = self.seat(&seat_key, &spec.id);
2038 jobs.push(SeatJob {
2039 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2040 spec,
2041 seat,
2042 cwd: worktrees[i % worktrees.len()].clone(),
2043 timeout,
2044 allow_write: false,
2045 sessions: false,
2046 artifacts: artifacts.clone(),
2047 stem: seat_key,
2048 handover: None,
2049 });
2050 }
2051
2052 self.state.event(
2053 "advise",
2054 format!(
2055 "{} advisor seat(s) sketching a design in parallel",
2056 jobs.len()
2057 ),
2058 );
2059 let mut quota_losses = Vec::new();
2060 let cache = self.state.config.cache_dir();
2061 let ctx = WaveCtx {
2062 carry_seats: false,
2063 pauses: Some((&self.pause, &self.interrupt)),
2064 run: &run_id,
2065 node: "advise",
2066 prompts: &prompts,
2067 cache: cache.as_deref(),
2068 round: None,
2069 };
2070 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2071 let results = ask_json_wave::<Proposal>(
2072 jobs,
2073 Arc::clone(&self.sem),
2074 self.state.config.graph.retries,
2075 &advisor_roster,
2076 &ctx,
2077 &mut quota_losses,
2078 &mut self.state,
2079 &|p: &Proposal| p.validate(),
2080 )
2081 .await;
2082 if self.state.resource_block.is_some() {
2083 return self.stop_for_resource();
2084 }
2085 self.state.quota.extend(quota_losses);
2086
2087 let mut records = Vec::with_capacity(results.len());
2088 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2089 let agent_id = seat.agent.clone();
2090 self.state.seats.insert(seat.key.clone(), seat);
2091 match res {
2092 Ok((proposal, out)) => {
2093 self.state
2094 .event("advise", format!("advisor-{} proposed a design", i + 1));
2095 records.push(advise::AdvisorRecord::proposed(
2096 i + 1,
2097 agent_id,
2098 proposal,
2099 out.duration_ms,
2100 ));
2101 }
2102 Err(e) => {
2103 self.state.event(
2104 "advise",
2105 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2106 );
2107 records.push(advise::AdvisorRecord::failed(
2108 i + 1,
2109 agent_id,
2110 e.to_string(),
2111 ));
2112 }
2113 }
2114 }
2115
2116 let mut advice = advise::Advice {
2117 records,
2118 synthesis: None,
2119 };
2120 if advice.proposals().is_empty() {
2121 self.state.event(
2122 "advise",
2123 "no advisor produced a usable proposal; continuing without a \
2124 synthesis brief"
2125 .to_owned(),
2126 );
2127 } else {
2128 match self
2129 .synthesize_brief(
2130 &advice,
2131 &instruction,
2132 &language,
2133 &worktrees[0],
2134 &artifacts,
2135 &run_id,
2136 &prompts,
2137 cache.as_deref(),
2138 )
2139 .await
2140 {
2141 Ok(Some(text)) => {
2142 self.state.event(
2143 "advise",
2144 "synthesized a design brief for the implementer".to_owned(),
2145 );
2146 advice.synthesis = Some(text);
2147 }
2148 Ok(None) => {
2149 self.state.event(
2150 "advise",
2151 "the synthesis seat produced nothing usable; continuing \
2152 without a design brief"
2153 .to_owned(),
2154 );
2155 }
2156 Err(e) => {
2157 if e.is::<crate::repo_lock::ResourceBlocked>() {
2158 return Err(e);
2159 }
2160 self.state.event(
2161 "advise",
2162 format!("could not synthesize a design brief: {e:#}"),
2163 );
2164 }
2165 }
2166 }
2167 advise::apply_reflection(&mut advice);
2168
2169 self.state.advice = Some(advice);
2170 self.state.advise_attempted = true;
2171 self.state.save()?;
2172 Ok(())
2173 }
2174
2175 #[allow(clippy::too_many_arguments)]
2187 async fn synthesize_brief(
2188 &mut self,
2189 advice: &advise::Advice,
2190 instruction: &str,
2191 language: &str,
2192 cwd: &Path,
2193 artifacts: &Path,
2194 run_id: &str,
2195 prompts: &Prompts,
2196 cache: Option<&Path>,
2197 ) -> Result<Option<String>> {
2198 let chain = agent::pick_chain(
2199 &self.state.config.agents,
2200 self.state.config.roles.synthesizer.as_ref(),
2201 &agent::installed,
2202 "synthesizer",
2203 )?;
2204 let proposals = advice.proposals();
2205 let mut prompt = prompt::with_overlay(
2206 prompt::synthesize_brief(instruction, &proposals, language),
2207 prompts.overlay("advise"),
2208 );
2209 if cache.is_some() {
2210 prompt.push('\n');
2215 prompt.push_str(&prompt::build_cache_note("advise", false));
2216 }
2217 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2218 let mut last = None;
2223 for (n, spec) in chain.iter().enumerate() {
2224 if n > 0 {
2225 self.state
2226 .event("advise", format!("synthesis falling back to {}", spec.id));
2227 }
2228 let mut seat = self.seat("advise-synthesis", &spec.id);
2229 let wait = self.repository_wait();
2230 let outcome = wait
2231 .scope(agent::invoke(
2232 spec,
2233 &mut seat,
2234 &Invocation {
2235 scope: crate::agent::InvocationScope::Repository,
2236 coordination: self.state.config.coordination.as_ref(),
2237 fetched_base: None,
2238 cwd,
2239 prompt: &prompt,
2240 timeout,
2241 allow_write: false,
2242 unsandboxed: false,
2243 sessions: false,
2244 artifacts,
2245 stem: &if n == 0 {
2246 "advise-synthesis".to_owned()
2247 } else {
2248 format!("advise-synthesis-{}", spec.id)
2249 },
2250 run: run_id,
2251 node: "advise",
2252 cache_dir: None,
2253 attachments: &[],
2254 writable: &[],
2255 },
2256 ))
2257 .await;
2258 if outcome.is_ok() {
2259 self.state.seats.insert(seat.key.clone(), seat);
2260 }
2261 let advance = agent::chain_advances(&outcome);
2262 last = Some(outcome);
2263 if !advance {
2264 break;
2265 }
2266 }
2267 let out = last.expect("a chain holds at least one agent")?;
2270 if !out.usable() {
2271 return Ok(None);
2272 }
2273 let text =
2274 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2275 Ok((!text.trim().is_empty()).then_some(text))
2276 }
2277
2278 async fn implement(&mut self) -> Result<()> {
2281 self.persist_candidates().await?;
2282 let run_id = self.state.id.clone();
2287 let prompts = self.state.config.prompts.clone();
2288 let todo: Vec<usize> = self
2289 .state
2290 .candidates
2291 .iter()
2292 .enumerate()
2293 .filter(|(_, c)| {
2294 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2295 && c.failed.is_none()
2296 && !c.empty
2297 })
2298 .map(|(i, _)| i)
2299 .collect();
2300 if todo.is_empty() {
2301 return self.after_implement();
2302 }
2303 self.state.status = RunStatus::Implementing;
2304
2305 let language = self.state.config.graph.language.clone();
2306 let github_language = self.state.config.graph.github_language.clone();
2307 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2308 let sessions = self.state.config.graph.sessions;
2309 let artifacts = agent::artifacts_dir(&self.state.dir());
2310 let brief = self
2314 .state
2315 .advice
2316 .as_ref()
2317 .and_then(|a| a.synthesis.as_deref())
2318 .map(str::to_owned);
2319 let attachments = self.state.attachments.clone();
2320
2321 let mut jobs = Vec::new();
2322 for &i in &todo {
2323 let (index, label, worktree) = {
2324 let c = &self.state.candidates[i];
2325 (c.index, c.label, c.worktree.clone())
2326 };
2327 let spec = self.roles.implementers[index].clone();
2328 let seat_key = format!("impl-{label}");
2329 let seat = self.seat(&seat_key, &spec.id);
2330 let instruction = seeded_instruction(&self.state);
2331 jobs.push(SeatJob {
2332 spec,
2333 seat,
2334 prompt: prompt::implement_in(
2335 &instruction,
2336 &worktree.to_string_lossy(),
2337 &language,
2338 &github_language,
2339 brief.as_deref(),
2340 &attachments,
2341 ),
2342 cwd: worktree,
2343 timeout,
2344 allow_write: true,
2345 sessions,
2346 artifacts: artifacts.clone(),
2347 stem: format!("impl-{label}"),
2348 handover: None,
2349 });
2350 }
2351
2352 self.state.event(
2353 "implement",
2354 format!("{} candidates in parallel", jobs.len()),
2355 );
2356 let mut sent = jobs.clone();
2362 let cache = self.state.config.cache_dir();
2363 let ctx = WaveCtx {
2364 carry_seats: false,
2365 pauses: Some((&self.pause, &self.interrupt)),
2366 run: &run_id,
2367 node: "implement",
2368 prompts: &prompts,
2369 cache: cache.as_deref(),
2370 round: None,
2371 };
2372 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2373 if !wave_blocked(&results) {
2374 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2375 .await;
2376 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2377 .await?;
2378 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2379 .await;
2380 }
2381 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2382
2383 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2384 && results
2385 .iter()
2386 .any(|(_, _, out)| FailClass::of(out).is_some());
2387 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2388 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2389 if self
2393 .state
2394 .candidate_preservation
2395 .contains_key(&self.state.candidates[i].label)
2396 {
2397 self.state.seats.insert(seat.key.clone(), seat);
2398 }
2399 continue;
2400 }
2401 let seat_key = seat.key.clone();
2402 let agent = seat.agent.clone();
2411 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2412 self.state.seats.insert(seat.key.clone(), seat);
2413 let label = self.state.candidates[i].label;
2414 self.state.interrupted_implementations.remove(&label);
2415
2416 let (summary, duration, failed, verified_claim) = match out {
2417 AgentOutcome::Ok(o) => {
2418 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2419 let failed = (!o.usable()).then(|| {
2420 if o.timed_out {
2421 "agent timed out".to_owned()
2422 } else {
2423 format!(
2424 "agent exited with {}",
2425 crate::notices::exit_status(o.exit_code)
2426 )
2427 }
2428 });
2429 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2430 (text, o.duration_ms, failed, verified_claim)
2431 }
2432 AgentOutcome::Dropped(o) => {
2438 let why = o
2439 .dropped
2440 .as_ref()
2441 .map(|d| d.why.as_str())
2442 .unwrap_or("the CLI ended the stream without delivering its answer");
2443 (
2444 String::new(),
2445 o.duration_ms,
2446 Some(format!("the CLI dropped the stream ({why})")),
2447 None,
2448 )
2449 }
2450 AgentOutcome::Quota(o) => {
2451 self.state.quota.push(QuotaLoss {
2452 seat: seat_key,
2453 node: "implement".to_owned(),
2454 at: Timestamp::now(),
2455 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2456 });
2457 (
2458 String::new(),
2459 o.duration_ms,
2460 Some("rate limited (quota); produced no change".to_owned()),
2461 None,
2462 )
2463 }
2464 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2465 (String::new(), 0, Some(e), None)
2466 }
2467 };
2468
2469 let c = &mut self.state.candidates[i];
2472 if !exhausted_the_fallback_chain {
2473 c.agent = agent;
2474 }
2475 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2476 c.duration_ms = duration;
2477 c.failed = failed;
2478 c.verified_noop = verified_claim;
2479 self.state
2480 .candidate_preservation
2481 .insert(label, "awaiting persistence".to_owned());
2482 }
2483 self.state.save()?;
2484 self.persist_candidates().await?;
2485
2486 if resource_blocked {
2487 return self.stop_for_resource();
2488 }
2489 if selected_failed {
2490 self.state.status = RunStatus::Blocked;
2491 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2492 self.state.save()?;
2493 bail!(
2494 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2495 );
2496 }
2497 self.after_implement()
2498 }
2499
2500 async fn persist_candidates(&mut self) -> Result<()> {
2502 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2503 for label in labels {
2504 let i = self
2505 .state
2506 .candidates
2507 .iter()
2508 .position(|c| c.label == label)
2509 .context("pending candidate is missing")?;
2510 let c = self.state.candidates[i].clone();
2511 let interrupted = self.state.interrupted_implementations.contains(&label);
2512 let result = async {
2513 let wait = crate::repo_lock::WaitPolicy::new(
2514 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2515 || false,
2516 );
2517 let _coordination =
2518 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2519 if let Err(e) = git::snapshot_uncommitted(
2522 &c.worktree,
2523 &self
2524 .state
2525 .dir()
2526 .join("artifacts")
2527 .join(format!("candidate-{label}-recovery")),
2528 )
2529 .await
2530 {
2531 self.state.event(
2532 "preserve",
2533 format!("candidate {label} snapshot incomplete: {e:#}"),
2534 );
2535 }
2536 let rescued = git::rescue_commit(
2537 &c.worktree,
2538 &format!("magi: candidate {label} (uncommitted work)"),
2539 )
2540 .await?;
2541 self.state.note_withheld("implement", &rescued.withheld);
2542 let base = &self.state.base_commit;
2543 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2544 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2545 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2546 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2547 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2548 if c.empty && commits > 0 && !patch.trim().is_empty()
2549 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2550 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2551 || self.state.tally.is_some())
2552 {
2553 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2557 }
2558 let c = &mut self.state.candidates[i];
2559 c.commits = commits;
2560 c.stat = stat;
2561 c.files = files;
2562 let no_changes = commits == 0 || patch.trim().is_empty();
2563 c.empty = no_changes && !interrupted;
2564 if interrupted && no_changes {
2565 c.failed = None;
2566 c.verified_noop = None;
2567 }
2568 if !c.empty {
2569 c.failed = None;
2570 c.verified_noop = None;
2571 }
2572 self.state.event(
2573 "preserve",
2574 format!("candidate {label}: {files} files, {commits} commits"),
2575 );
2576 Ok::<_, anyhow::Error>(())
2577 }
2578 .await;
2579 match result {
2580 Ok(()) => {
2581 self.state.candidate_preservation.remove(&label);
2582 if !interrupted || self.state.candidates[i].files > 0 {
2583 self.state.interrupted_implementations.remove(&label);
2584 }
2585 }
2586 Err(e) => {
2587 let reason = format!("candidate {label} preservation failed: {e:#}");
2588 self.state.event("preserve", &reason);
2589 self.state.candidate_preservation.insert(label, reason);
2590 }
2591 }
2592 if self.state.candidate_preservation.is_empty()
2593 && self.state.status == RunStatus::Blocked
2594 && self.state.candidates.iter().all(|c| c.failed.is_none())
2595 {
2596 self.state.status = RunStatus::Implementing;
2600 self.state.event(
2601 "preserve",
2602 "candidate persistence recovered; implementation can continue",
2603 );
2604 }
2605 self.state.save()?;
2606 }
2607 if !self.state.candidate_preservation.is_empty() {
2608 self.state.status = RunStatus::Blocked;
2609 self.state.save()?;
2610 bail!(
2611 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2612 );
2613 }
2614 Ok(())
2615 }
2616
2617 async fn resume_undelivered(
2645 &mut self,
2646 results: &mut [(usize, SeatState, AgentOutcome)],
2647 sent: &[SeatJob],
2648 prompts: &Prompts,
2649 run_id: &str,
2650 ) {
2651 for (wi, seat, out) in results.iter_mut() {
2652 let Some(dropped) = (match &*out {
2653 AgentOutcome::Dropped(o) => o.dropped.clone(),
2654 _ => None,
2655 }) else {
2656 continue;
2657 };
2658 let Some(job) = sent.get(*wi) else { continue };
2659 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2661 self.state.event(
2662 "implement",
2663 format!(
2664 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2665 work is in the tree",
2666 seat.key, dropped.output_tokens, dropped.why
2667 ),
2668 );
2669 continue;
2670 }
2671 if !has_context(&job.spec, seat, job.sessions) {
2679 self.state.event(
2680 "implement",
2681 format!(
2682 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2683 is no session left to resume",
2684 seat.key, dropped.output_tokens, dropped.why
2685 ),
2686 );
2687 continue;
2688 }
2689 self.state.event(
2690 "implement",
2691 format!(
2692 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2693 conversation",
2694 seat.key, dropped.output_tokens, dropped.why
2695 ),
2696 );
2697 let mut retry = job.clone();
2698 retry.seat = seat.clone();
2699 retry.prompt = prompt::resume_after_drop(&dropped.why);
2700 retry.timeout = retry_budget(job.timeout, true);
2701 retry.stem = format!("{}-resume", job.stem);
2702 let cache = self.state.config.cache_dir();
2703 let ctx = WaveCtx {
2704 carry_seats: false,
2705 pauses: Some((&self.pause, &self.interrupt)),
2706 run: run_id,
2707 node: "implement",
2708 prompts,
2709 cache: cache.as_deref(),
2710 round: None,
2711 };
2712 let (resumed_seat, resumed) =
2713 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2714 *seat = resumed_seat;
2715 *out = resumed;
2716 }
2717 }
2718
2719 async fn resume_seat_handovers(
2784 &mut self,
2785 results: &mut [(usize, SeatState, AgentOutcome)],
2786 sent: &mut [SeatJob],
2787 prompts: &Prompts,
2788 run_id: &str,
2789 ) -> Result<()> {
2790 let instruction = seeded_instruction(&self.state);
2791 let language = self.state.config.graph.language.clone();
2792 let github_language = self.state.config.graph.github_language.clone();
2793 let brief = self
2794 .state
2795 .advice
2796 .as_ref()
2797 .and_then(|a| a.synthesis.as_deref())
2798 .map(str::to_owned);
2799 let attachments = self.state.attachments.clone();
2800 for (wi, seat, out) in results.iter_mut() {
2801 let others: BTreeSet<String> = sent
2804 .iter()
2805 .enumerate()
2806 .filter(|(j, _)| j != wi)
2807 .map(|(_, j)| j.spec.id.clone())
2808 .collect();
2809 let Some(job) = sent.get_mut(*wi) else {
2810 continue;
2811 };
2812 let start = self
2817 .roles
2818 .implementer_roster
2819 .iter()
2820 .position(|s| s.id == job.spec.id)
2821 .unwrap_or(0);
2822 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2823 let mut fallback_attempt = 0usize;
2824 let mut prev: Option<FailClass> = None;
2825 while let Some(cur) = FailClass::of(&*out) {
2826 if !should_hand_over(prev.as_ref(), &cur) {
2827 break;
2828 }
2829 let Some(next) =
2830 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2831 .cloned()
2832 else {
2833 break;
2834 };
2835 tried.insert(next.id.clone());
2836 fallback_attempt += 1;
2837
2838 let coordination = self.coordinate(&job.cwd).await?;
2839 if let Ok(r) = git::rescue_commit(
2840 &job.cwd,
2841 &format!(
2842 "magi: candidate {} (uncommitted work before {} fallback)",
2843 seat.key,
2844 if cur == FailClass::Quota {
2845 "quota"
2846 } else {
2847 "handover"
2848 }
2849 ),
2850 )
2851 .await
2852 {
2853 self.state.note_withheld("implement", &r.withheld);
2854 }
2855 drop(coordination);
2856
2857 record_handover(
2858 &mut self.state,
2859 "implement",
2860 &seat.key,
2861 &seat.agent,
2862 &next.id,
2863 &cur,
2864 &fail_reason(&*out),
2865 );
2866 prev = Some(cur.clone());
2867
2868 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2869 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2870 job.spec = next.clone();
2878 let mut retry = job.clone();
2879 retry.seat = new_seat;
2880 retry.prompt = prompt::implement_in(
2881 &instruction,
2882 &job.cwd.to_string_lossy(),
2883 &language,
2884 &github_language,
2885 brief.as_deref(),
2886 &attachments,
2887 );
2888 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2889 let cache = self.state.config.cache_dir();
2890 let ctx = WaveCtx {
2891 carry_seats: false,
2892 pauses: Some((&self.pause, &self.interrupt)),
2893 run: run_id,
2894 node: "implement",
2895 prompts,
2896 cache: cache.as_deref(),
2897 round: None,
2898 };
2899 let (fallback_seat, fallback_out) = run_one(
2900 retry,
2901 Arc::clone(&self.sem),
2902 &ctx,
2903 &mut self.state,
2904 fallback_attempt,
2905 )
2906 .await;
2907 *seat = fallback_seat;
2908 *out = fallback_out;
2909 }
2910 }
2911 Ok(())
2912 }
2913
2914 async fn resume_unconfirmed_commands(
2938 &mut self,
2939 results: &mut [(usize, SeatState, AgentOutcome)],
2940 sent: &[SeatJob],
2941 prompts: &Prompts,
2942 run_id: &str,
2943 ) {
2944 for (wi, seat, out) in results.iter_mut() {
2945 let AgentOutcome::Ok(o) = &*out else {
2946 continue;
2947 };
2948 if !has_unconfirmed_command(&o.commands) {
2949 continue;
2950 }
2951 let Some(job) = sent.get(*wi) else { continue };
2952 if !has_context(&job.spec, seat, job.sessions) {
2953 self.state.event(
2954 "implement",
2955 format!(
2956 "{}: the reply named a command whose own CLI never confirmed the exit \
2957 status of, but there is no session left to resume",
2958 seat.key
2959 ),
2960 );
2961 continue;
2962 }
2963 self.state.event(
2964 "implement",
2965 format!(
2966 "{}: the reply named a command whose own CLI never confirmed the exit \
2967 status of; resuming the conversation",
2968 seat.key
2969 ),
2970 );
2971 let mut retry = job.clone();
2972 retry.seat = seat.clone();
2973 retry.prompt = prompt::resume_incomplete(
2974 "a command in your last reply had no confirmed exit status",
2975 );
2976 retry.timeout = retry_budget(job.timeout, true);
2977 retry.stem = format!("{}-confirm", job.stem);
2978 let cache = self.state.config.cache_dir();
2979 let ctx = WaveCtx {
2980 carry_seats: false,
2981 pauses: Some((&self.pause, &self.interrupt)),
2982 run: run_id,
2983 node: "implement",
2984 prompts,
2985 cache: cache.as_deref(),
2986 round: None,
2987 };
2988 let (resumed_seat, resumed) =
2989 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2990 *seat = resumed_seat;
2991 *out = resumed;
2992 }
2993 }
2994
2995 async fn continue_fix_report(
3016 &mut self,
3017 mut seat: SeatState,
3018 parse_err: String,
3019 job: &SeatJob,
3020 prompts: &Prompts,
3021 run_id: &str,
3022 round: usize,
3023 ) -> (
3024 SeatState,
3025 Option<FixReport>,
3026 Option<String>,
3027 ContinuationRecord,
3028 ) {
3029 let mut last_err = parse_err;
3030 let mut cumulative_wait_ms = 0u64;
3031 let mut attempts = 0usize;
3032 loop {
3033 if !has_context(&job.spec, &seat, job.sessions) {
3034 self.state.event(
3035 "fix",
3036 format!(
3037 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3038 session left to resume into"
3039 ),
3040 );
3041 let outcome = if attempts == 0 {
3042 ContinuationOutcome::NoSession
3043 } else {
3044 ContinuationOutcome::Exhausted
3045 };
3046 return (
3047 seat,
3048 None,
3049 Some(format!("unparsable fix report: {last_err}")),
3050 ContinuationRecord {
3051 attempts,
3052 cumulative_wait_ms,
3053 outcome,
3054 },
3055 );
3056 }
3057 if attempts >= MAX_FIX_CONTINUATIONS {
3058 self.state.event(
3059 "fix",
3060 format!(
3061 "round {round}: fixer's reply still had no adoption report after \
3062 {attempts} continuation(s) ({last_err}); giving up"
3063 ),
3064 );
3065 return (
3066 seat,
3067 None,
3068 Some(format!(
3069 "unparsable fix report after {attempts} continuation(s): {last_err}"
3070 )),
3071 ContinuationRecord {
3072 attempts,
3073 cumulative_wait_ms,
3074 outcome: ContinuationOutcome::Exhausted,
3075 },
3076 );
3077 }
3078 attempts += 1;
3079 self.state.event(
3080 "fix",
3081 format!(
3082 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3083 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3084 ),
3085 );
3086 let mut retry = job.clone();
3087 retry.seat = seat.clone();
3088 retry.prompt = prompt::resume_incomplete(&last_err);
3089 retry.timeout = retry_budget(job.timeout, true);
3090 retry.stem = format!("{}-continue{attempts}", job.stem);
3091 let cache = self.state.config.cache_dir();
3092 let ctx = WaveCtx {
3093 carry_seats: false,
3094 pauses: Some((&self.pause, &self.interrupt)),
3095 run: run_id,
3096 node: "fix",
3097 prompts,
3098 cache: cache.as_deref(),
3099 round: Some(round),
3100 };
3101 let (resumed_seat, resumed_out) = run_one(
3102 retry,
3103 Arc::clone(&self.sem),
3104 &ctx,
3105 &mut self.state,
3106 attempts,
3107 )
3108 .await;
3109 seat = resumed_seat;
3110 match resumed_out {
3111 AgentOutcome::Ok(o) => {
3112 cumulative_wait_ms += o.duration_ms;
3113 match verdict::extract_json::<FixReport>(&o.text) {
3114 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3115 self.state.event(
3116 "fix",
3117 format!(
3118 "round {round}: fixer's adoption report recovered after \
3119 {attempts} continuation(s)"
3120 ),
3121 );
3122 return (
3123 seat,
3124 Some(report),
3125 None,
3126 ContinuationRecord {
3127 attempts,
3128 cumulative_wait_ms,
3129 outcome: ContinuationOutcome::Resumed,
3130 },
3131 );
3132 }
3133 Ok(_) => {
3141 last_err = "the reply parsed, but it reported a command whose own CLI \
3142 never confirmed an exit status"
3143 .to_owned();
3144 }
3145 Err(e) => last_err = e.to_string(),
3146 }
3147 }
3148 AgentOutcome::Quota(o) => {
3149 cumulative_wait_ms += o.duration_ms;
3150 self.state.quota.push(QuotaLoss {
3151 seat: seat.key.clone(),
3152 node: "fix".to_owned(),
3153 at: Timestamp::now(),
3154 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3155 });
3156 self.state.event(
3157 "fix",
3158 format!(
3159 "round {round}: continuation rate limited (quota); not retrying now"
3160 ),
3161 );
3162 return (
3163 seat,
3164 None,
3165 Some("rate limited (quota) while recovering the fix report".to_owned()),
3166 ContinuationRecord {
3167 attempts,
3168 cumulative_wait_ms,
3169 outcome: ContinuationOutcome::QuotaLost,
3170 },
3171 );
3172 }
3173 AgentOutcome::Dropped(o) => {
3174 cumulative_wait_ms += o.duration_ms;
3175 let why = o
3176 .dropped
3177 .as_ref()
3178 .map(|d| d.why.as_str())
3179 .unwrap_or("the CLI ended the stream without delivering its answer");
3180 last_err = format!("the CLI dropped the stream ({why})");
3181 }
3182 AgentOutcome::Failed(e) => last_err = e,
3183 AgentOutcome::ResourceBlocked(e) => {
3186 return (
3187 seat,
3188 None,
3189 Some(format!("the continuation was not started: {e}")),
3190 ContinuationRecord {
3191 attempts: attempts.saturating_sub(1),
3192 cumulative_wait_ms,
3193 outcome: ContinuationOutcome::NoSession,
3194 },
3195 );
3196 }
3197 }
3198 }
3199 }
3200
3201 fn after_implement(&mut self) -> Result<()> {
3202 if !self.state.candidate_preservation.is_empty() {
3203 bail!("candidate preservation incomplete; judging is unsafe");
3204 }
3205 if self.state.leaks.is_empty() {
3207 let cfg = self.state.config.blind.clone();
3208 let mut leaks = Vec::new();
3209 for c in &self.state.candidates {
3210 let Some(patch) =
3211 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3212 else {
3213 continue;
3214 };
3215 leaks.extend(blind::scan(
3216 &format!("candidate {} patch", c.label),
3217 &patch,
3218 &cfg.vendor_tokens,
3219 ));
3220 }
3221 if !leaks.is_empty() {
3222 let summary = leaks
3223 .iter()
3224 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3225 .collect::<Vec<_>>()
3226 .join(", ");
3227 match cfg.on_leak {
3228 LeakPolicy::Fail => {
3229 self.state.status = RunStatus::Failed;
3230 self.state
3231 .event("blind", format!("vendor text in a patch: {summary}"));
3232 self.state.leaks = leaks;
3233 self.state.save()?;
3234 self.settle_questions();
3235 bail!(
3236 "blind.on_leak = \"fail\" and vendor text reached a \
3237 judged patch: {summary}"
3238 );
3239 }
3240 LeakPolicy::Redact => self.state.event(
3241 "blind",
3242 format!("redacting vendor text for judging: {summary}"),
3243 ),
3244 LeakPolicy::Warn => self.state.event(
3245 "blind",
3246 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3247 ),
3248 }
3249 self.state.leaks = leaks;
3250 }
3251 }
3252
3253 if self.state.viable().is_empty() {
3254 if self.state.all_candidates_verified_noop() {
3255 self.state.status = RunStatus::VerifiedNoop;
3266 self.state.save()?;
3267 self.settle_questions();
3268 return Ok(());
3269 }
3270 self.state.status = RunStatus::Failed;
3271 self.state.save()?;
3272 self.settle_questions();
3273 bail!("no candidate produced a change; nothing to judge");
3274 }
3275 self.state.status = RunStatus::Judging;
3276 self.state.save()?;
3277 Ok(())
3278 }
3279
3280 async fn judge(&mut self) -> Result<()> {
3283 let run_id = self.state.id.clone();
3288 let prompts = self.state.config.prompts.clone();
3289 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3290 return Ok(());
3291 }
3292 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3293 if viable.len() == 1 {
3294 self.state.judge_skipped = true;
3301 self.state.event(
3302 "judge",
3303 format!(
3304 "only candidate {} produced a change; judging skipped",
3305 viable[0].label
3306 ),
3307 );
3308 self.state.save()?;
3309 return Ok(());
3310 }
3311 self.state.status = RunStatus::Judging;
3312
3313 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3314 let language = self.state.config.graph.language.clone();
3315 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3316 let sessions = self.state.config.graph.sessions;
3317 let artifacts = agent::artifacts_dir(&self.state.dir());
3318 let root = self.state.worktree_root();
3319 let base_short = short(&self.state.base_commit);
3320
3321 let mut jobs = Vec::new();
3322 let mut orders = Vec::new();
3323 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3324 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3325 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3326 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3327 let seat_key = format!("judge-{}", j + 1);
3328 let seat = self.seat(&seat_key, &spec.id);
3329 jobs.push(SeatJob {
3330 prompt: prompt::judge(
3331 &self.state.instruction,
3332 &views,
3333 self.roles.judges.len(),
3334 &base_short,
3335 &language,
3336 ),
3337 spec,
3338 seat,
3339 cwd: root.join(format!("judge-{}", j + 1)),
3340 timeout,
3341 allow_write: false,
3342 sessions,
3343 artifacts: artifacts.clone(),
3344 stem: format!("judge-{}", j + 1),
3345 handover: None,
3346 });
3347 }
3348
3349 self.state.event(
3350 "judge",
3351 format!(
3352 "{} judges ranking {} candidates blind",
3353 jobs.len(),
3354 viable.len()
3355 ),
3356 );
3357 let labels_for_check = labels.clone();
3358 let mut quota_losses = Vec::new();
3359 let cache = self.state.config.cache_dir();
3360 let ctx = WaveCtx {
3361 carry_seats: false,
3362 pauses: Some((&self.pause, &self.interrupt)),
3363 run: &run_id,
3364 node: "judge",
3365 prompts: &prompts,
3366 cache: cache.as_deref(),
3367 round: None,
3368 };
3369 let results = ask_json_wave::<Ranking>(
3370 jobs,
3371 Arc::clone(&self.sem),
3372 self.state.config.graph.retries,
3373 &self.roles.judge_roster,
3374 &ctx,
3375 &mut quota_losses,
3376 &mut self.state,
3377 &move |r: &Ranking| r.validate(&labels_for_check),
3378 )
3379 .await;
3380 if self.state.resource_block.is_some() {
3381 return self.stop_for_resource();
3382 }
3383 self.state.quota.extend(quota_losses);
3384
3385 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3386 let agent_id = seat.agent.clone();
3387 self.state.seats.insert(seat.key.clone(), seat);
3388 let mut record = Judgement {
3389 judge: j + 1,
3390 seat: format!("judge-{}", j + 1),
3391 agent: agent_id,
3392 ranking: Vec::new(),
3393 reasons: BTreeMap::new(),
3394 confidence: None,
3395 strengths: BTreeMap::new(),
3396 order: orders[j].clone(),
3397 failed: None,
3398 duration_ms: 0,
3399 };
3400 match res {
3401 Ok((ranking, out)) => {
3402 record.ranking = ranking.normalized();
3403 record.reasons = ranking.reasons;
3404 record.strengths = ranking.strengths;
3405 record.confidence = ranking.confidence;
3406 record.duration_ms = out.duration_ms;
3407 self.state.event(
3408 "judge",
3409 format!(
3410 "judge {} ranked {}",
3411 j + 1,
3412 record.ranking.iter().collect::<String>()
3413 ),
3414 );
3415 }
3416 Err(e) => {
3417 record.failed = Some(e.to_string());
3418 self.state
3419 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3420 }
3421 }
3422 self.state.judgements.push(record);
3423 self.state.save()?;
3424 }
3425 Ok(())
3426 }
3427
3428 async fn deliberate(&mut self) -> Result<()> {
3431 let run_id = self.state.id.clone();
3436 let prompts = self.state.config.prompts.clone();
3437 if !self.state.deliberation.is_empty() {
3438 return Ok(());
3439 }
3440 let tops: Vec<char> = self
3441 .state
3442 .judgements
3443 .iter()
3444 .filter_map(|j| j.ranking.first().copied())
3445 .collect();
3446 let rounds = self.state.config.graph.deliberate_rounds;
3447 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3448 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3449 self.state.event(
3450 "deliberate",
3451 format!("judges agreed on {} outright; no deliberation", tops[0]),
3452 );
3453 }
3454 self.state.status = RunStatus::Voting;
3455 self.state.save()?;
3456 return Ok(());
3457 }
3458
3459 self.state.status = RunStatus::Deliberating;
3460 self.state.event(
3461 "deliberate",
3462 format!(
3463 "split: first choices were {} — opening {rounds} round(s)",
3464 tops.iter().collect::<String>()
3465 ),
3466 );
3467
3468 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3469 let language = self.state.config.graph.language.clone();
3470 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3471 let sessions = self.state.config.graph.sessions;
3472 let artifacts = agent::artifacts_dir(&self.state.dir());
3473 let root = self.state.worktree_root();
3474 let base_short = short(&self.state.base_commit);
3475
3476 for round in 1..=rounds {
3480 let mut turns: Vec<DeliberationTurn> = Vec::new();
3481 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3482 if self.state.judgements[j].failed.is_some() {
3483 continue;
3484 }
3485 let seat_key = format!("judge-{}", j + 1);
3486 let spec = self.occupant(&seat_key, spec);
3487 let mut seat = self.seat(&seat_key, &spec.id);
3488 let transcript = self.transcript(&turns, j);
3489 let build = |context: Option<&str>| {
3490 prompt::deliberate(
3491 &self.state.instruction,
3492 context,
3493 &transcript,
3494 round,
3495 rounds,
3496 &language,
3497 )
3498 };
3499 let block = self.candidate_block(&viable, &base_short);
3500 let full = build(Some(&block));
3501 let text = if has_context(&spec, &seat, sessions) {
3502 build(None)
3503 } else {
3504 full.clone()
3505 };
3506 let job = SeatJob {
3507 spec,
3508 seat: seat.clone(),
3509 prompt: text,
3510 cwd: root.join(format!("judge-{}", j + 1)),
3511 timeout,
3512 allow_write: false,
3513 sessions,
3514 artifacts: artifacts.clone(),
3515 stem: format!("delib-{round}-judge-{}", j + 1),
3516 handover: Some(full),
3517 };
3518 let cache = self.state.config.cache_dir();
3519 let ctx = WaveCtx {
3520 carry_seats: false,
3521 pauses: Some((&self.pause, &self.interrupt)),
3522 run: &run_id,
3523 node: "deliberate",
3524 prompts: &prompts,
3525 cache: cache.as_deref(),
3526 round: None,
3527 };
3528 let mut losses = Vec::new();
3532 let mut results = ask_wave_with::<String>(
3533 vec![job],
3534 Arc::clone(&self.sem),
3535 0,
3536 &self.roles.judge_roster,
3537 &ctx,
3538 &mut losses,
3539 &mut self.state,
3540 &|text: &str| {
3541 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3542 },
3543 )
3544 .await;
3545 self.state.quota.extend(losses);
3546 let (updated, res, _) = results.pop().expect("one job in, one result out");
3547 seat = updated;
3548 let agent_id = seat.agent.clone();
3549 self.state.seats.insert(seat.key.clone(), seat);
3550 let body = match res {
3551 Ok((body, _)) => body,
3552 Err(e) => {
3555 self.state
3556 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3557 continue;
3558 }
3559 };
3560 let tentative = verdict::extract_json::<Position>(&body)
3561 .ok()
3562 .and_then(|p| p.tentative)
3563 .and_then(|s| s.trim().chars().next())
3564 .map(|c| c.to_ascii_uppercase());
3565 self.state.event(
3566 "deliberate",
3567 format!(
3568 "round {round}: judge {} now favours {}",
3569 j + 1,
3570 tentative.map_or("—".to_owned(), |c| c.to_string())
3571 ),
3572 );
3573 turns.push(DeliberationTurn {
3574 judge: j + 1,
3575 agent: agent_id,
3576 body: blind::sanitize_prose(&body, &self.state.config.blind),
3577 tentative,
3578 });
3579 }
3580 self.state
3581 .deliberation
3582 .push(DeliberationRound { round, turns });
3583 self.state.save()?;
3584 }
3585
3586 self.state.status = RunStatus::Voting;
3587 self.state.save()?;
3588 Ok(())
3589 }
3590
3591 async fn vote(&mut self) -> Result<()> {
3594 let run_id = self.state.id.clone();
3599 let prompts = self.state.config.prompts.clone();
3600 if !self.state.votes.is_empty() {
3601 return Ok(());
3602 }
3603 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3604 if viable.len() == 1 {
3605 return Ok(());
3606 }
3607 self.state.status = RunStatus::Voting;
3608
3609 let language = self.state.config.graph.language.clone();
3610 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3611 let sessions = self.state.config.graph.sessions;
3612 let artifacts = agent::artifacts_dir(&self.state.dir());
3613 let root = self.state.worktree_root();
3614 let base_short = short(&self.state.base_commit);
3615 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3616
3617 let mut jobs = Vec::new();
3618 let mut seats_at = Vec::new();
3619 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3620 if self
3621 .state
3622 .judgements
3623 .get(j)
3624 .is_some_and(|r| r.failed.is_some())
3625 {
3626 continue;
3627 }
3628 let seat_key = format!("judge-{}", j + 1);
3629 let spec = self.occupant(&seat_key, spec);
3630 let seat = self.seat(&seat_key, &spec.id);
3631 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3632 let text = if has_context(&spec, &seat, sessions) {
3633 prompt::final_vote(&viable, &language)
3634 } else {
3635 full.clone()
3636 };
3637 jobs.push(SeatJob {
3638 spec,
3639 seat,
3640 prompt: text,
3641 cwd: root.join(format!("judge-{}", j + 1)),
3642 timeout,
3643 allow_write: false,
3644 sessions,
3645 artifacts: artifacts.clone(),
3646 stem: format!("vote-judge-{}", j + 1),
3647 handover: Some(full),
3648 });
3649 seats_at.push(j);
3650 }
3651
3652 self.state.event(
3653 "vote",
3654 format!(
3655 "collecting {} final votes one by one, privately",
3656 jobs.len()
3657 ),
3658 );
3659 let allowed = viable.clone();
3660 let mut quota_losses = Vec::new();
3661 let cache = self.state.config.cache_dir();
3662 let ctx = WaveCtx {
3663 carry_seats: false,
3664 pauses: Some((&self.pause, &self.interrupt)),
3665 run: &run_id,
3666 node: "vote",
3667 prompts: &prompts,
3668 cache: cache.as_deref(),
3669 round: None,
3670 };
3671 let results = ask_json_wave::<FinalVote>(
3672 jobs,
3673 Arc::clone(&self.sem),
3674 self.state.config.graph.retries,
3675 &self.roles.judge_roster,
3676 &ctx,
3677 &mut quota_losses,
3678 &mut self.state,
3679 &move |v: &FinalVote| match v.label() {
3680 Some(c) if allowed.contains(&c) => Ok(()),
3681 other => bail!("vote {other:?} is not one of {allowed:?}"),
3682 },
3683 )
3684 .await;
3685 if self.state.resource_block.is_some() {
3686 return self.stop_for_resource();
3687 }
3688 self.state.quota.extend(quota_losses);
3689
3690 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3691 let agent_id = seat.agent.clone();
3692 self.state.seats.insert(seat.key.clone(), seat);
3693 let initial = self
3694 .state
3695 .judgements
3696 .get(j)
3697 .and_then(|r| r.ranking.first().copied());
3698 let mut record = VoteRecord {
3699 judge: j + 1,
3700 agent: agent_id,
3701 vote: None,
3702 reason: String::new(),
3703 changed: false,
3704 };
3705 match res {
3706 Ok((v, _)) => {
3707 record.vote = v.label();
3708 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3709 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3710 self.state.event(
3711 "vote",
3712 format!(
3713 "judge {} voted {}{}",
3714 j + 1,
3715 record.vote.unwrap_or('?'),
3716 if record.changed { " (changed)" } else { "" }
3717 ),
3718 );
3719 }
3720 Err(e) => {
3721 self.state
3722 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3723 }
3724 }
3725 self.state.votes.push(record);
3726 self.state.save()?;
3727 }
3728 Ok(())
3729 }
3730
3731 fn tally(&mut self) -> Result<()> {
3734 if self.state.tally.is_some() {
3735 return Ok(());
3736 }
3737 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3738 let tops: Vec<char> = self
3739 .state
3740 .judgements
3741 .iter()
3742 .filter_map(|j| j.ranking.first().copied())
3743 .collect();
3744 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3745
3746 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3749 let mut cast: Vec<char> = Vec::new();
3750 for (i, j) in self.state.judgements.iter().enumerate() {
3751 let vote = self
3752 .state
3753 .votes
3754 .iter()
3755 .find(|v| v.judge == i + 1)
3756 .and_then(|v| v.vote)
3757 .or_else(|| j.ranking.first().copied());
3758 if let Some(v) = vote {
3759 *first_choice.entry(v).or_insert(0) += 1;
3760 cast.push(v);
3761 }
3762 }
3763
3764 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3765 for j in &self.state.judgements {
3766 let n = j.ranking.len();
3767 for (pos, label) in j.ranking.iter().enumerate() {
3768 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3769 }
3770 }
3771
3772 let best = first_choice.values().copied().max().unwrap_or(0);
3773 let mut leaders: Vec<char> = first_choice
3774 .iter()
3775 .filter(|(_, v)| **v == best)
3776 .map(|(k, _)| *k)
3777 .collect();
3778 let mut tie_break = None;
3779 if leaders.len() > 1 {
3780 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3781 let borda_leaders: Vec<char> = leaders
3782 .iter()
3783 .copied()
3784 .filter(|l| borda[l] == top_borda)
3785 .collect();
3786 tie_break = Some(if borda_leaders.len() == 1 {
3787 format!(
3788 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3789 leaders.len()
3790 )
3791 } else {
3792 format!(
3793 "{} way tie on both first-choice votes and Borda points, broken by label order",
3794 leaders.len()
3795 )
3796 });
3797 leaders = borda_leaders;
3798 leaders.sort_unstable();
3799 }
3800 let winner = *leaders
3801 .first()
3802 .or(viable.first())
3803 .context("no candidate to declare a winner from")?;
3804
3805 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3806 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3807 let deliberated = !self.state.deliberation.is_empty();
3808
3809 let quota_seats: std::collections::BTreeSet<&str> =
3813 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3814 let mut present = 0usize;
3815 for (i, j) in self.state.judgements.iter().enumerate() {
3816 if quota_seats.contains(j.seat.as_str()) {
3817 continue;
3818 }
3819 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3820 let voted = self
3821 .state
3822 .votes
3823 .iter()
3824 .any(|v| v.judge == i + 1 && v.vote.is_some());
3825 if ranked || voted {
3826 present += 1;
3827 }
3828 }
3829 let needs_quorum = viable.len() > 1;
3835 let judges_total = if needs_quorum {
3836 self.roles.judges.len()
3837 } else {
3838 0
3839 };
3840 let quorum = if needs_quorum {
3841 judges_total / 2 + 1
3842 } else {
3843 0
3844 };
3845 let met_quorum = !needs_quorum || present >= quorum;
3846 let uncontested = (!needs_quorum).then(|| {
3847 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3848 });
3849
3850 self.state.event(
3851 "tally",
3852 match &uncontested {
3853 Some(reason) => format!("winner {winner} — {reason}"),
3854 None => format!(
3855 "winner {winner} — votes {} | initial {} | {} changed | \
3856 {present}/{judges_total} judges{}",
3857 first_choice
3858 .iter()
3859 .map(|(k, v)| format!("{k}:{v}"))
3860 .collect::<Vec<_>>()
3861 .join(" "),
3862 if unanimous_initial {
3863 "unanimous"
3864 } else {
3865 "split"
3866 },
3867 changed_votes,
3868 if met_quorum {
3869 String::new()
3870 } else {
3871 format!(" — below quorum ({quorum} required)")
3872 },
3873 ),
3874 },
3875 );
3876 if !met_quorum {
3877 self.state.event(
3878 "stall",
3879 format!(
3880 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3881 the run stops here, resumable"
3882 ),
3883 );
3884 }
3885 self.state.tally = Some(Tally {
3886 first_choice,
3887 borda,
3888 winner,
3889 rankings: tops.len(),
3890 unanimous_initial,
3891 deliberated,
3892 changed_votes,
3893 unanimous_final,
3894 tie_break,
3895 judges: judges_total,
3896 present,
3897 quorum,
3898 met_quorum,
3899 uncontested,
3900 });
3901 self.state.status = if met_quorum {
3902 RunStatus::Reviewing
3903 } else {
3904 RunStatus::Stalled
3905 };
3906 self.state.save()?;
3907 Ok(())
3908 }
3909
3910 #[allow(clippy::too_many_lines)]
3931 async fn recover_stall(&mut self) -> Result<bool> {
3932 let run_id = self.state.id.clone();
3937 let prompts = self.state.config.prompts.clone();
3938 let quota_seats: BTreeSet<&str> =
3943 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3944 let absent: Vec<String> = self
3945 .state
3946 .judgements
3947 .iter()
3948 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3949 .map(|j| j.seat.clone())
3950 .collect();
3951 if absent.is_empty() {
3952 return Ok(false);
3953 }
3954 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3955 if viable.len() <= 1 {
3956 return Ok(false);
3957 }
3958 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3959 let language = self.state.config.graph.language.clone();
3960 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3961 let sessions = self.state.config.graph.sessions;
3962 let artifacts = agent::artifacts_dir(&self.state.dir());
3963 let root = self.state.worktree_root();
3964 let base_short = short(&self.state.base_commit);
3965 let candidates: Vec<Candidate> = viable.clone();
3966
3967 let mut positions: Vec<usize> = absent
3969 .iter()
3970 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3971 .collect();
3972 if positions.is_empty() {
3973 return Ok(false);
3974 }
3975 positions.sort_unstable();
3976 positions.dedup();
3977
3978 let mut judge_jobs = Vec::new();
3980 for &j in &positions {
3981 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3982 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3983 let seat_key = format!("judge-{}", j + 1);
3984 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3985 let seat = self.seat(&seat_key, &spec.id);
3986 judge_jobs.push(SeatJob {
3987 spec,
3988 seat,
3989 prompt: prompt::judge(
3990 &self.state.instruction,
3991 &views,
3992 self.roles.judges.len(),
3993 &base_short,
3994 &language,
3995 ),
3996 cwd: root.join(seat_key),
3997 timeout,
3998 allow_write: false,
3999 sessions,
4000 artifacts: artifacts.clone(),
4001 stem: format!("judge-{}-recover", j + 1),
4002 handover: None,
4003 });
4004 }
4005
4006 let labels_for_check = labels.clone();
4007 let mut judge_losses = Vec::new();
4008 let retries = self.state.config.graph.retries;
4009 let cache = self.state.config.cache_dir();
4010 let ctx = WaveCtx {
4011 carry_seats: false,
4012 pauses: Some((&self.pause, &self.interrupt)),
4013 run: &run_id,
4014 node: "judge",
4015 prompts: &prompts,
4016 cache: cache.as_deref(),
4017 round: None,
4018 };
4019 let results = ask_json_wave::<Ranking>(
4020 judge_jobs,
4021 Arc::clone(&self.sem),
4022 retries,
4023 &self.roles.judge_roster,
4024 &ctx,
4025 &mut judge_losses,
4026 &mut self.state,
4027 &move |r: &Ranking| r.validate(&labels_for_check),
4028 )
4029 .await;
4030 if self.state.resource_block.is_some() {
4031 self.stop_for_resource()?;
4032 return Ok(false);
4033 }
4034
4035 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4037 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4038 let agent_id = seat.agent.clone();
4039 self.state.seats.insert(seat.key.clone(), seat);
4040 let record = &mut self.state.judgements[j];
4041 match res {
4042 Ok((ranking, out)) => {
4043 record.agent = agent_id;
4044 record.ranking = ranking.normalized();
4045 record.reasons = ranking.reasons;
4046 record.strengths = ranking.strengths;
4047 record.confidence = ranking.confidence;
4048 record.failed = None;
4049 record.duration_ms = out.duration_ms;
4050 recovered.insert(j);
4051 self.state.event(
4052 "recover",
4053 format!("judge {} ranked again after the limit", j + 1),
4054 );
4055 }
4056 Err(e) => {
4057 self.state
4058 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4059 }
4060 }
4061 }
4062
4063 let mut vote_jobs = Vec::new();
4065 let mut vote_pos: Vec<usize> = Vec::new();
4066 for &j in &recovered {
4067 let seat_key = format!("judge-{}", j + 1);
4068 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4069 let seat = self.seat(&seat_key, &spec.id);
4070 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4071 let text = if has_context(&spec, &seat, sessions) {
4072 prompt::final_vote(&labels, &language)
4073 } else {
4074 full.clone()
4075 };
4076 vote_jobs.push(SeatJob {
4077 spec,
4078 seat,
4079 prompt: text,
4080 cwd: root.join(seat_key),
4081 timeout,
4082 allow_write: false,
4083 sessions,
4084 artifacts: artifacts.clone(),
4085 stem: format!("vote-judge-{}-recover", j + 1),
4086 handover: Some(full),
4087 });
4088 vote_pos.push(j);
4089 }
4090 let allowed = labels.clone();
4091 let mut vote_losses = Vec::new();
4092 let vote_retries = self.state.config.graph.retries;
4093 let vote_cache = self.state.config.cache_dir();
4094 let ctx = WaveCtx {
4095 carry_seats: false,
4096 pauses: Some((&self.pause, &self.interrupt)),
4097 run: &run_id,
4098 node: "vote",
4099 prompts: &prompts,
4100 cache: vote_cache.as_deref(),
4101 round: None,
4102 };
4103 let votes = ask_json_wave::<FinalVote>(
4104 vote_jobs,
4105 Arc::clone(&self.sem),
4106 vote_retries,
4107 &self.roles.judge_roster,
4108 &ctx,
4109 &mut vote_losses,
4110 &mut self.state,
4111 &move |v: &FinalVote| match v.label() {
4112 Some(c) if allowed.contains(&c) => Ok(()),
4113 other => bail!("vote {other:?} is not one of {allowed:?}"),
4114 },
4115 )
4116 .await;
4117 if self.state.resource_block.is_some() {
4118 self.stop_for_resource()?;
4119 return Ok(false);
4120 }
4121 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4122 let agent_id = seat.agent.clone();
4123 self.state.seats.insert(seat.key.clone(), seat);
4124 match res {
4125 Ok((v, _)) => {
4126 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4127 rec.vote = v.label();
4128 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4129 } else {
4130 self.state.votes.push(VoteRecord {
4131 judge: j + 1,
4132 agent: agent_id,
4133 vote: v.label(),
4134 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4135 changed: false,
4136 });
4137 }
4138 self.state.event(
4139 "recover",
4140 format!("judge {} voted again after the limit", j + 1),
4141 );
4142 }
4143 Err(e) => {
4144 self.state
4145 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4146 }
4147 }
4148 }
4149
4150 let recovered_keys: BTreeSet<String> = recovered
4154 .iter()
4155 .map(|&j| format!("judge-{}", j + 1))
4156 .collect();
4157 self.state
4158 .quota
4159 .retain(|q| !recovered_keys.contains(&q.seat));
4160 for loss in judge_losses.into_iter().chain(vote_losses) {
4164 if recovered_keys.contains(&loss.seat) {
4165 continue;
4166 }
4167 self.state.quota.retain(|q| q.seat != loss.seat);
4168 self.state.quota.push(loss);
4169 }
4170
4171 self.state.tally = None;
4173 self.tally()?;
4174 Ok(self
4175 .state
4176 .tally
4177 .as_ref()
4178 .map(|t| t.met_quorum)
4179 .unwrap_or(false))
4180 }
4181
4182 async fn graft(&mut self) -> Result<()> {
4198 let Err(e) = self.graft_inner().await else {
4199 return Ok(());
4200 };
4201 if e.is::<crate::repo_lock::ResourceBlocked>() {
4202 return Err(e);
4203 }
4204 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4205 self.state.status = RunStatus::Blocked;
4206 self.state.event("graft", format!("blocked: {e:#}"));
4207 self.state.save()?;
4208 return Err(e);
4209 }
4210 if let Some(rec) = self.state.graft.clone()
4215 && !rec.outcome.settled()
4216 {
4217 if let Some(winner) = self.state.winner().cloned() {
4218 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4219 if re.is::<crate::repo_lock::ResourceBlocked>() {
4220 return Err(re);
4221 }
4222 self.state.status = RunStatus::Blocked;
4223 self.state.event("graft", format!("blocked: {re:#}"));
4224 self.state.save()?;
4225 return Err(re);
4226 }
4227 }
4228 if let Some(g) = self.state.graft.as_mut() {
4229 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4230 }
4231 }
4232 self.state.event("graft", format!("skipped: {e:#}"));
4233 if let Err(e) = self.state.save() {
4234 tracing::warn!("save after graft: {e:#}");
4235 }
4236 Ok(())
4237 }
4238
4239 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4242 let _coordination = self.coordinate(wt).await?;
4243 crate::recovery::restore_repair(wt, pre)
4244 .await
4245 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4246 }
4247
4248 fn graft_skipped(&mut self, why: &str) {
4250 let message = format!("skipped: {why}");
4251 if !self
4252 .state
4253 .events
4254 .iter()
4255 .any(|e| e.node == "graft" && e.message == message)
4256 {
4257 self.state.event("graft", message);
4258 }
4259 }
4260
4261 async fn graft_inner(&mut self) -> Result<()> {
4262 if let Some(rec) = self.state.graft.clone()
4265 && !rec.outcome.settled()
4266 {
4267 if self.state.reviews.is_empty()
4271 && let Some(winner) = self.state.winner().cloned()
4272 {
4273 self.graft_restore(&winner.worktree, &rec.pre).await?;
4274 }
4275 if rec.outcome == GraftOutcome::WaitingForRestore {
4276 self.state.graft = None;
4277 self.state.save()?;
4278 } else {
4279 if let Some(g) = self.state.graft.as_mut() {
4280 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4281 }
4282 self.state.event(
4283 "graft",
4284 "rolled back: the previous round was interrupted; the winner is untouched",
4285 );
4286 self.state.save()?;
4287 return Ok(());
4288 }
4289 }
4290 let plan = match graft::plan(&self.state) {
4291 Ok(plan) => plan,
4292 Err(skip) => {
4293 if skip != graft::Skip::AlreadyDone {
4294 self.graft_skipped(skip.reason());
4295 self.state.save()?;
4296 }
4297 return Ok(());
4298 }
4299 };
4300 let Some(winner) = self.state.winner().cloned() else {
4301 return Ok(());
4302 };
4303 let wt = winner.worktree.clone();
4304 if !git::is_clean(&wt).await? {
4305 self.graft_skipped("the winner's worktree has uncommitted changes");
4306 self.state.save()?;
4307 return Ok(());
4308 }
4309 let pre = git::rev_parse(&wt, "HEAD").await?;
4310 self.state.graft = Some(GraftRecord {
4313 pre: pre.clone(),
4314 post: None,
4315 lines: None,
4316 limit: None,
4317 agent: None,
4318 outcome: GraftOutcome::Running,
4319 requested: graft::requested(&plan),
4320 adoption: None,
4321 adoption_error: None,
4322 });
4323 self.state.event(
4324 "graft",
4325 format!(
4326 "porting strengths of candidate(s) {} onto {}",
4327 plan.strengths
4328 .iter()
4329 .map(|(l, _, _)| l.to_string())
4330 .collect::<Vec<_>>()
4331 .join(", "),
4332 plan.winner
4333 ),
4334 );
4335 self.state.save()?;
4336
4337 let fix_prompt = prompt::graft(
4338 &self.state.instruction,
4339 plan.winner,
4340 &plan.strengths,
4341 &self.state.config.graph.language,
4342 );
4343 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4344 let sessions = self.state.config.graph.sessions;
4345 let artifacts = agent::artifacts_dir(&self.state.dir());
4346 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4347 let (_, seat, out) = self
4348 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4349 SeatJob {
4350 prompt: fix_prompt.clone(),
4351 spec,
4352 seat,
4353 cwd: wt.clone(),
4354 timeout,
4355 allow_write: true,
4356 sessions,
4357 artifacts: artifacts.clone(),
4358 stem: "graft".to_owned(),
4359 handover: None,
4360 }
4361 })
4362 .await;
4363 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4364 self.state.graft.as_mut().expect("graft record").outcome =
4367 GraftOutcome::WaitingForRestore;
4368 return self.stop_for_resource();
4369 }
4370 let mut why: Option<String> = None;
4371 match &out {
4372 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4373 why = Some("the fixer produced nothing usable".to_owned());
4374 }
4375 AgentOutcome::Ok(_) => {}
4376 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4377 AgentOutcome::Quota(o) => {
4378 self.state.quota.push(QuotaLoss {
4379 seat: seat.key.clone(),
4380 node: "graft".to_owned(),
4381 at: Timestamp::now(),
4382 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4383 });
4384 why = Some("rate limited (quota)".to_owned());
4385 }
4386 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4387 why = Some(format!(
4388 "the fixer failed: {}",
4389 e.lines()
4390 .next()
4391 .unwrap_or("")
4392 .chars()
4393 .take(160)
4394 .collect::<String>()
4395 ));
4396 }
4397 }
4398 let agent_id = seat.agent.clone();
4399 self.state.seats.insert(seat.key.clone(), seat);
4400
4401 let mut post = None;
4402 let mut lines = None;
4403 let _coordination = self.coordinate(&wt).await?;
4404 if why.is_none() {
4405 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4406 if let Ok(r) = &rescued {
4407 self.state.note_withheld("graft", &r.withheld);
4408 }
4409 let head = git::rev_parse(&wt, "HEAD").await?;
4410 let dirty = match &rescued {
4414 Ok(r) => git::status_porcelain(&wt)
4415 .await
4416 .map(|st| {
4417 st.lines().any(|l| {
4418 !r.withheld
4419 .iter()
4420 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4421 })
4422 })
4423 .unwrap_or(true),
4424 Err(_) => true,
4425 };
4426 if dirty {
4427 why = Some("the fixer left changes that could not be committed".to_owned());
4428 } else if head == pre {
4429 why = Some("the fixer changed nothing".to_owned());
4430 } else {
4431 lines = git::changed_lines(&wt, &pre, &head).await;
4432 post = Some(head);
4433 }
4434 }
4435 let (adoption, adoption_error) = match &out {
4436 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4437 _ => (None, None),
4438 };
4439 let requested = graft::requested(&plan);
4440 let outcome = match why {
4441 None => {
4442 self.state.event(
4443 "graft",
4444 format!(
4445 "applied: {}",
4446 lines.map_or_else(
4447 || "changed lines unavailable".to_owned(),
4448 |n| format!("{n} changed lines")
4449 )
4450 ),
4451 );
4452 GraftOutcome::Applied
4453 }
4454 Some(why) => {
4455 crate::recovery::restore_repair(&wt, &pre)
4458 .await
4459 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4460 self.state.event(
4461 "graft",
4462 format!("rolled back: {why}; the winner is untouched"),
4463 );
4464 GraftOutcome::RolledBack(why)
4465 }
4466 };
4467 self.state.graft = Some(GraftRecord {
4468 pre,
4469 post,
4470 lines,
4471 limit: None,
4472 agent: Some(agent_id),
4473 outcome,
4474 requested,
4475 adoption,
4476 adoption_error,
4477 });
4478 self.state.save()?;
4479 Ok(())
4480 }
4481
4482 async fn fold_losers(&mut self) -> Result<()> {
4485 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4486 return Ok(());
4487 };
4488 if !self.state.candidate_preservation.is_empty() {
4489 bail!("candidate preservation incomplete; refusing cleanup");
4490 }
4491 let repo = self.state.repo.clone();
4492 let _coordination = self.coordinate(&repo).await?;
4493 let mut folded = Vec::new();
4494 for i in 0..self.state.candidates.len() {
4495 let c = &self.state.candidates[i];
4496 if c.label == winner || c.folded {
4497 continue;
4498 }
4499 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4500 remove_preserved_worktree(&repo, &wt).await?;
4501 if git::branch_exists(&repo, &branch).await?
4502 && !git::branch_delete(&repo, &branch).await?
4503 {
4504 bail!("candidate branch removal refused");
4505 }
4506 self.state.candidates[i].folded = true;
4507 self.state.save()?;
4508 folded.push(label.to_string());
4509 }
4510 let root = self.state.worktree_root();
4512 for j in 1..=self.roles.judges.len() {
4513 let wt = root.join(format!("judge-{j}"));
4514 if wt.exists() {
4515 remove_preserved_worktree(&repo, &wt).await?;
4516 }
4517 }
4518 if self.state.config.graph.advise {
4521 for k in 1..=self.state.config.graph.advisors {
4522 let wt = root.join(format!("advisor-{k}"));
4523 if wt.exists() {
4524 remove_preserved_worktree(&repo, &wt).await?;
4525 }
4526 }
4527 }
4528 if !folded.is_empty() {
4529 self.state
4530 .event("fold", format!("folded candidates {}", folded.join(", ")));
4531 self.state.save()?;
4532 }
4533 Ok(())
4534 }
4535
4536 async fn sync_to_base(&mut self) -> Result<()> {
4576 if self.state.status == RunStatus::AlreadyInBase {
4577 return Ok(());
4578 }
4579 let conflicted = self
4580 .state
4581 .base_sync
4582 .as_ref()
4583 .is_some_and(|s| s.conflict.is_some());
4584 let Some(winner) = self.state.winner().cloned() else {
4585 return Ok(());
4586 };
4587
4588 let repo = self.state.repo.clone();
4589 let remote = self.state.config.merge.remote.clone();
4590 let base_branch = self.state.base_branch.clone();
4591 let tracking = format!("{remote}/{base_branch}");
4592
4593 git::fetch(&repo, &remote, &base_branch).await.ok();
4594 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4599 return Ok(());
4600 };
4601
4602 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4603 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4604 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4605
4606 if (behind > 0 || conflicted || head == tip)
4615 && self
4616 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4617 .await?
4618 {
4619 return Ok(());
4620 }
4621 if conflicted {
4622 return Ok(());
4623 }
4624
4625 if behind == 0 {
4626 if let Some(from) = self
4635 .state
4636 .rebase_fixes
4637 .iter()
4638 .rev()
4639 .find_map(|r| r.from.clone())
4640 && from != head
4641 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4642 .await
4643 .is_ok_and(|o| o.ok())
4644 {
4645 git::sync_to_head(&winner.worktree).await?;
4646 }
4647 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4653 if let Some(why) = &conflict {
4654 self.state.status = RunStatus::Blocked;
4655 self.state.event("land", why.clone());
4656 }
4657 self.state.base_sync = Some(BaseSync {
4658 tip,
4659 behind: 0,
4660 attempts,
4661 conflict,
4662 already_in: None,
4663 });
4664 self.state.save()?;
4665 return Ok(());
4666 }
4667
4668 if attempts >= BASE_SYNC_ROUNDS {
4669 let why = format!(
4670 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4671 rebase(s); rebasing again would only race it",
4672 winner.branch
4673 );
4674 self.state.status = RunStatus::Blocked;
4675 self.state.base_sync = Some(BaseSync {
4676 tip,
4677 behind,
4678 attempts,
4679 conflict: Some(why.clone()),
4680 already_in: None,
4681 });
4682 self.state.event("land", why);
4683 self.state.save()?;
4684 return Ok(());
4685 }
4686
4687 self.state.event(
4688 "land",
4689 format!(
4690 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4691 winner.branch
4692 ),
4693 );
4694 self.state.save()?;
4695
4696 let branch_tracking = format!("{remote}/{}", winner.branch);
4702 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4703 let remote_tip = if self.state.task_pr_branch().is_none()
4704 && matches!(&fetched_branch, Ok(o) if o.ok())
4705 {
4706 git::rev_parse(&repo, &branch_tracking).await.ok()
4707 } else {
4708 None
4709 };
4710 if let Some(theirs) = &remote_tip
4714 && !git::is_ancestor(&repo, theirs, &head).await
4715 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4716 .await
4717 .is_ok_and(|missing| missing.is_empty())
4718 {
4719 let why = format!(
4720 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4721 them",
4722 short(theirs),
4723 winner.branch
4724 );
4725 self.state.status = RunStatus::Blocked;
4726 self.state.base_sync = Some(BaseSync {
4727 tip,
4728 behind,
4729 attempts,
4730 conflict: Some(why.clone()),
4731 already_in: None,
4732 });
4733 self.state.event("land", why);
4734 self.state.save()?;
4735 return Ok(());
4736 }
4737
4738 let scratch = self.state.dir().join("base-sync");
4739 let rebased = match crate::rebase::rebase_with_fixer(
4740 &mut self.state,
4741 &scratch,
4742 &winner.branch,
4743 &tracking,
4744 )
4745 .await
4746 {
4747 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4748 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4749 Err(e) => Err(e),
4750 };
4751 let attempts = attempts + 1;
4752 match rebased {
4753 Ok(None) => {
4754 git::sync_to_head(&winner.worktree).await?;
4758 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4759 let mut conflict = None;
4760 if let Some(pinned) = &remote_tip {
4761 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4762 match pushed {
4763 Ok(o) if o.ok() => self.state.event(
4764 "land",
4765 format!("pushed rebased {} to {remote}", winner.branch),
4766 ),
4767 Ok(o) => {
4768 conflict = Some(format!(
4769 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4770 winner.branch,
4771 short(pinned),
4772 o.stderr.chars().take(600).collect::<String>()
4773 ));
4774 }
4775 Err(e) => {
4776 conflict = Some(format!(
4777 "rebased {} locally but could not push it: {e:#}",
4778 winner.branch
4779 ));
4780 }
4781 }
4782 }
4783 if let Some(why) = &conflict {
4784 self.state.status = RunStatus::Blocked;
4785 self.state.event("land", why.clone());
4786 }
4787 self.state.base_sync = Some(BaseSync {
4788 tip: tip.clone(),
4789 behind: 0,
4790 attempts,
4791 conflict,
4792 already_in: None,
4793 });
4794 self.state
4795 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4796 }
4797 Ok(Some(conflict)) => {
4798 let why = format!(
4799 "{} conflicts with {tracking} and did not rebase: {}",
4800 winner.branch,
4801 conflict.chars().take(600).collect::<String>()
4802 );
4803 self.state.status = RunStatus::Blocked;
4804 self.state.base_sync = Some(BaseSync {
4805 tip,
4806 behind,
4807 attempts,
4808 conflict: Some(why.clone()),
4809 already_in: None,
4810 });
4811 self.state.event("land", why);
4812 }
4813 Err(e) => {
4814 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4815 self.state.status = RunStatus::Blocked;
4816 self.state.base_sync = Some(BaseSync {
4817 tip,
4818 behind,
4819 attempts,
4820 conflict: Some(why.clone()),
4821 already_in: None,
4822 });
4823 self.state.event("land", why);
4824 }
4825 }
4826 self.state.save()?;
4827 Ok(())
4828 }
4829
4830 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4836 if self.state.task_pr_branch().is_some() {
4839 return None;
4840 }
4841 let repo = self.state.repo.clone();
4842 let remote = self.state.config.merge.remote.clone();
4843 let fetched = git::fetch(&repo, &remote, branch).await;
4844 if !matches!(&fetched, Ok(o) if o.ok()) {
4845 return None;
4846 }
4847 let branch_tracking = format!("{remote}/{branch}");
4848 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4849 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4850 return None;
4851 }
4852 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4853 .await
4854 .is_ok_and(|missing| missing.is_empty())
4855 {
4856 return Some(format!(
4857 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4858 them",
4859 short(&theirs)
4860 ));
4861 }
4862 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4863 Ok(o) if o.ok() => {
4864 self.state
4865 .event("land", format!("pushed rebased {branch} to {remote}"));
4866 None
4867 }
4868 Ok(o) => Some(format!(
4869 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4870 someone may have pushed): {}",
4871 short(&theirs),
4872 o.stderr.chars().take(600).collect::<String>()
4873 )),
4874 Err(e) => Some(format!(
4875 "{branch} is rebased locally but could not be pushed: {e:#}"
4876 )),
4877 }
4878 }
4879
4880 async fn settle_already_in(
4897 &mut self,
4898 branch: &str,
4899 tip: &str,
4900 head: &str,
4901 attempts: usize,
4902 behind: usize,
4903 ) -> Result<bool> {
4904 let repo = self.state.repo.clone();
4905 let remote = self.state.config.merge.remote.clone();
4906 let start = self.state.base_commit.clone();
4907 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4908 Ok(Some(e)) => e,
4909 Ok(None) => return Ok(false),
4910 Err(e) => {
4911 tracing::warn!("already-in-base check for {branch}: {e:#}");
4912 return Ok(false);
4913 }
4914 };
4915 let mut verified = vec![head.to_owned()];
4916 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4917 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4918 if matches!(&fetched, Ok(o) if o.ok())
4919 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4920 && theirs != head
4921 {
4922 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4923 Ok(Some(_)) => verified.push(theirs),
4924 _ => return Ok(false),
4925 }
4926 }
4927 let base_branch = self.state.base_branch.clone();
4928 let message = format!(
4929 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4930 land",
4931 evidence.names(),
4932 evidence.proof.as_str()
4933 );
4934 let closed =
4935 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4936 .await;
4937 match closed {
4938 Ok(Ok(url)) => self
4939 .state
4940 .event("land", format!("closed {url}: superseded on {base_branch}")),
4941 Ok(Err(why)) => self
4942 .state
4943 .event("land", format!("did not close a pull request: {why}")),
4944 Err(e) => {
4945 let why = format!(
4946 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4947 not be closed ({e:#}); resume to retry"
4948 );
4949 self.state.status = RunStatus::Blocked;
4950 self.state.base_sync = Some(BaseSync {
4951 tip: tip.to_owned(),
4952 behind,
4953 attempts,
4954 conflict: Some(why.clone()),
4955 already_in: None,
4956 });
4957 self.state.event("land", why);
4958 self.state.save()?;
4959 return Ok(true);
4960 }
4961 }
4962 self.state.status = RunStatus::AlreadyInBase;
4963 self.state.base_sync = Some(BaseSync {
4964 tip: tip.to_owned(),
4965 behind,
4966 attempts,
4967 conflict: None,
4968 already_in: Some(evidence),
4969 });
4970 self.state.event("land", message);
4971 self.state.save()?;
4972 self.settle_questions();
4973 Ok(true)
4974 }
4975
4976 fn landing_base(&self) -> String {
4986 self.state
4987 .base_sync
4988 .as_ref()
4989 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4990 }
4991
4992 pub async fn fix_selected(
5025 &mut self,
5026 ids: &[String],
5027 reason: &str,
5028 allow_stale: bool,
5029 ) -> Result<()> {
5030 let reason = reason.trim();
5031 if reason.is_empty() {
5032 bail!("a fix request needs a reason — that is the operator's own record of why");
5033 }
5034 if ids.is_empty() {
5035 bail!("no finding id given");
5036 }
5037 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5038 bail!(
5039 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5040 has already concluded — can be given a targeted fix. A run still \
5041 in progress should simply be resumed; a `merged` run's branch has \
5042 already landed, so its answer is a fresh `magi review <branch>`, \
5043 not reopening this run's own record",
5044 self.state.id,
5045 self.state.status.as_str()
5046 );
5047 }
5048 let Some(winner) = self.state.winner().cloned() else {
5049 bail!("run {} has no winning candidate to fix", self.state.id);
5050 };
5051 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5052 bail!(
5053 "branch `{}` no longer exists; this run cannot be extended",
5054 winner.branch
5055 );
5056 }
5057 let home = crate::run::home();
5058 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5059 bail!(
5060 "run {} is currently being worked on by another magi process",
5061 self.state.id
5062 );
5063 }
5064 let _claim = FixClaim::acquire(&self.state.dir())?;
5070
5071 let mut seen = BTreeSet::new();
5075 let mut findings = Vec::new();
5076 let mut missing = Vec::new();
5077 for id in ids {
5078 if !seen.insert(id.clone()) {
5079 continue;
5080 }
5081 match self.state.finding(id) {
5082 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5083 id: f.id.clone(),
5084 severity: f.severity,
5085 reviewer_vote: rec.vote,
5086 round: round.round,
5087 round_head: round.head.clone(),
5088 reviewer: rec.reviewer,
5089 agent: rec.agent.clone(),
5090 file: f.file.clone(),
5091 line: f.line,
5092 title: f.title.clone(),
5093 detail: f.detail.clone(),
5094 outcome: OperatorFixOutcome::Pending,
5095 }),
5096 None => missing.push(id.clone()),
5097 }
5098 }
5099 if !missing.is_empty() {
5100 bail!(
5101 "unknown finding id(s): {}; nothing was changed",
5102 missing.join(", ")
5103 );
5104 }
5105
5106 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5107 let stale_details: Vec<(String, String)> = findings
5108 .iter()
5109 .filter(|f| f.round_head != head_at_request)
5110 .map(|f| (f.id.clone(), f.round_head.clone()))
5111 .collect();
5112 let stale = !stale_details.is_empty();
5113 if stale && !allow_stale {
5114 bail!(
5115 "the branch has moved since some finding(s) were raised — {} — now \
5116 at {}; pass --allow-stale to fix anyway, or re-run review first",
5117 stale_details
5118 .iter()
5119 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5120 .collect::<Vec<_>>()
5121 .join(", "),
5122 short(&head_at_request)
5123 );
5124 }
5125
5126 let request = OperatorFixRequest {
5127 requested_at: Timestamp::now(),
5128 reason: reason.to_owned(),
5129 findings,
5130 head_at_request: head_at_request.clone(),
5131 allow_stale,
5132 stale,
5133 fix: None,
5134 result_head: None,
5135 follow_up_review_run: None,
5136 };
5137 self.state.event(
5138 "fix",
5139 format!(
5140 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5141 request.findings.len(),
5142 request
5143 .findings
5144 .iter()
5145 .map(|f| f.id.as_str())
5146 .collect::<Vec<_>>()
5147 .join(", "),
5148 ),
5149 );
5150 self.state.operator_fixes.push(request);
5157 self.state.save()?;
5158 let request_index = self.state.operator_fixes.len() - 1;
5159
5160 let repo = self.state.repo.clone();
5169 let coordination = self.coordinate(&repo).await?;
5170 if winner.worktree.exists() {
5171 let dirty = git::git(
5174 &winner.worktree,
5175 &["status", "--porcelain", "--untracked-files=all"],
5176 )
5177 .await?;
5178 let only_withheld = dirty.lines().all(|l| {
5179 l.strip_prefix("?? ")
5180 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5181 });
5182 if !only_withheld {
5183 bail!(
5184 "`{}` has uncommitted changes; refusing to touch it — commit or \
5185 discard them first",
5186 winner.worktree.display()
5187 );
5188 }
5189 git::worktree_remove(&self.state.repo, &winner.worktree)
5190 .await
5191 .ok();
5192 }
5193 let fix_worktree = self.state.worktree_root().join("operator-fix");
5194 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5195 git::git(
5196 &self.state.repo,
5197 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5198 )
5199 .await
5200 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5201 if !git::is_clean(&fix_worktree).await? {
5202 git::worktree_remove(&self.state.repo, &fix_worktree)
5203 .await
5204 .ok();
5205 bail!(
5206 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5207 winner.branch
5208 );
5209 }
5210 let repo_for_setup = self.state.repo.clone();
5211 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5212 Ok(()) => {
5213 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5214 }
5215 Err(e) => Err(e),
5216 };
5217 if let Err(e) = setup {
5218 git::worktree_remove(&self.state.repo, &fix_worktree)
5219 .await
5220 .ok();
5221 return Err(e);
5222 }
5223
5224 let run_id = self.state.id.clone();
5225 let prompts = self.state.config.prompts.clone();
5226 let language = self.state.config.graph.language.clone();
5227 let sessions = self.state.config.graph.sessions;
5228 let artifacts = agent::artifacts_dir(&self.state.dir());
5229 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5230 .findings
5231 .iter()
5232 .map(|f| Finding {
5233 id: f.id.clone(),
5234 severity: f.severity,
5235 file: f.file.clone(),
5236 line: f.line,
5237 title: f.title.clone(),
5238 detail: f.detail.clone(),
5239 })
5240 .collect();
5241 let fix_prompt = prompt::operator_fix_in(
5242 &self.state.instruction,
5243 &finding_list,
5244 reason,
5245 &stale_details,
5246 &head_at_request,
5247 &language,
5248 &self.state.config.graph.github_language,
5249 );
5250 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5251 drop(coordination);
5252 let (job, seat, out) = self
5253 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5254 prompt: fix_prompt.clone(),
5255 spec,
5256 seat,
5257 cwd: fix_worktree.clone(),
5258 timeout,
5259 allow_write: true,
5260 sessions,
5261 artifacts: artifacts.clone(),
5262 stem: "operator-fix".to_owned(),
5263 handover: None,
5264 })
5265 .await;
5266 let agent_id = seat.agent.clone();
5267
5268 let mut fix = FixRecord {
5269 agent: agent_id,
5270 addressed: Vec::new(),
5271 rejected: Vec::new(),
5272 notes: String::new(),
5273 committed: false,
5274 failed: None,
5275 duration_ms: 0,
5276 continuation: None,
5277 };
5278 let mut final_seat = seat.clone();
5279 match out {
5280 AgentOutcome::Ok(o) => {
5281 fix.duration_ms = o.duration_ms;
5282 let parsed = verdict::extract_json::<FixReport>(&o.text);
5283 let incomplete_reason = match &parsed {
5284 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5285 "the reply parsed, but it reported a command whose own CLI \
5286 never confirmed an exit status"
5287 .to_owned(),
5288 ),
5289 Ok(_) => None,
5290 Err(e) => Some(e.to_string()),
5291 };
5292 match incomplete_reason {
5293 None => {
5294 let report = parsed.expect("checked Ok above");
5295 fix.addressed = report.addressed;
5296 fix.rejected = report.rejected;
5297 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5298 }
5299 Some(reason) => {
5300 let (resumed_seat, resolved, failure, cont) = self
5301 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5302 .await;
5303 fix.duration_ms += cont.cumulative_wait_ms;
5304 fix.continuation = Some(cont);
5305 final_seat = resumed_seat;
5306 match resolved {
5307 Some(report) => {
5308 fix.addressed = report.addressed;
5309 fix.rejected = report.rejected;
5310 fix.notes =
5311 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5312 }
5313 None => fix.failed = failure,
5314 }
5315 }
5316 }
5317 }
5318 AgentOutcome::Dropped(o) => {
5319 fix.duration_ms = o.duration_ms;
5320 let why = o
5321 .dropped
5322 .as_ref()
5323 .map(|d| d.why.as_str())
5324 .unwrap_or("the CLI ended the stream without delivering its answer");
5325 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5326 }
5327 AgentOutcome::Quota(o) => {
5328 self.state.quota.push(QuotaLoss {
5329 seat: final_seat.key.clone(),
5330 node: "fix".to_owned(),
5331 at: Timestamp::now(),
5332 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5333 });
5334 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5335 }
5336 AgentOutcome::Failed(e) => fix.failed = Some(e),
5337 AgentOutcome::ResourceBlocked(e) => {
5338 fix.failed = Some(format!("the fixer was not started: {e}"));
5339 }
5340 }
5341 if fix.continuation.is_none() {
5342 fix.continuation = Some(ContinuationRecord::not_needed());
5343 }
5344 self.state.seats.insert(final_seat.key.clone(), final_seat);
5345
5346 let rescue_message = format!(
5347 "magi: operator-selected fix ({}) (uncommitted work)",
5348 self.state.operator_fixes[request_index]
5349 .findings
5350 .iter()
5351 .map(|f| f.id.as_str())
5352 .collect::<Vec<_>>()
5353 .join(", ")
5354 );
5355 let repo = self.state.repo.clone();
5356 let coordination = self.coordinate(&repo).await?;
5357 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5358 self.state.note_withheld("fix", &r.withheld);
5359 }
5360 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5361 fix.committed = after != head_at_request;
5362 git::worktree_remove(&self.state.repo, &fix_worktree)
5363 .await
5364 .ok();
5365 drop(coordination);
5366
5367 self.state.event(
5368 "fix",
5369 match &fix.failed {
5370 Some(reason) => format!(
5371 "operator fix: adoption report was lost ({reason}); {}",
5372 if fix.committed {
5373 "committed"
5374 } else {
5375 "NO new commit"
5376 }
5377 ),
5378 None => format!(
5379 "operator fix: {} addressed, {} rejected, {}",
5380 fix.addressed.len(),
5381 fix.rejected.len(),
5382 if fix.committed {
5383 "committed"
5384 } else {
5385 "NO new commit"
5386 }
5387 ),
5388 },
5389 );
5390
5391 for f in &mut self.state.operator_fixes[request_index].findings {
5398 f.outcome = if fix.failed.is_some() {
5399 OperatorFixOutcome::Unreported
5400 } else if fix.addressed.contains(&f.id) {
5401 OperatorFixOutcome::Addressed
5402 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5403 OperatorFixOutcome::Rejected { why: r.why.clone() }
5404 } else {
5405 OperatorFixOutcome::Unreported
5406 };
5407 }
5408
5409 let committed = fix.committed;
5410 if committed {
5411 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5412 }
5413 self.state.operator_fixes[request_index].fix = Some(fix);
5414 self.state.save()?;
5417
5418 if committed {
5419 self.state.event(
5420 "fix",
5421 format!(
5422 "operator fix committed {}; opening a follow-up review-only run",
5423 short(&after)
5424 ),
5425 );
5426 let origin =
5429 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5430 match Self::review(
5431 &self.state.repo,
5432 &winner.branch,
5433 self.state.config.clone(),
5434 origin,
5435 )
5436 .await
5437 {
5438 Ok(mut follow_up) => {
5439 follow_up.state.event(
5440 "start",
5441 format!(
5442 "requested by an operator fix on run {} for finding(s) {}",
5443 self.state.id,
5444 self.state.operator_fixes[request_index]
5445 .findings
5446 .iter()
5447 .map(|f| f.id.as_str())
5448 .collect::<Vec<_>>()
5449 .join(", "),
5450 ),
5451 );
5452 follow_up.state.save()?;
5453 let follow_up_id = follow_up.state.id.clone();
5454 let adopted = match crate::direct::adopt(
5457 &follow_up.state,
5458 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5459 ) {
5460 Ok(a) => a,
5461 Err(e) => {
5462 self.state.event(
5465 "fix",
5466 format!(
5467 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5468 ),
5469 );
5470 self.state.save()?;
5471 return Ok(());
5472 }
5473 };
5474 let executed = follow_up.execute().await;
5475 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5476 if let Some(a) = adopted {
5477 a.finish(&follow_up.state, executed);
5478 }
5479 if let Some(e) = failure {
5480 self.state.event(
5481 "fix",
5482 format!(
5483 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5484 ),
5485 );
5486 }
5487 self.state.operator_fixes[request_index].follow_up_review_run =
5488 Some(follow_up_id);
5489 }
5490 Err(e) => {
5491 self.state.event(
5492 "fix",
5493 format!("committed the fix but could not open a follow-up review: {e:#}"),
5494 );
5495 }
5496 }
5497 self.state.save()?;
5498 }
5499
5500 Ok(())
5501 }
5502
5503 async fn ask_fixer(
5523 &mut self,
5524 winner: &Candidate,
5525 node: &'static str,
5526 round: Option<usize>,
5527 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5528 ) -> (SeatJob, SeatState, AgentOutcome) {
5529 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5530 self.ask_fixer_among(attempts, node, round, None, build)
5531 .await
5532 }
5533
5534 async fn ask_fixer_among(
5539 &mut self,
5540 attempts: Vec<(AgentSpec, String)>,
5541 node: &'static str,
5542 round: Option<usize>,
5543 restore: Option<(&Path, &str)>,
5544 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5545 ) -> (SeatJob, SeatState, AgentOutcome) {
5546 let run_id = self.state.id.clone();
5547 let prompts = self.state.config.prompts.clone();
5548 let cache = self.state.config.cache_dir();
5549 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5550 let mut last = None;
5551 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5552 let seat = self.seat(&key, &spec.id);
5553 let mut job = build(spec.clone(), seat);
5554 if i > 0 {
5555 job.stem = format!("{}-{}", job.stem, spec.id);
5556 }
5557 let ctx = WaveCtx {
5558 carry_seats: false,
5559 pauses: Some((&self.pause, &self.interrupt)),
5560 run: &run_id,
5561 node,
5562 prompts: &prompts,
5563 cache: cache.as_deref(),
5564 round,
5565 };
5566 let (seat, out) =
5567 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5568 let advances = match &out {
5571 AgentOutcome::Ok(o) => agent::output_advances(o),
5572 AgentOutcome::ResourceBlocked(_) => false,
5573 _ => true,
5574 };
5575 if let Some(next) = ids.get(i + 1)
5576 && advances
5577 {
5578 self.state.seats.insert(seat.key.clone(), seat.clone());
5579 let class =
5580 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5581 let reason = match &out {
5582 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5583 other => fail_reason(other),
5584 };
5585 if let Some((wt, rev)) = restore
5589 && let Err(e) = self.graft_restore(wt, rev).await
5590 {
5591 let why =
5592 format!("could not restore the worktree before the next fixer: {e:#}");
5593 tracing::warn!("{why}");
5594 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5595 AgentOutcome::ResourceBlocked(why)
5596 } else {
5597 AgentOutcome::Failed(why)
5598 };
5599 last = Some((job, seat, out));
5600 break;
5601 }
5602 record_handover(
5603 &mut self.state,
5604 node,
5605 &seat.key,
5606 &spec.id,
5607 next,
5608 &class,
5609 &reason,
5610 );
5611 continue;
5612 }
5613 last = Some((job, seat, out));
5614 break;
5615 }
5616 last.expect("the fixer chain always has an entry")
5617 }
5618
5619 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5630 use crate::recovery::{Mode, Repair};
5631 if !self.state.reviews.is_empty() {
5632 return Ok(());
5633 }
5634 let Some(mut rec) = self.state.recovery.clone() else {
5635 return Ok(());
5636 };
5637 if rec.mode != Mode::VerifyRepair {
5638 return Ok(());
5639 }
5640 let wt = winner.worktree.clone();
5641 let mut head = git::rev_parse(&wt, "HEAD").await?;
5642 match rec.repair.clone() {
5643 Repair::Done { .. } => return Ok(()),
5644 Repair::WaitingForRestore { before } => {
5645 self.graft_restore(&wt, &before).await?;
5646 head = before;
5647 rec.repair = Repair::NotStarted;
5648 self.state.recovery = Some(rec.clone());
5649 self.state.save()?;
5650 }
5651 Repair::Started { before } => {
5652 if let Err(e) = self.graft_restore(&wt, &before).await {
5656 if e.is::<crate::repo_lock::ResourceBlocked>() {
5657 return Err(e);
5658 }
5659 return self
5660 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5661 .await;
5662 }
5663 rec.repair = Repair::Done {
5664 before: before.clone(),
5665 after: before,
5666 committed: false,
5667 failed: Some("interrupted; rolled back".to_owned()),
5668 };
5669 self.state
5670 .event("recovery", "the repair was interrupted and rolled back");
5671 self.state.recovery = Some(rec);
5672 self.state.save()?;
5673 return Ok(());
5674 }
5675 Repair::NotStarted => {}
5676 }
5677 if crate::recovery::recheck(&mut rec, &head) {
5678 self.state.event("recovery", rec.summary());
5679 self.state.recovery = Some(rec);
5680 self.state.save()?;
5681 return Ok(());
5682 }
5683 rec.repair = Repair::Started {
5684 before: head.clone(),
5685 };
5686 self.state.recovery = Some(rec.clone());
5687 self.state.save()?;
5688
5689 let language = self.state.config.graph.language.clone();
5690 let sessions = self.state.config.graph.sessions;
5691 let artifacts = agent::artifacts_dir(&self.state.dir());
5692 let fix_prompt = prompt::recovery_fix(
5693 &self.state.instruction,
5694 &rec,
5695 self.state.config.graph.review_rounds,
5696 &language,
5697 );
5698 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5699 self.state.event(
5700 "recovery",
5701 format!(
5702 "repair: the fixer starts from the failed verification on {}",
5703 short(&head)
5704 ),
5705 );
5706 let (_job, seat, out) = self
5707 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5708 prompt: fix_prompt.clone(),
5709 spec,
5710 seat,
5711 cwd: wt.clone(),
5712 timeout,
5713 allow_write: true,
5714 sessions,
5715 artifacts: artifacts.clone(),
5716 stem: "fix-recovery".to_owned(),
5717 handover: None,
5718 })
5719 .await;
5720 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5721 rec.repair = Repair::WaitingForRestore { before: head };
5722 self.state.recovery = Some(rec);
5723 return self.stop_for_resource();
5724 }
5725 let failed = match &out {
5726 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5727 AgentOutcome::Quota(o) => {
5728 self.state.quota.push(QuotaLoss {
5729 seat: seat.key.clone(),
5730 node: "fix".to_owned(),
5731 at: Timestamp::now(),
5732 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5733 });
5734 Some("rate limited (quota); the fixer could not run".to_owned())
5735 }
5736 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5737 AgentOutcome::Failed(e) => Some(e.clone()),
5738 };
5739 self.state.seats.insert(seat.key.clone(), seat);
5740 let coordination = self.coordinate(&wt).await?;
5741 let failed = match failed {
5742 Some(why) => {
5744 if let Err(e) = git::restore_to(&wt, &head).await {
5745 return self
5746 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5747 .await;
5748 }
5749 Some(why)
5750 }
5751 None => {
5752 if let Ok(r) = git::rescue_commit(
5753 &wt,
5754 "magi: recovery repair of the failed verification (uncommitted work)",
5755 )
5756 .await
5757 {
5758 self.state.note_withheld("fix", &r.withheld);
5759 }
5760 None
5761 }
5762 };
5763 let after = git::rev_parse(&wt, "HEAD").await?;
5764 drop(coordination);
5765 self.state.event(
5766 "recovery",
5767 match &failed {
5768 Some(why) => format!("repair: nothing kept ({why})"),
5769 None if after != head => {
5770 format!("repair: committed {}", short(&after))
5771 }
5772 None => "repair: the fixer made no commit".to_owned(),
5773 },
5774 );
5775 rec.repair = Repair::Done {
5776 before: head.clone(),
5777 committed: after != head,
5778 after,
5779 failed,
5780 };
5781 self.state.recovery = Some(rec);
5782 self.state.save()?;
5783 Ok(())
5784 }
5785
5786 async fn stop_recovery(
5789 &mut self,
5790 mut rec: crate::recovery::RecoveryRecord,
5791 head: &str,
5792 why: String,
5793 ) -> Result<()> {
5794 rec.repair = crate::recovery::Repair::Started {
5796 before: head.to_owned(),
5797 };
5798 self.state.recovery = Some(rec);
5799 self.state
5800 .event("recovery", format!("repair could not be undone: {why}"));
5801 self.state.status = RunStatus::Blocked;
5802 self.state.save()?;
5803 bail!("recovery repair could not be rolled back: {why}")
5804 }
5805
5806 async fn review_loop(&mut self) -> Result<()> {
5807 if self
5812 .state
5813 .base_sync
5814 .as_ref()
5815 .is_some_and(|s| s.conflict.is_some())
5816 {
5817 return Ok(());
5818 }
5819 let run_id = self.state.id.clone();
5824 let prompts = self.state.config.prompts.clone();
5825 let Some(winner) = self.state.winner().cloned() else {
5826 return Ok(());
5827 };
5828 let max_rounds = self.state.config.graph.review_rounds;
5829 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5839 if status == RunStatus::Gating {
5842 self.record_contested_handoff();
5843 }
5844 self.state.status = status;
5845 self.state.save()?;
5846 return Ok(());
5847 }
5848 self.state.status = RunStatus::Reviewing;
5849 if self
5859 .state
5860 .reviews
5861 .last()
5862 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5863 {
5864 let shell = self.state.config.shell();
5865 return self
5866 .stop_reviewing(
5867 "the last round's own verification never resolved",
5868 &shell,
5869 &winner.worktree,
5870 )
5871 .await;
5872 }
5873
5874 self.recovery_repair(&winner).await?;
5875 if self.resource_halt()? {
5876 return Ok(());
5877 }
5878
5879 let repo = self.state.repo.clone();
5880 let root = self.state.worktree_root();
5881 let language = self.state.config.graph.language.clone();
5882 let sessions = self.state.config.graph.sessions;
5883 let artifacts = self.merge_repair_artifacts();
5884 let base = self.landing_base();
5885 let base_short = short(&base);
5886 let reviewers = self.roles.reviewers.clone();
5887 let shell = self.state.config.shell();
5888
5889 for round in (self.state.reviews.len() + 1)..=max_rounds {
5890 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5891 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5892 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5893 let prev_verification = self
5902 .state
5903 .reviews
5904 .last()
5905 .and_then(|r| r.verification_summary(&head));
5906 let prior_section = match (&self.state.recovery, round) {
5909 (Some(rec), 1) if self.state.reviews.is_empty() => {
5910 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5911 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5912 Some(crate::recovery::head_of(
5913 &git::diff(&winner.worktree, reviewed, "HEAD")
5914 .await
5915 .unwrap_or_default(),
5916 crate::recovery::SINCE_DIFF_MAX,
5917 ))
5918 }
5919 _ => None,
5920 };
5921 Some(prompt::recovery_review(rec, since.as_deref()))
5922 }
5923 _ => None,
5924 };
5925
5926 let mut jobs = Vec::new();
5930 let coordination = self.coordinate(&repo).await?;
5931 for (r, spec) in reviewers.iter().cloned().enumerate() {
5932 let wt = root.join(format!("review-{}", r + 1));
5933 if wt.exists() {
5936 git::reset_detached(&wt, &head).await?;
5937 } else {
5938 git::worktree_add_detached(&repo, &wt, &head).await?;
5939 }
5940 self.setup_seat_worktree(&repo, &wt, false).await?;
5941 let seat_key = format!("review-{}", r + 1);
5942 let spec = pick_start_spec(
5945 &self.roles.reviewer_roster,
5946 spec,
5947 self.state.seat_history.get(&seat_key),
5948 );
5949 let seat = self.seat(&seat_key, &spec.id);
5950 jobs.push(SeatJob {
5951 prompt: prompt::review(&prompt::ReviewCtx {
5952 instruction: &self.state.instruction,
5953 branch: &winner.branch,
5954 base_short: &base_short,
5955 stat: &stat,
5956 patch: &patch,
5957 verification: prev_verification.as_ref(),
5958 prior: prior_section.as_deref(),
5959 reviewers: reviewers.len(),
5960 round,
5961 rounds: max_rounds,
5962 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5965 lens: Lens::for_seat(r),
5966 language: &language,
5967 github_language: &self.state.config.graph.github_language,
5968 }),
5969 spec,
5970 seat,
5971 cwd: wt,
5972 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5973 allow_write: false,
5974 sessions,
5975 artifacts: artifacts.clone(),
5976 stem: format!("review-{round}-{}", r + 1),
5977 handover: None,
5978 });
5979 }
5980
5981 drop(coordination);
5982 self.state.event(
5983 "review",
5984 format!(
5985 "round {round}: {} reviewers on {}",
5986 jobs.len(),
5987 short(&head)
5988 ),
5989 );
5990 let mut quota_losses = Vec::new();
5991 let review_retries = self.state.config.graph.retries;
5992 let review_cache = self.state.config.cache_dir();
5993 let ctx = WaveCtx {
5994 carry_seats: true,
5995 pauses: Some((&self.pause, &self.interrupt)),
5996 run: &run_id,
5997 node: "review",
5998 prompts: &prompts,
5999 cache: review_cache.as_deref(),
6000 round: Some(round),
6001 };
6002 let results = ask_json_wave::<Review>(
6003 jobs,
6004 Arc::clone(&self.sem),
6005 review_retries,
6006 &self.roles.reviewer_roster,
6007 &ctx,
6008 &mut quota_losses,
6009 &mut self.state,
6010 &|_: &Review| Ok(()),
6011 )
6012 .await;
6013 if self.state.resource_block.is_some() {
6014 return self.stop_for_resource();
6015 }
6016 let round_quota_missing = quota_losses.len();
6020 self.state.quota.extend(quota_losses);
6021
6022 let mut records = Vec::new();
6023 let mut all_findings = Vec::new();
6024 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6025 let agent_id = seat.agent.clone();
6026 self.state.seats.insert(seat.key.clone(), seat);
6027 let mut record = ReviewRecord {
6028 reviewer: r + 1,
6029 agent: agent_id,
6030 summary: String::new(),
6031 findings: Vec::new(),
6032 vote: None,
6033 failed: None,
6034 duration_ms: 0,
6035 attempts,
6041 };
6042 match res {
6043 Ok((review, out)) => {
6044 record.summary =
6052 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6053 record.vote = Some(review.vote);
6054 record.duration_ms = out.duration_ms;
6055 for (n, mut f) in review.findings.into_iter().enumerate() {
6056 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6059 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6060 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6061 f.file = f
6067 .file
6068 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6069 all_findings.push(f.clone());
6070 record.findings.push(f);
6071 }
6072 self.state.event(
6073 "review",
6074 format!(
6075 "round {round}: reviewer {} voted {} with {} finding(s)",
6076 r + 1,
6077 review.vote.label(),
6078 record.findings.len()
6079 ),
6080 );
6081 }
6082 Err(e) => {
6083 record.failed = Some(e.to_string());
6084 self.state.event(
6085 "review",
6086 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6087 );
6088 }
6089 }
6090 records.push(record);
6091 }
6092
6093 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6100 let vote_split =
6101 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6102 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6103 if vote_split {
6104 self.state.event(
6105 "review",
6106 format!(
6107 "round {round}: votes split ({}) — one round of reconsideration",
6108 initial_votes
6109 .iter()
6110 .map(|v| v.label())
6111 .collect::<Vec<_>>()
6112 .join(", ")
6113 ),
6114 );
6115 let panel: Vec<ReviewSeatReport<'_>> = records
6118 .iter()
6119 .filter_map(|r| {
6120 r.vote.map(|vote| ReviewSeatReport {
6121 reviewer: r.reviewer,
6122 vote,
6123 summary: &r.summary,
6124 findings: &r.findings,
6125 })
6126 })
6127 .collect();
6128
6129 let mut jobs = Vec::new();
6130 let mut seats_at = Vec::new();
6131 for (r, spec) in reviewers.iter().cloned().enumerate() {
6132 if records[r].vote.is_none() {
6136 continue;
6137 }
6138 let wt = root.join(format!("review-{}", r + 1));
6139 let seat_key = format!("review-{}", r + 1);
6140 let spec = self.occupant(&seat_key, spec);
6141 let seat = self.seat(&seat_key, &spec.id);
6142 let build = |with_patch: bool| {
6150 prompt::review_reconsider(&ReviewReconsiderCtx {
6151 instruction: &self.state.instruction,
6152 reviewer: r + 1,
6153 lens: Lens::for_seat(r),
6154 panel: &panel,
6155 patch: with_patch.then_some(ReviewPatch {
6156 branch: &winner.branch,
6157 base_short: &base_short,
6158 stat: &stat,
6159 patch: &patch,
6160 }),
6161 round,
6162 rounds: max_rounds,
6163 language: &language,
6164 })
6165 };
6166 let full = build(true);
6167 let prompt = if has_context(&spec, &seat, sessions) {
6168 build(false)
6169 } else {
6170 full.clone()
6171 };
6172 jobs.push(SeatJob {
6173 prompt,
6174 spec,
6175 seat,
6176 cwd: wt,
6177 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6178 allow_write: false,
6179 sessions,
6180 artifacts: artifacts.clone(),
6181 stem: format!("review-{round}-reconsider-{}", r + 1),
6182 handover: Some(full),
6183 });
6184 seats_at.push(r);
6185 }
6186
6187 let mut recon_quota_losses = Vec::new();
6188 let recon_cache = self.state.config.cache_dir();
6189 let recon_ctx = WaveCtx {
6190 carry_seats: true,
6191 pauses: Some((&self.pause, &self.interrupt)),
6192 run: &run_id,
6193 node: "review",
6194 prompts: &prompts,
6195 cache: recon_cache.as_deref(),
6196 round: Some(round),
6197 };
6198 let recon_results = ask_json_wave::<ReviewRevote>(
6199 jobs,
6200 Arc::clone(&self.sem),
6201 review_retries,
6202 &self.roles.reviewer_roster,
6203 &recon_ctx,
6204 &mut recon_quota_losses,
6205 &mut self.state,
6206 &|_: &ReviewRevote| Ok(()),
6207 )
6208 .await;
6209 if self.state.resource_block.is_some() {
6210 return self.stop_for_resource();
6211 }
6212 self.state.quota.extend(recon_quota_losses);
6213
6214 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6215 let agent_id = seat.agent.clone();
6216 self.state.seats.insert(seat.key.clone(), seat);
6217 let mut rec = ReviewRevoteRecord {
6218 reviewer: r + 1,
6219 agent: agent_id,
6220 vote: None,
6221 reason: String::new(),
6222 failed: None,
6223 };
6224 match res {
6225 Ok((rv, _)) => {
6226 rec.vote = Some(rv.vote);
6227 rec.reason =
6228 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6229 self.state.event(
6230 "review",
6231 format!(
6232 "round {round}: reviewer {} revoted {}",
6233 r + 1,
6234 rv.vote.label()
6235 ),
6236 );
6237 }
6238 Err(e) => {
6239 rec.failed = Some(e.to_string());
6240 self.state.event(
6241 "review",
6242 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6243 );
6244 }
6245 }
6246 reconsideration.push(rec);
6247 }
6248 } else if initial_votes.len() > 1 {
6249 self.state.event(
6250 "review",
6251 format!(
6252 "round {round}: votes agreed ({}) — no reconsideration",
6253 initial_votes[0].label()
6254 ),
6255 );
6256 }
6257
6258 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6259 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6260 let defer_e2e =
6271 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6272 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6273 let reason =
6274 format!("{blocking} blocking finding(s) already required a fix this round");
6275 self.state.event(
6276 "verify",
6277 format!(
6278 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6279 {}); it will run once a round has none left",
6280 short(&head)
6281 ),
6282 );
6283 (Vec::new(), false, true, Some(reason))
6284 } else {
6285 let e2e_commands = self.state.config.verify.e2e.clone();
6286 let cache_dir = self.state.config.cache_dir();
6287 let context = format!("round {round}");
6288 let wait = self.repository_wait();
6289 let (e2e, verify_retried) = with_cache_lease(
6290 &mut self.state,
6291 cache_dir.as_deref(),
6292 "e2e",
6293 "e2e",
6294 &winner.worktree,
6295 &head,
6296 verify_timeout,
6297 &context,
6298 &wait,
6299 |state, budget| {
6300 let shell = shell.clone();
6301 let e2e_commands = e2e_commands.clone();
6302 let worktree = winner.worktree.clone();
6303 let context = context.clone();
6304 async move {
6305 run_e2e_with_retry(
6306 state,
6307 &shell,
6308 &e2e_commands,
6309 &worktree,
6310 budget,
6311 &context,
6312 )
6313 .await
6314 }
6315 },
6316 )
6317 .await?;
6318 (e2e, verify_retried, false, None)
6319 };
6320
6321 let expected = records.len();
6322 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6323 let incomplete = answered < expected;
6324 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6325 let policy = self.state.config.graph.incomplete_review;
6326 let clean = round_is_clean(
6327 blocking,
6328 e2e_ok,
6329 answered,
6330 expected,
6331 round_quota_missing,
6332 policy,
6333 );
6334
6335 let mut round_record = ReviewRound {
6336 round,
6337 head: head.clone(),
6338 verified_head: None,
6339 verified_at: None,
6340 reviews: records,
6341 e2e,
6342 verify_retried,
6343 e2e_deferred,
6344 e2e_defer_reason,
6345 fix: None,
6346 blocking,
6347 answered,
6348 expected,
6349 clean,
6350 progressed: false,
6351 vote_split,
6352 reconsideration,
6353 verdict: None,
6354 };
6355 round_record.verdict = ReviewVote::worst(
6359 round_record
6360 .final_votes()
6361 .into_iter()
6362 .map(|(_, _, vote)| vote),
6363 );
6364 if !matches!(
6375 round_record.e2e_status(),
6376 E2eStatus::Deferred | E2eStatus::NotConfigured
6377 ) {
6378 round_record.verified_head = Some(head.clone());
6379 round_record.verified_at = Some(Timestamp::now());
6380 }
6381 let this_round_verification = round_record.verification_summary(&head);
6382
6383 if incomplete {
6384 let missing: Vec<String> = round_record
6385 .reviews
6386 .iter()
6387 .filter(|r| r.failed.is_some())
6388 .map(|r| format!("review-{}", r.reviewer))
6389 .collect();
6390 self.state.event(
6391 "review",
6392 format!(
6393 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6394 missing.join(", ")
6395 ),
6396 );
6397 }
6398
6399 if clean {
6400 self.state.event(
6401 "review",
6402 if incomplete && policy == IncompleteReviewPolicy::Warn {
6403 format!(
6404 "round {round}: clean (warn policy, incomplete panel) — no \
6405 blocking findings from the seats that answered, verification green"
6406 )
6407 } else if incomplete {
6408 format!(
6409 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6410 quorum) — no blocking findings from the seats that answered, \
6411 verification green",
6412 expected - answered
6413 )
6414 } else {
6415 format!("round {round}: clean — no blocking findings, verification green")
6416 },
6417 );
6418 self.state.reviews.push(round_record);
6419 self.state.status = RunStatus::Gating;
6420 self.state.save()?;
6421 return Ok(());
6422 }
6423
6424 if incomplete && blocking == 0 && e2e_ok {
6432 self.state.reviews.push(round_record);
6433 self.state.save()?;
6434 if round == max_rounds {
6435 self.state.status = RunStatus::Blocked;
6436 self.state.event(
6437 "review",
6438 format!(
6439 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6440 refusing to call it clean",
6441 expected - answered
6442 ),
6443 );
6444 return Ok(());
6445 }
6446 continue;
6447 }
6448
6449 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6461 self.state.reviews.push(round_record);
6462 return self
6463 .stop_reviewing(
6464 "the round's own verification could not run",
6465 &shell,
6466 &winner.worktree,
6467 )
6468 .await;
6469 }
6470
6471 if round == max_rounds {
6472 self.state.reviews.push(round_record);
6473 return self
6474 .stop_reviewing(
6475 &format!(
6476 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6477 ),
6478 &shell,
6479 &winner.worktree,
6480 )
6481 .await;
6482 }
6483
6484 let blocking_findings: Vec<_> = all_findings
6487 .iter()
6488 .filter(|f| f.severity.blocks())
6489 .cloned()
6490 .collect();
6491 let fix_prompt = prompt::fix_in(
6492 &self.state.instruction,
6493 &blocking_findings,
6494 this_round_verification.as_ref(),
6495 round,
6496 max_rounds,
6497 &language,
6498 &self.state.config.graph.github_language,
6499 );
6500 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6501 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6502 let (job, seat, out) = self
6503 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6504 prompt: fix_prompt.clone(),
6505 spec,
6506 seat,
6507 cwd: winner.worktree.clone(),
6508 timeout,
6509 allow_write: true,
6510 sessions,
6511 artifacts: artifacts.clone(),
6512 stem: format!("fix-{round}"),
6513 handover: None,
6514 })
6515 .await;
6516 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6519 return self.stop_for_resource();
6520 }
6521 let agent_id = seat.agent.clone();
6522
6523 let mut fix = FixRecord {
6524 agent: agent_id,
6525 addressed: Vec::new(),
6526 rejected: Vec::new(),
6527 notes: String::new(),
6528 committed: false,
6529 failed: None,
6530 duration_ms: 0,
6531 continuation: None,
6532 };
6533 let mut continuation = ContinuationRecord::not_needed();
6534 let mut final_seat = seat.clone();
6535 match out {
6536 AgentOutcome::Ok(o) => {
6537 fix.duration_ms = o.duration_ms;
6538 let parsed = verdict::extract_json::<FixReport>(&o.text);
6539 let incomplete_reason = match &parsed {
6546 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6547 "the reply parsed, but it reported a command whose own CLI never \
6548 confirmed an exit status"
6549 .to_owned(),
6550 ),
6551 Ok(_) => None,
6552 Err(e) => Some(e.to_string()),
6553 };
6554 match incomplete_reason {
6555 None => {
6556 let report = parsed.expect("checked Ok above");
6557 fix.addressed = report.addressed;
6558 fix.rejected = report.rejected;
6559 fix.notes =
6560 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6561 }
6562 Some(reason) => {
6563 let (resumed_seat, resolved, failure, cont) = self
6564 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6565 .await;
6566 fix.duration_ms += cont.cumulative_wait_ms;
6567 continuation = cont;
6568 final_seat = resumed_seat;
6569 match resolved {
6570 Some(report) => {
6571 fix.addressed = report.addressed;
6572 fix.rejected = report.rejected;
6573 fix.notes = blind::sanitize_prose(
6574 &report.notes,
6575 &self.state.config.blind,
6576 );
6577 }
6578 None => fix.failed = failure,
6579 }
6580 }
6581 }
6582 }
6583 AgentOutcome::Dropped(o) => {
6585 fix.duration_ms = o.duration_ms;
6586 let why = o
6587 .dropped
6588 .as_ref()
6589 .map(|d| d.why.as_str())
6590 .unwrap_or("the CLI ended the stream without delivering its answer");
6591 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6592 }
6593 AgentOutcome::Quota(o) => {
6594 self.state.quota.push(QuotaLoss {
6595 seat: final_seat.key.clone(),
6596 node: "fix".to_owned(),
6597 at: Timestamp::now(),
6598 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6599 });
6600 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6601 }
6602 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6603 }
6604 fix.continuation = Some(continuation);
6605 self.state.seats.insert(final_seat.key.clone(), final_seat);
6606 let coordination = self.coordinate(&winner.worktree).await?;
6607 if let Ok(r) = git::rescue_commit(
6608 &winner.worktree,
6609 &format!("magi: review round {round} fixes (uncommitted work)"),
6610 )
6611 .await
6612 {
6613 self.state.note_withheld("fix", &r.withheld);
6614 }
6615 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6616 fix.committed = after != before;
6617 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6625 drop(coordination);
6626 let progressed = diff_after != patch;
6627 let commit_note = if fix.committed {
6628 "committed"
6629 } else {
6630 "NO new commit"
6631 };
6632 let tree_note = if progressed {
6633 "changed vs base"
6634 } else {
6635 "unchanged vs base"
6636 };
6637 self.state.event(
6638 "fix",
6639 match &fix.failed {
6640 Some(reason) => {
6646 format!(
6647 "round {round}: fixer's adoption report was lost ({reason}); \
6648 {commit_note}, tree {tree_note}"
6649 )
6650 }
6651 None => format!(
6652 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6653 {tree_note}{}",
6654 fix.addressed.len(),
6655 fix.rejected.len(),
6656 if continuation.outcome == ContinuationOutcome::Resumed {
6657 format!(
6658 " (adoption report recovered after {} continuation(s))",
6659 continuation.attempts
6660 )
6661 } else {
6662 String::new()
6663 },
6664 ),
6665 },
6666 );
6667 round_record.fix = Some(fix);
6668 round_record.progressed = progressed;
6669 self.state.reviews.push(round_record);
6670 self.state.save()?;
6671
6672 if matches!(
6685 continuation.outcome,
6686 ContinuationOutcome::Exhausted
6687 | ContinuationOutcome::QuotaLost
6688 | ContinuationOutcome::NoSession
6689 ) {
6690 return self
6691 .stop_reviewing(
6692 "the fixer's adoption report never came back, even after resuming its \
6693 own seat; refusing to start another round against the same worktree \
6694 while that is unresolved",
6695 &shell,
6696 &winner.worktree,
6697 )
6698 .await;
6699 }
6700
6701 let streak = self
6702 .state
6703 .reviews
6704 .iter()
6705 .rev()
6706 .take_while(|r| !r.progressed)
6707 .count();
6708 if streak >= STAGNANT_LIMIT {
6709 return self
6710 .stop_reviewing(
6711 &format!(
6712 "the tree has not moved against base for {streak} round(s) in a row"
6713 ),
6714 &shell,
6715 &winner.worktree,
6716 )
6717 .await;
6718 }
6719 }
6720 Ok(())
6721 }
6722
6723 fn record_contested_handoff(&mut self) {
6757 if self.state.contested_handoff.is_some() {
6758 return;
6759 }
6760 let Some(contested) = self
6761 .state
6762 .reviews
6763 .last()
6764 .and_then(ReviewRound::contested_handoff)
6765 else {
6766 return;
6767 };
6768 self.state.event(
6769 "review",
6770 format!(
6771 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6772 wait for the owner's approval",
6773 contested.findings.len(),
6774 contested.rejecters.len()
6775 ),
6776 );
6777 self.state.contested_handoff = Some(contested);
6778 }
6779
6780 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6781 let round_idx = self.state.reviews.len() - 1;
6782 let needs_catchup_run = matches!(
6790 self.state.reviews[round_idx].e2e_status(),
6791 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6792 );
6793 if needs_catchup_run {
6794 let round = self.state.reviews[round_idx].round;
6795 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6796 let commands = self.state.config.verify.e2e.clone();
6797 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6798 let cache_dir = self.state.config.cache_dir();
6799 let context = format!(
6800 "round {round}: verification unresolved, catching up before the final decision"
6801 );
6802 let wait = self.repository_wait();
6803 let (outcomes, verify_retried) = with_cache_lease(
6804 &mut self.state,
6805 cache_dir.as_deref(),
6806 "e2e",
6807 "e2e",
6808 worktree,
6809 &attempted_head,
6810 timeout,
6811 &context,
6812 &wait,
6813 |state, budget| {
6814 let shell = shell.to_vec();
6815 let commands = commands.clone();
6816 let context = context.clone();
6817 async move {
6818 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6819 .await
6820 }
6821 },
6822 )
6823 .await?;
6824 let last = &mut self.state.reviews[round_idx];
6825 last.e2e = outcomes;
6826 last.verify_retried = verify_retried;
6827 last.verified_head = Some(attempted_head);
6834 last.verified_at = Some(Timestamp::now());
6835 if verify_inconclusive(&last.e2e) {
6836 self.state.save()?;
6843 return Ok(());
6844 }
6845 last.e2e_deferred = false;
6846 }
6847 let last = &self.state.reviews[round_idx];
6848 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6849
6850 match last.e2e_status() {
6851 E2eStatus::Failed => {
6852 let red: Vec<String> = last
6853 .e2e
6854 .iter()
6855 .filter(|o| !o.ok())
6856 .map(|o| {
6857 format!(
6858 "`{}` -> {:?}\n{}",
6859 o.command,
6860 o.code,
6861 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6862 )
6863 })
6864 .collect();
6865 self.state
6866 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6867 self.state.status = RunStatus::Blocked;
6868 }
6869 E2eStatus::ResourceBlocked => {
6874 self.state.event(
6875 "review",
6876 format!(
6877 "{why}; e2e could not run (shared build cache unavailable); not \
6878 deciding yet"
6879 ),
6880 );
6881 }
6882 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6883 self.state.event(
6884 "review",
6885 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6886 );
6887 self.record_contested_handoff();
6888 self.state.status = RunStatus::Gating;
6889 }
6890 }
6891 self.state.save()?;
6892 Ok(())
6893 }
6894
6895 async fn gate(&mut self) -> Result<()> {
6898 if self.state.status == RunStatus::Failed
6910 || self
6911 .state
6912 .base_sync
6913 .as_ref()
6914 .is_some_and(|s| s.conflict.is_some())
6915 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6916 != Some(RunStatus::Gating)
6917 {
6918 return Ok(());
6919 }
6920 if self.state.gate_ran {
6921 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6932 self.state.status = RunStatus::Blocked;
6933 self.state.save()?;
6934 }
6935 return Ok(());
6936 }
6937 let Some(winner) = self.state.winner().cloned() else {
6938 return Ok(());
6939 };
6940 self.state.status = RunStatus::Gating;
6941 let mut outcomes = self.run_gate(&winner).await?;
6942 loop {
6943 if verify_inconclusive(&outcomes) {
6954 self.state.save()?;
6955 return Ok(());
6956 }
6957 if outcomes.iter().all(CommandOutcome::ok) {
6958 break;
6959 }
6960 match self.gate_fix_round(&winner, &outcomes).await? {
6961 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6962 GateFix::Stop => break,
6963 GateFix::Defer => {
6964 self.state.save()?;
6965 return Ok(());
6966 }
6967 }
6968 }
6969 let passed = outcomes.iter().all(CommandOutcome::ok);
6970 self.state.gate = outcomes;
6971 self.state.gate_ran = true;
6972 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6973 if !passed {
6974 self.state.status = RunStatus::Blocked;
6975 let spent = self.state.gate_fixes.len();
6976 self.state.event(
6977 "gate",
6978 if spent == 0 {
6979 "gate failed; not merging".to_owned()
6980 } else {
6981 format!("gate failed after {spent} gate-fix round(s); not merging")
6982 },
6983 );
6984 }
6985 self.state.save()?;
6986 Ok(())
6987 }
6988
6989 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6999 let commands = state.config.verify.pre_gate.clone();
7000 if commands.is_empty() {
7001 return;
7002 }
7003 let shell = state.config.shell();
7004 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7005 let (outcomes, _) = run_commands(
7006 state,
7007 "pre_gate",
7008 "pre_gate",
7009 0,
7010 &shell,
7011 &commands,
7012 &winner.worktree,
7013 timeout,
7014 )
7015 .await;
7016 for o in &outcomes {
7017 if !o.ok() {
7018 tracing::warn!(
7019 "pre_gate `{}` failed ({:?}); the gate decides",
7020 o.command,
7021 o.code
7022 );
7023 }
7024 state.event(
7025 "pre_gate",
7026 format!(
7027 "`{}` -> {}",
7028 o.command,
7029 if o.ok() {
7030 "pass".to_owned()
7031 } else {
7032 format!(
7033 "FAIL ({:?})\n{}",
7034 o.code,
7035 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7036 )
7037 }
7038 ),
7039 );
7040 }
7041 state.pre_gate = outcomes;
7042 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7043 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7044 Ok(head) => {
7045 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7046 state.pre_gate_commit = Some(head);
7047 }
7048 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7049 },
7050 Ok(false) => {}
7051 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7052 }
7053 if let Err(e) = state.save() {
7054 tracing::warn!("could not persist the pre_gate record: {e:#}");
7055 }
7056 }
7057
7058 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7061 let shell = self.state.config.shell();
7062 let gate_commands = self.state.config.verify.gate.clone();
7063 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7064 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7065 Vec::new()
7066 } else {
7067 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7068 let cache_dir = self.state.config.cache_dir();
7069 let wait = self.repository_wait();
7070 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7071 let (outcomes, _) = with_cache_lease(
7072 &mut self.state,
7073 cache_dir.as_deref(),
7074 "gate",
7075 "gate",
7076 &winner.worktree,
7077 &head,
7078 timeout,
7079 "final gate",
7080 &wait,
7081 |state, budget| {
7082 let shell = shell.clone();
7083 let gate_commands = gate_commands.clone();
7084 let winner = winner.clone();
7085 async move {
7086 Self::run_pre_gate(state, &winner).await;
7087 let (outcomes, timed_out_pids) = run_commands(
7088 state,
7089 "gate",
7090 "gate",
7091 0,
7092 &shell,
7093 &gate_commands,
7094 &winner.worktree,
7095 budget,
7096 )
7097 .await;
7098 (outcomes, false, timed_out_pids)
7099 }
7100 },
7101 )
7102 .await?;
7103 outcomes
7104 };
7105 if outcomes.is_empty() {
7106 self.state
7111 .event("gate", "no gate commands configured; not tested");
7112 }
7113 for o in &outcomes {
7114 self.state.event(
7115 "gate",
7116 format!(
7117 "`{}` -> {}",
7118 o.command,
7119 if o.ok() {
7120 "pass".to_owned()
7121 } else {
7122 format!(
7123 "FAIL ({:?})\n{}",
7124 o.code,
7125 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7126 )
7127 }
7128 ),
7129 );
7130 }
7131 Ok(outcomes)
7132 }
7133
7134 async fn gate_fix_round(
7147 &mut self,
7148 winner: &Candidate,
7149 outcomes: &[CommandOutcome],
7150 ) -> Result<GateFix> {
7151 let cap = self.state.config.graph.gate_fix_rounds;
7152 let spent = self.state.gate_fixes.len();
7153 if spent >= cap {
7154 if cap > 0 {
7155 self.state.event(
7156 "gate",
7157 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7158 );
7159 }
7160 return Ok(GateFix::Stop);
7161 }
7162 if !gate_fixable(outcomes) {
7163 self.state.event(
7164 "gate",
7165 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7166 command or similar); not spending a fix round on it",
7167 );
7168 return Ok(GateFix::Stop);
7169 }
7170 let min_free = self.state.config.disk.min_free_bytes;
7171 if min_free > 0 {
7172 match crate::disk::free_bytes(&winner.worktree) {
7173 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7174 Ok(free) => {
7175 self.state.event(
7176 "gate",
7177 format!(
7178 "only {free} bytes free ({min_free} required by `[disk] \
7179 min_free_bytes`); not spending a fix round on a failure the disk \
7180 may explain"
7181 ),
7182 );
7183 return Ok(GateFix::Stop);
7184 }
7185 Err(e) => {
7186 self.state.event(
7187 "gate",
7188 format!("free disk space could not be measured ({e:#}); no fix round"),
7189 );
7190 return Ok(GateFix::Stop);
7191 }
7192 }
7193 }
7194
7195 let attempt = spent + 1;
7196 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7197 let base = self.landing_base();
7198 let fix_prompt = prompt::gate_fix_in(
7199 &self.state.instruction,
7200 &failed,
7201 attempt,
7202 cap,
7203 &self.state.config.graph.language,
7204 &self.state.config.graph.github_language,
7205 );
7206 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7207 let sessions = self.state.config.graph.sessions;
7208 let artifacts = self.merge_repair_artifacts();
7209 self.state.event(
7210 "gate",
7211 format!("gate failed; gate-fix round {attempt} of {cap}"),
7212 );
7213 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7214 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7215 let (_, seat, out) = self
7216 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7217 prompt: fix_prompt.clone(),
7218 spec,
7219 seat,
7220 cwd: winner.worktree.clone(),
7221 timeout,
7222 allow_write: true,
7223 sessions,
7224 artifacts: artifacts.clone(),
7225 stem: format!("gate-fix-{attempt}"),
7226 handover: None,
7227 })
7228 .await;
7229 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7232 return Ok(GateFix::Defer);
7233 }
7234 let mut record = GateFixRecord {
7235 agent: seat.agent.clone(),
7236 failed,
7237 notes: String::new(),
7238 committed: false,
7239 error: None,
7240 };
7241 match out {
7242 AgentOutcome::Ok(o) => {
7243 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7246 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7247 }
7248 }
7249 AgentOutcome::Dropped(_) => {
7250 record.error = Some("the CLI dropped the stream".to_owned());
7251 }
7252 AgentOutcome::Quota(o) => {
7253 self.state.quota.push(QuotaLoss {
7254 seat: seat.key.clone(),
7255 node: "gate-fix".to_owned(),
7256 at: Timestamp::now(),
7257 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7258 });
7259 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7260 }
7261 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7262 }
7263 self.state.seats.insert(seat.key.clone(), seat);
7264 let coordination = self.coordinate(&winner.worktree).await?;
7265 if let Ok(r) = git::rescue_commit(
7266 &winner.worktree,
7267 &format!("magi: gate fix {attempt} (uncommitted work)"),
7268 )
7269 .await
7270 {
7271 self.state.note_withheld("gate-fix", &r.withheld);
7272 }
7273 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7274 record.committed = after != before;
7275 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7276 let note = record.error.clone();
7277 self.state.gate_fixes.push(record);
7278 self.state.save()?;
7279 if !changed {
7280 self.state.event(
7281 "gate",
7282 match note {
7283 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7284 None => format!("gate-fix round {attempt}: the tree did not change"),
7285 },
7286 );
7287 return Ok(GateFix::Stop);
7288 }
7289 self.state.event(
7290 "gate",
7291 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7292 );
7293
7294 drop(coordination);
7295 let commands = self.state.config.verify.e2e.clone();
7296 if !commands.is_empty() {
7297 let shell = self.state.config.shell();
7298 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7299 let cache_dir = self.state.config.cache_dir();
7300 let context = format!("gate-fix round {attempt}");
7301 let wait = self.repository_wait();
7302 let (e2e, _) = with_cache_lease(
7303 &mut self.state,
7304 cache_dir.as_deref(),
7305 "e2e",
7306 "e2e",
7307 &winner.worktree,
7308 &after,
7309 timeout,
7310 &context,
7311 &wait,
7312 |state, budget| {
7313 let shell = shell.clone();
7314 let commands = commands.clone();
7315 let context = context.clone();
7316 let worktree = winner.worktree.clone();
7317 async move {
7318 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7319 .await
7320 }
7321 },
7322 )
7323 .await?;
7324 if verify_inconclusive(&e2e) {
7325 return Ok(GateFix::Defer);
7326 }
7327 if e2e.iter().any(|o| !o.ok()) {
7328 self.state.event(
7329 "gate",
7330 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7331 );
7332 return Ok(GateFix::Stop);
7333 }
7334 }
7335 Ok(GateFix::Retry)
7336 }
7337
7338 async fn guarded_pr_message(
7345 &mut self,
7346 winner: &Candidate,
7347 facts: Option<&BranchFacts>,
7348 posting: bool,
7349 ) -> Result<Option<PrMessage>> {
7350 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7351 let head = match final_head.as_deref() {
7352 Some(h) => crate::pr_outcome::FinalHead::At(h),
7353 None => crate::pr_outcome::FinalHead::Unknown,
7354 };
7355 let summarized = if posting {
7357 Some(self.summarize_change(winner, final_head.as_deref()).await)
7358 } else {
7359 None
7360 };
7361 if self.state.resource_block.is_some() {
7366 self.state.event(
7367 "pr-summary",
7368 "a shared resource was busy; nothing was published, resume to summarize again",
7369 );
7370 return Ok(None);
7371 }
7372 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7373 anyhow::bail!(
7374 "the final commit moved during public description generation; retry before publishing"
7375 );
7376 }
7377 let note = match (&summarized, &self.state.change_summary) {
7378 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7379 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7380 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7381 (None, _) => ChangeNote::NotAsked,
7382 };
7383 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7384 if !posting {
7385 let identity = crate::scrub::Identity::current();
7386 return Ok(Some(PrMessage {
7387 title: crate::scrub::scrub(&pr.title, &identity),
7388 body: crate::scrub::scrub(&pr.body, &identity),
7389 }));
7390 }
7391 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7394 if let Some(known) = self
7395 .state
7396 .github_text
7397 .clone()
7398 .filter(|g| g.fingerprint == raw_fp)
7399 {
7400 return self.settle_withheld(winner, &pr, known).await;
7401 }
7402 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7403 let mut violations =
7404 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7405 let identity = crate::scrub::Identity::current();
7406 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7407 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7408 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7409 {
7410 violations.push(crate::github_text::Violation::SensitiveData);
7411 }
7412 let has_language = violations
7416 .iter()
7417 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7418 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7419 self.state.event(
7420 "github-text",
7421 if has_language {
7422 format!("description rejected: {violations:?}; requesting one rewrite")
7423 } else {
7424 format!("description rejected: {violations:?}; asking the owner")
7425 },
7426 );
7427 let mut rewritten = false;
7428 let mut rejected_rewrite = None;
7429 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7430 {
7431 let output = text;
7432 if let Some(title) = summary_title(&output) {
7433 let raw_body = summary_without_title(&output);
7436 let clean_title = crate::scrub::scrub(&title, &identity);
7437 let mut body = crate::scrub::scrub(&raw_body, &identity);
7438 let redacted = clean_title != title || body != raw_body;
7439 let title = clean_title;
7440 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7441 if !complete {
7442 self.state.event(
7443 "github-text",
7444 "description rewrite omitted required findings",
7445 );
7446 }
7447 if !body.trim().is_empty() {
7448 body.push_str("\n\n");
7449 body.push_str(&public_pr_evidence(&self.state, head));
7450 }
7451 let marker = format!("magi:run/{}", self.state.id);
7452 if !body.trim().is_empty() && !body.contains(&marker) {
7453 body.push_str(&format!("\n\n{marker}"));
7454 }
7455 let rewrite = PrMessage {
7456 title: title.clone(),
7457 body: body.clone(),
7458 };
7459 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7460 let mut failed = crate::github_text::check_for(
7461 &self.state.config.graph,
7462 &title,
7463 &body,
7464 redecision,
7465 );
7466 if (!crate::github_text::shareable(&title)
7467 || !crate::github_text::shareable(&body))
7468 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7469 {
7470 failed.push(crate::github_text::Violation::SensitiveData);
7471 }
7472 if complete
7473 && !title.trim().is_empty()
7474 && !body.trim().is_empty()
7475 && !failed.is_empty()
7476 {
7477 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7478 title: title.clone(),
7479 body: body.clone(),
7480 failed: failed
7481 .iter()
7482 .map(|v| crate::github_text::category(*v).to_owned())
7483 .collect(),
7484 });
7485 }
7486 if complete
7487 && !title.trim().is_empty()
7488 && !raw_body.trim().is_empty()
7489 && failed.is_empty()
7490 {
7491 decision = redecision;
7492 pr = PrMessage { title, body };
7493 rewritten = true;
7494 self.state
7495 .event("github-text", "description rewrite passed");
7496 if redacted {
7497 self.state.event(
7498 "github-text",
7499 "sensitive data removed from description rewrite",
7500 );
7501 }
7502 }
7503 }
7504 }
7505 if !rewritten {
7506 self.state.event(
7507 "github-text",
7508 "description rewrite unavailable or rejected; asking the owner",
7509 );
7510 if let Some(w) =
7511 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7512 {
7513 self.retire_github_text_question("the withheld text changed");
7516 let record = crate::run::GithubTextAsk {
7517 rewrite: rejected_rewrite,
7518 fingerprint: raw_fp,
7519 title: w.title,
7520 body: w.body,
7521 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7522 question: None,
7523 asks: 0,
7524 resolved: false,
7525 chosen_title: None,
7526 choice: None,
7527 };
7528 return self.settle_withheld(winner, &pr, record).await;
7529 }
7530 }
7531 }
7532 let (title, body) =
7533 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7534 Ok(Some(PrMessage { title, body }))
7535 }
7536
7537 async fn summarize_change(
7548 &mut self,
7549 winner: &Candidate,
7550 head: Option<&str>,
7551 ) -> std::result::Result<(), String> {
7552 let Some(head) = head else {
7553 return Err("the final commit could not be read".to_owned());
7554 };
7555 let remote = self.state.config.merge.remote.clone();
7556 let tracking = format!("{remote}/{}", self.state.base_branch);
7557 if !git::rev_exists(&winner.worktree, &tracking).await {
7558 return Err(format!("the remote base `{tracking}` could not be read"));
7559 }
7560 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7561 Ok(b) if !b.is_empty() => b,
7562 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7563 };
7564 let evidence = crate::pr_outcome::summary_evidence(
7565 &self.state,
7566 crate::pr_outcome::FinalHead::At(head),
7567 );
7568 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7569 if self
7570 .state
7571 .change_summary
7572 .as_ref()
7573 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7574 {
7575 return Ok(());
7576 }
7577 if self.state.change_summary.take().is_some() {
7578 self.state.event(
7579 "pr-summary",
7580 "the final diff or evidence changed; the earlier change summary was discarded",
7581 );
7582 }
7583 if let Err(e) = write_artifact(
7585 &self.state,
7586 &format!("public-description-{fingerprint}.md"),
7587 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7588 ) {
7589 tracing::warn!("could not save public description evidence: {e:#}");
7590 }
7591 let chain = match agent::pick_chain(
7592 &self.state.config.agents,
7593 self.state.config.roles.synthesizer.as_ref(),
7594 &agent::installed,
7595 "synthesizer",
7596 ) {
7597 Ok(chain) => chain,
7598 Err(e) => {
7599 tracing::warn!("no agent for the change summary: {e:#}");
7600 self.state.event("pr-summary", "no summary agent available");
7601 return Err("no agent was available to summarize it".to_owned());
7602 }
7603 };
7604 let subjects = git::subjects(&winner.worktree, &base, head)
7605 .await
7606 .unwrap_or_default();
7607 let stat = git::diff_stat(&winner.worktree, &base, head)
7608 .await
7609 .unwrap_or_default();
7610 let stat: String = stat.chars().take(6_000).collect();
7611 let mut prompt = crate::prompt::change_summary_in(
7612 &base,
7613 head,
7614 &subjects,
7615 stat.trim(),
7616 &self.state.config.graph.github_language,
7617 );
7618 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7619 let sessions = self.state.config.graph.sessions;
7620 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7621 let prompts = self.state.config.prompts.clone();
7622 let cache = self.state.config.cache_dir();
7623 let run = self.state.id.clone();
7624 for (i, spec) in chain.into_iter().enumerate() {
7625 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7626 let job = SeatJob {
7627 spec: spec.clone(),
7628 seat,
7629 cwd: winner.worktree.clone(),
7630 prompt: prompt.clone(),
7631 timeout,
7632 allow_write: false,
7633 sessions,
7634 artifacts: agent::artifacts_dir(&self.state.dir()),
7635 stem: if i == 0 {
7636 "change-summary".to_owned()
7637 } else {
7638 format!("change-summary-{}", spec.id)
7639 },
7640 handover: None,
7641 };
7642 let ctx = WaveCtx {
7643 run: &run,
7644 node: "pr-summary",
7645 prompts: &prompts,
7646 cache: cache.as_deref(),
7647 round: None,
7648 carry_seats: false,
7649 pauses: Some((&self.pause, &self.interrupt)),
7650 };
7651 let (seat, outcome) =
7652 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7653 self.state.seats.insert(seat.key.clone(), seat);
7654 let text = match outcome {
7655 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7656 AgentOutcome::ResourceBlocked(why) => {
7657 if self.state.resource_block.is_none() {
7660 self.state.resource_block = Some(crate::run::ResourceBlock {
7661 node: "pr-summary".to_owned(),
7662 kind: "shared resource".to_owned(),
7663 detail: why,
7664 at: Timestamp::now(),
7665 });
7666 }
7667 return Err("a shared resource was busy".to_owned());
7668 }
7669 _ => continue,
7670 };
7671 let blind = &self.state.config.blind;
7672 let Some(title) = summary_title(&text) else {
7673 continue;
7674 };
7675 let body: String = summary_without_title(&text)
7676 .lines()
7677 .filter(|l| !l.contains("magi:run/"))
7678 .collect::<Vec<_>>()
7679 .join("\n");
7680 let body = blind::sanitize_prose(body.trim(), blind);
7681 let title = blind::sanitize_prose(&title, blind);
7682 if body.trim().is_empty() || title.trim().is_empty() {
7683 continue;
7684 }
7685 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7686 continue;
7687 }
7688 if git::rev_parse(&winner.worktree, "HEAD")
7690 .await
7691 .ok()
7692 .as_deref()
7693 != Some(head)
7694 {
7695 return Err("the branch moved while the summary was written".to_owned());
7696 }
7697 if git::merge_base(&winner.worktree, &tracking, head)
7698 .await
7699 .ok()
7700 .as_deref()
7701 != Some(base.as_str())
7702 || crate::pr_outcome::summary_evidence(
7703 &self.state,
7704 crate::pr_outcome::FinalHead::At(head),
7705 ) != evidence
7706 {
7707 return Err("the summary evidence changed while it was written".to_owned());
7708 }
7709 self.state.change_summary = Some(crate::run::ChangeSummary {
7710 head: head.to_owned(),
7711 base,
7712 evidence: fingerprint,
7713 title,
7714 body,
7715 });
7716 self.state.event(
7717 "pr-summary",
7718 format!("change summary written by {} for the final diff", spec.id),
7719 );
7720 if let Err(e) = self.state.save() {
7721 tracing::warn!("could not save the change summary: {e:#}");
7722 }
7723 return Ok(());
7724 }
7725 self.state
7726 .event("pr-summary", "no agent produced a change summary");
7727 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7728 }
7729
7730 async fn rewrite_pr_text(
7738 &mut self,
7739 winner: &Candidate,
7740 pr: &PrMessage,
7741 violations: &[crate::github_text::Violation],
7742 ) -> Option<String> {
7743 let prompt = format!(
7744 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7745 pr.title, pr.body
7746 );
7747 let sessions = self.state.config.graph.sessions;
7748 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7749 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7751 let key = format!("impl-{}", winner.label);
7752 if let Some(spec) = self
7753 .state
7754 .config
7755 .agents
7756 .iter()
7757 .find(|a| a.id == winner.agent)
7758 .cloned()
7759 && let Some(existing) = self.state.seats.get(&key)
7760 && existing.agent == spec.id
7761 && agent::has_session(spec.kind, existing, sessions)
7762 {
7763 attempts.push((spec, existing.clone(), true));
7764 } else {
7765 let chain = agent::pick_chain(
7766 &self.state.config.agents,
7767 self.state.config.roles.language_judge.as_ref(),
7768 &agent::installed,
7769 "language_judge",
7770 );
7771 match chain {
7772 Ok(chain) => {
7773 for spec in chain {
7774 let seat = crate::fixer::seat_for(
7775 &mut self.state,
7776 "github-text-rewrite",
7777 &spec.id,
7778 );
7779 attempts.push((spec, seat, false));
7780 }
7781 }
7782 Err(e) => {
7783 tracing::warn!("no agent for the description rewrite: {e:#}");
7784 self.state
7785 .event("github-text", "no rewrite agent available");
7786 return None;
7787 }
7788 }
7789 }
7790 let prompts = self.state.config.prompts.clone();
7791 let cache = self.state.config.cache_dir();
7792 let run = self.state.id.clone();
7793 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7794 let job = SeatJob {
7795 spec: spec.clone(),
7796 seat,
7797 cwd: winner.worktree.clone(),
7798 prompt: prompt.clone(),
7799 timeout: retry_budget(timeout_full, resumed),
7800 allow_write: false,
7801 sessions,
7802 artifacts: agent::artifacts_dir(&self.state.dir()),
7803 stem: if i == 0 {
7804 "github-text-rewrite".to_owned()
7805 } else {
7806 format!("github-text-rewrite-{}", spec.id)
7807 },
7808 handover: None,
7809 };
7810 let ctx = WaveCtx {
7811 run: &run,
7812 node: "github-text",
7813 prompts: &prompts,
7814 cache: cache.as_deref(),
7815 round: None,
7816 carry_seats: false,
7817 pauses: Some((&self.pause, &self.interrupt)),
7818 };
7819 let (seat, outcome) =
7820 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7821 self.state.seats.insert(seat.key.clone(), seat);
7822 match outcome {
7823 AgentOutcome::Ok(o)
7824 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7825 {
7826 return Some(o.text);
7827 }
7828 AgentOutcome::ResourceBlocked(_) => return None,
7829 _ => {}
7830 }
7831 }
7832 None
7833 }
7834
7835 fn retire_github_text_question(&mut self, why: &str) {
7837 let Some(id) = self
7838 .state
7839 .github_text
7840 .as_ref()
7841 .and_then(|g| g.question.clone())
7842 else {
7843 return;
7844 };
7845 let _ = ask::Questions::open().update(&id, |q| {
7846 q.abandon(why);
7847 Ok(())
7848 });
7849 }
7850
7851 async fn settle_withheld(
7855 &mut self,
7856 winner: &Candidate,
7857 pr: &PrMessage,
7858 mut g: crate::run::GithubTextAsk,
7859 ) -> Result<Option<PrMessage>> {
7860 use crate::github_text as gt;
7861 let store = ask::Questions::open();
7862 let timeout = self.state.config.graph.answer_timeout;
7863 if !g.resolved {
7864 let mut standing = g
7865 .question
7866 .as_ref()
7867 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7868 if let Some(q) = standing.as_ref()
7869 && q.status.open()
7870 && gt::expired(q, timeout)
7871 {
7872 let why = format!("no answer within {}s of asking", timeout.max(1));
7873 standing = store
7874 .update(&q.id, |q| {
7875 q.abandon(&why);
7876 Ok(())
7877 })
7878 .ok()
7879 .map(|(q, ())| q);
7880 }
7881 let mut ask_again: Option<Option<Vec<String>>> = None;
7882 match standing {
7883 None if g.asks <= 1 => {
7886 g.asks = 0;
7887 ask_again = Some(None);
7888 }
7889 None => {
7890 g.resolved = true;
7891 g.chosen_title = None;
7892 g.choice = Some(GithubTextChoice::Fallback);
7893 }
7894 Some(q) if q.status.open() => {
7895 self.park_on_github_text(&g)?;
7896 return Ok(None);
7897 }
7898 Some(q) => match gt::read_reply(&q) {
7899 gt::Reply::Fallback => {
7900 g.resolved = true;
7901 g.chosen_title = None;
7902 g.choice = Some(GithubTextChoice::Fallback);
7903 }
7904 gt::Reply::Original => {
7905 g.resolved = true;
7906 g.chosen_title = None;
7907 g.choice = Some(GithubTextChoice::Original);
7908 }
7909 gt::Reply::Rewrite => {
7910 g.resolved = true;
7911 g.chosen_title = None;
7912 g.choice = Some(if g.rewrite.is_some() {
7913 GithubTextChoice::Rewrite
7914 } else {
7915 GithubTextChoice::Fallback
7916 });
7917 }
7918 gt::Reply::Title(text) => {
7919 let first = text
7920 .lines()
7921 .map(str::trim)
7922 .find(|l| !l.is_empty())
7923 .unwrap_or("");
7924 let decision = if first.is_empty() {
7925 None
7926 } else {
7927 let candidate = PrMessage {
7928 title: first.to_owned(),
7929 body: String::new(),
7930 };
7931 self.judge_pr_language(&winner.worktree, &candidate).await
7932 };
7933 match gt::vet_title(
7934 &text,
7935 decision,
7936 self.state.config.graph.github_english(),
7937 ) {
7938 Ok(title) => {
7939 g.resolved = true;
7940 g.chosen_title = Some(title);
7941 g.choice = Some(GithubTextChoice::MyTitle);
7942 }
7943 Err(cats) if g.asks < 2 => {
7944 ask_again = Some(Some(
7945 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7946 ))
7947 }
7948 Err(_) => {
7949 g.resolved = true;
7950 g.chosen_title = None;
7951 g.choice = Some(GithubTextChoice::Fallback);
7952 }
7953 }
7954 }
7955 },
7956 }
7957 if let Some(rejected) = ask_again {
7958 let w = gt::Withheld {
7961 title: g.title,
7962 body: g.body,
7963 categories: g.categories.clone(),
7964 };
7965 let artifact = match crate::run::write_artifact(
7968 &self.state,
7969 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7970 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7971 ) {
7972 Ok(p) => Some(p),
7973 Err(e) => {
7974 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7975 None
7976 }
7977 };
7978 let (detail, comparison) = gt::question_content(
7979 &w,
7980 &pr.title,
7981 &pr.body,
7982 &self.state.id,
7983 rejected.as_deref(),
7984 artifact.as_deref(),
7985 g.rewrite.as_ref(),
7986 );
7987 let mut q = ask::Question::new(
7988 self.state.id.clone(),
7989 gt::ASK_NODE.to_owned(),
7990 gt::ASK_SEAT.to_owned(),
7991 gt::question_summary(&self.state.config.graph.language, &w),
7992 detail,
7993 gt::question_choices(&w, g.rewrite.as_ref()),
7994 );
7995 q.posting_comparison = Some(comparison);
7996 q.answer_timeout = timeout;
7997 g.question = Some(q.id.clone());
8000 g.asks += 1;
8001 self.state.github_text = Some(g.clone());
8002 self.state.event(
8003 "github-text",
8004 format!("asking the owner about the withheld text ({})", q.short()),
8005 );
8006 self.state.save()?;
8007 store
8008 .put_executable(&mut q)
8009 .context("file the posting-gate question")?;
8010 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
8011 tracing::warn!(
8012 "could not notify about posting-gate question {}: {e:#}",
8013 q.short()
8014 );
8015 }
8016 self.park_on_github_text(&g)?;
8017 return Ok(None);
8018 }
8019 }
8020 let resolved = g.chosen_title.clone();
8021 let original = g.choice == Some(GithubTextChoice::Original);
8022 self.state.event(
8024 "github-text",
8025 if original {
8026 format!(
8027 "posting the original text for: {}",
8028 match (g.title, g.body) {
8029 (true, true) => "title, description",
8030 (true, false) => "title",
8031 _ => "description",
8032 }
8033 )
8034 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
8035 "posting the saved rewrite title and description".to_owned()
8036 } else if resolved.is_some() {
8037 "posting the owner's replacement title".to_owned()
8038 } else {
8039 "posting the neutral text for the withheld fields".to_owned()
8040 },
8041 );
8042 self.state.github_text = Some(g.clone());
8043 self.state.save()?;
8044 if g.choice == Some(GithubTextChoice::Rewrite)
8045 && let Some(r) = g.rewrite.as_ref()
8046 {
8047 return Ok(Some(PrMessage {
8048 title: r.title.clone(),
8049 body: r.body.clone(),
8050 }));
8051 }
8052 let w = gt::Withheld {
8053 title: g.title,
8054 body: g.body,
8055 categories: g.categories.clone(),
8056 };
8057 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8058 if let Some(t) = resolved {
8059 title = t;
8060 }
8061 let english = gt::LanguageDecision {
8063 title_english: true,
8064 body_english: true,
8065 body_complete: true,
8066 };
8067 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8068 if original {
8069 if g.title {
8072 title = pr.title.clone();
8073 }
8074 if g.body {
8075 body = pr.body.clone();
8076 }
8077 }
8078 Ok(Some(PrMessage { title, body }))
8079 }
8080
8081 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8082 self.state.github_text = Some(g.clone());
8083 self.state.end_pass_parked(ParkCause::Question);
8084 self.state.parked = true;
8085 self.state.event(
8086 "github-text",
8087 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8088 );
8089 self.state.save()
8090 }
8091
8092 async fn judge_pr_language(
8095 &mut self,
8096 cwd: &Path,
8097 pr: &PrMessage,
8098 ) -> Option<crate::github_text::LanguageDecision> {
8099 self.state.config.roles.language_judge.as_ref()?;
8100 if !self.state.config.graph.language_gate() {
8101 return None;
8102 }
8103 let decision =
8104 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8105 self.state.event(
8106 "github-text",
8107 if decision.is_some() {
8108 "language decided by the language judge"
8109 } else {
8110 "language judge unavailable; using built-in heuristics"
8111 },
8112 );
8113 decision
8114 }
8115
8116 async fn merge(&mut self) -> Result<()> {
8117 if self
8132 .state
8133 .base_sync
8134 .as_ref()
8135 .is_some_and(|s| s.conflict.is_some())
8136 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8137 != Some(RunStatus::Gating)
8138 || !self.state.gate_allows_progress()
8146 {
8147 return Ok(());
8148 }
8149 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8158 return Ok(());
8159 }
8160 let Some(winner) = self.state.winner().cloned() else {
8161 return Ok(());
8162 };
8163 let repo = self.state.repo.clone();
8164 let base = self.state.base_branch.clone();
8165 let mode = self.state.config.merge.mode;
8166 let style = self.state.config.merge.style;
8167 let facts = if is_review_run(&self.state) {
8168 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8169 let start = review_base(
8170 &repo,
8171 &self.state.config.merge.remote,
8172 &base,
8173 &self.state.base_commit,
8174 &winner.branch,
8175 )
8176 .await;
8177 branch_facts(&repo, &start, &winner.branch).await
8178 } else {
8179 None
8180 };
8181 let leaked = if is_review_run(&self.state) {
8184 leaked_subjects(&self.state, &winner.branch).await
8185 } else {
8186 Some(Vec::new())
8187 };
8188 let Some(pr) = self
8192 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8193 .await?
8194 else {
8195 return Ok(());
8196 };
8197 let message = pr.commit_message();
8198
8199 let outcome = match mode {
8200 MergeMode::None => MergeOutcome {
8201 mode,
8202 ok: true,
8203 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8204 empty: false,
8205 },
8206 MergeMode::Pr | MergeMode::Local
8207 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8208 {
8209 MergeOutcome {
8210 mode,
8211 ok: false,
8212 detail: empty_candidate_detail(&self.state, &base),
8213 empty: true,
8214 }
8215 }
8216 MergeMode::Local => {
8217 let on = git::current_branch(&repo).await?;
8218 if on.as_deref() != Some(base.as_str()) {
8219 MergeOutcome {
8220 mode,
8221 ok: false,
8222 detail: format!(
8223 "{} has {} checked out, not the base branch {base}",
8224 repo.display(),
8225 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8226 ),
8227 empty: false,
8228 }
8229 } else if !git::is_clean(&repo).await? {
8230 MergeOutcome {
8231 mode,
8232 ok: false,
8233 detail: format!("{} is dirty; refusing to merge", repo.display()),
8234 empty: false,
8235 }
8236 } else {
8237 let out = match style {
8238 MergeStyle::Merge => {
8239 git::merge_no_ff(&repo, &winner.branch, &message).await?
8240 }
8241 MergeStyle::Squash => {
8242 git::merge_squash(&repo, &winner.branch, &message).await?
8243 }
8244 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8245 };
8246 MergeOutcome {
8247 mode,
8248 ok: out.ok(),
8249 detail: if out.ok() { out.stdout } else { out.stderr },
8250 empty: false,
8251 }
8252 }
8253 }
8254 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8255 self.state.pr_publication_pending = true;
8256 self.state.save()?;
8257 let out =
8258 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8259 self.state.pr_publication_pending = out.is_err();
8260 MergeOutcome {
8261 mode,
8262 ok: out.is_ok(),
8263 detail: match out {
8264 Ok(url) => url,
8265 Err(e) => format!("{e:#}"),
8266 },
8267 empty: false,
8268 }
8269 }
8270 MergeMode::Pr => {
8271 let remote = self.state.config.merge.remote.clone();
8272 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8273 if !pushed.ok() {
8274 MergeOutcome {
8275 mode,
8276 ok: false,
8277 detail: pushed.stderr,
8278 empty: false,
8279 }
8280 } else {
8281 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8287 let out = match pr_merge_plan(found) {
8288 PrPlan::Create => {
8289 gh_pr_create(
8290 &winner.worktree,
8291 &base,
8292 &winner.branch,
8293 &pr.title,
8294 &pr.body,
8295 )
8296 .await
8297 }
8298 PrPlan::Adopt { url, title } => {
8299 self.state
8300 .event("merge", format!("Pr: adopted open pull request {url}"));
8301 if self.state.github_text.as_ref().is_some_and(|g| {
8302 g.resolved
8303 && g.choice == Some(GithubTextChoice::Rewrite)
8304 && g.rewrite.is_some()
8305 }) {
8306 land::set_pr_text_verbatim(
8309 &winner.worktree,
8310 &url,
8311 &pr.title,
8312 &pr.body,
8313 )
8314 .await
8315 .map(|()| url)
8316 } else {
8317 if title != pr.title
8318 && (!is_review_run(&self.state)
8319 || leaked
8320 .as_deref()
8321 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8322 && let Err(e) =
8323 if self.state.github_text.as_ref().is_some_and(|g| {
8324 g.resolved
8325 && g.title
8326 && g.choice == Some(GithubTextChoice::Original)
8327 }) {
8328 land::set_pr_title_verbatim(
8330 &winner.worktree,
8331 &url,
8332 &pr.title,
8333 )
8334 .await
8335 } else {
8336 land::set_pr_title(
8337 &mut self.state,
8338 &winner.worktree,
8339 &url,
8340 &pr.title,
8341 )
8342 .await
8343 }
8344 {
8345 tracing::warn!("could not refresh title of {url}: {e:#}");
8346 self.state
8347 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8348 }
8349 Ok(url)
8350 }
8351 }
8352 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8353 };
8354 match out {
8355 Ok(url) => MergeOutcome {
8356 mode,
8357 ok: true,
8358 detail: url,
8359 empty: false,
8360 },
8361 Err(e) => MergeOutcome {
8362 mode,
8363 ok: false,
8364 detail: e.to_string(),
8365 empty: false,
8366 },
8367 }
8368 }
8369 }
8370 };
8371
8372 self.state.status = match (mode, outcome.ok) {
8373 (MergeMode::None, _) => RunStatus::Ready,
8374 (_, true) if outcome.empty => RunStatus::Ready,
8375 (MergeMode::Pr, true) => RunStatus::Ready,
8376 (_, true) => RunStatus::Merged,
8377 (_, false) => RunStatus::Blocked,
8378 };
8379 self.state.event(
8380 "merge",
8381 format!(
8382 "{:?}: {}",
8383 mode,
8384 outcome.detail.lines().next().unwrap_or("")
8385 ),
8386 );
8387 self.state.merge = Some(outcome);
8388 self.state.save()?;
8389
8390 if self.state.config.graph.land
8396 && mode == MergeMode::Pr
8397 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8398 {
8399 self.run_land().await?;
8400 }
8401 self.settle_questions();
8406 Ok(())
8407 }
8408
8409 async fn run_land(&mut self) -> Result<()> {
8420 crate::pr_head::ensure_owner(&self.state)?;
8421 let url = self
8422 .state
8423 .merge
8424 .as_ref()
8425 .map(|m| m.detail.clone())
8426 .unwrap_or_default();
8427 let url = url.lines().next().unwrap_or("").trim().to_owned();
8428 if !url.starts_with("http") {
8429 return Ok(());
8430 }
8431 loop {
8434 match land::land(&mut self.state, &url).await {
8435 Ok(_)
8436 if self
8437 .state
8438 .merge_repairs
8439 .last()
8440 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8441 {
8442 if self.repair_merge_review().await? {
8443 continue;
8444 }
8445 }
8446 Ok(pr) if self.state.parked => {
8447 let _ = pr;
8451 }
8452 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8453 self.state.status = RunStatus::Superseded;
8454 self.state.save()?;
8455 }
8456 Ok(pr) => {
8457 self.state.status = match pr.state {
8458 land::PrLifecycle::Merged => RunStatus::Merged,
8459 _ => RunStatus::Blocked,
8460 };
8461 if bump::should_release_bump(self.state.status)
8468 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8469 {
8470 self.state
8476 .event("bump", format!("release bump skipped: {e:#}"));
8477 }
8478 if self.state.status == RunStatus::Merged {
8481 crate::followup::after_merge(&mut self.state, &pr.url).await;
8482 }
8483 self.state.save()?;
8484 }
8485 Err(e) => {
8486 if e.is::<crate::repo_lock::ResourceBlocked>() {
8487 return Err(e);
8488 }
8489 self.state.status = RunStatus::Blocked;
8490 self.state.event("land", format!("gave up: {e}"));
8491 self.state.save()?;
8492 }
8493 }
8494 break;
8495 }
8496 Ok(())
8497 }
8498
8499 async fn repair_merge_review(&mut self) -> Result<bool> {
8502 use crate::run::MergeRepairPhase as Phase;
8503 crate::pr_head::ensure_owner(&self.state)?;
8504 let i = self.state.merge_repairs.len() - 1;
8505 let request = self.state.merge_repairs[i].clone();
8506 if request.phase == Phase::Fixing {
8507 let restored = match self.state.winner().cloned() {
8508 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8509 None => Err(anyhow::anyhow!("repair has no winning branch")),
8510 };
8511 let why = match restored {
8512 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8513 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8514 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8515 };
8516 self.hold_merge_repair(i, why)?;
8517 return Ok(false);
8518 }
8519 let failure = if i >= self.state.config.graph.land_rounds
8520 || self.state.config.graph.review_rounds == 0
8521 {
8522 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8523 } else {
8524 None
8525 };
8526 if let Some(why) = failure {
8527 self.hold_merge_repair(i, why)?;
8528 return Ok(false);
8529 }
8530 if request.phase == Phase::Requested {
8531 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8532 Ok(pr) => pr,
8533 Err(e) => {
8534 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8535 return Ok(false);
8536 }
8537 };
8538 self.state.merge_repairs[i].phase = Phase::Fixing;
8539 self.state.land_approval = None;
8540 self.state.land_armed_head = None;
8541 self.state.save()?;
8542 let evidence = serde_json::to_string_pretty(&request)?;
8545 let reason = format!(
8546 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8547 );
8548 let budget = self.state.config.graph.land_rounds;
8549 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8550 match result {
8551 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8552 Ok(land::Fixed::Failed(why)) => {
8553 self.hold_merge_repair(
8554 i,
8555 format!(
8556 "repair failed: {why}; inspect the branch and run an explicit review"
8557 ),
8558 )?;
8559 return Ok(false);
8560 }
8561 Err(e) => {
8562 self.hold_merge_repair(
8563 i,
8564 format!(
8565 "repair failed: {e:#}; inspect the branch and run an explicit review"
8566 ),
8567 )?;
8568 return Ok(false);
8569 }
8570 }
8571 self.begin_merge_repair_review(i)?;
8575 }
8576 self.review_loop().await?;
8577 self.gate().await?;
8578 if self.resource_halt()? || self.state.parked {
8579 return Ok(false);
8580 }
8581 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8582 != Some(RunStatus::Gating)
8583 || !self.state.gate_allows_progress()
8584 {
8585 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8586 return Ok(false);
8587 }
8588 let winner = self
8591 .state
8592 .winner()
8593 .cloned()
8594 .context("repair has no winning branch")?;
8595 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8596 if self.state.gate_head.as_deref() != Some(after.as_str())
8597 || !git::is_clean(&winner.worktree).await?
8598 {
8599 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8600 return Ok(false);
8601 }
8602 let expected = self
8603 .state
8604 .pr_pushed_head
8605 .as_deref()
8606 .unwrap_or(&request.head)
8607 .to_owned();
8608 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8609 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8612 let refspec = format!("{after}:refs/heads/{public}");
8613 let push = git::git_raw(
8614 &winner.worktree,
8615 &["push", &lease, &self.state.config.merge.remote, &refspec],
8616 )
8617 .await;
8618 match push {
8619 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8620 Ok(out) => {
8621 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8622 return Ok(false);
8623 }
8624 Err(e) => {
8625 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8626 return Ok(false);
8627 }
8628 }
8629 self.state.merge_repairs[i].phase = Phase::Complete;
8630 self.state.status = RunStatus::Landing;
8631 self.state.event(
8632 "repair",
8633 "new review and gate completed; fresh owner merge decision required",
8634 );
8635 self.state.save()?;
8636 Ok(true)
8637 }
8638
8639 fn merge_repair_artifacts(&self) -> PathBuf {
8640 let artifacts = agent::artifacts_dir(&self.state.dir());
8641 match self
8642 .state
8643 .merge_repairs
8644 .last()
8645 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8646 {
8647 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8648 None => artifacts,
8649 }
8650 }
8651
8652 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8653 self.state.reviews.clear();
8654 self.state.gate.clear();
8655 self.state.gate_ran = false;
8656 self.state.gate_head = None;
8657 self.state.gate_fixes.clear();
8658 self.state.pre_gate.clear();
8659 self.state.pre_gate_commit = None;
8660 self.state.contested_handoff = None;
8661 self.state.land_reruns = None;
8662 self.state.land_approval = None;
8663 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8664 self.state.status = RunStatus::Reviewing;
8665 self.state.save()
8666 }
8667
8668 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8669 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8670 self.state.merge_repairs[index].failure = Some(why.clone());
8671 self.state.status = RunStatus::Blocked;
8672 self.state.event("repair", why);
8673 self.state.save()
8674 }
8675
8676 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8680 if let Some(existing) = self.state.seats.get(key)
8681 && existing.agent == agent
8682 {
8683 return existing.clone();
8684 }
8685 let fresh = if self.state.seats.contains_key(key) {
8688 handover_seat(key, agent, self.state.next_seat_seed())
8689 } else {
8690 SeatState::new(key, agent, self.state.seed)
8691 };
8692 self.state.seats.insert(key.to_owned(), fresh.clone());
8693 fresh
8694 }
8695
8696 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8702 match self.state.seats.get(key) {
8703 Some(s) if s.agent != spec.id => {
8704 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8705 }
8706 _ => spec,
8707 }
8708 }
8709
8710 fn view(&self, c: &Candidate) -> CandidateView {
8712 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8713 .unwrap_or_default();
8714 let (patch, _) = blind::sanitize_patch(
8715 &format!("candidate {} patch", c.label),
8716 &raw,
8717 &self.state.config.blind,
8718 );
8719 CandidateView {
8720 label: c.label,
8721 branch: c.branch.clone(),
8722 summary: c.summary.clone(),
8723 stat: c.stat.clone(),
8724 patch,
8725 }
8726 }
8727
8728 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8730 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8731 prompt::judge(
8732 "(see above)",
8733 &views,
8734 self.roles.judges.len(),
8735 base_short,
8736 "en",
8737 )
8738 }
8739
8740 fn vote_prompt_full(
8746 &self,
8747 j: usize,
8748 labels: &[char],
8749 language: &str,
8750 candidates: &[Candidate],
8751 base_short: &str,
8752 ) -> String {
8753 let mut text = format!(
8754 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8755 prompt::final_vote(labels, language),
8756 self.state.instruction,
8757 self.candidate_block(candidates, base_short)
8758 );
8759 if let Some(own) = self
8760 .state
8761 .judgements
8762 .get(j)
8763 .filter(|r| !r.ranking.is_empty())
8764 {
8765 let reasons = own
8766 .reasons
8767 .iter()
8768 .map(|(k, v)| format!("- {k}: {v}"))
8769 .collect::<Vec<_>>()
8770 .join("\n");
8771 text.push_str(&format!(
8772 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8773 own.ranking.iter().collect::<String>(),
8774 if reasons.is_empty() {
8775 ""
8776 } else {
8777 ", because:\n"
8778 }
8779 ));
8780 }
8781 if !self.state.deliberation.is_empty() {
8782 text.push_str("\n# What was argued before this vote\n");
8783 for t in self.transcript(&[], j) {
8784 text.push_str(&format!(
8785 "\n## {}{}\n\n{}\n",
8786 t.who,
8787 if t.is_self { " (you)" } else { "" },
8788 t.body.trim()
8789 ));
8790 }
8791 }
8792 text
8793 }
8794
8795 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8802 let mut turns = Vec::new();
8803 for j in &self.state.judgements {
8804 if j.ranking.is_empty() {
8805 continue;
8806 }
8807 let reasons = j
8808 .reasons
8809 .iter()
8810 .map(|(k, v)| format!("- {k}: {v}"))
8811 .collect::<Vec<_>>()
8812 .join("\n");
8813 turns.push(Turn {
8814 who: format!("Judge {} (opening ranking)", j.judge),
8815 is_self: j.judge == self_idx + 1,
8816 body: format!(
8817 "Ranked {}{}{reasons}",
8818 j.ranking.iter().collect::<String>(),
8819 if reasons.is_empty() {
8820 ""
8821 } else {
8822 ", because:\n"
8823 }
8824 ),
8825 });
8826 }
8827 for t in self
8828 .state
8829 .deliberation
8830 .iter()
8831 .flat_map(|r| r.turns.iter())
8832 .chain(current)
8833 {
8834 turns.push(Turn {
8835 who: format!("Judge {}", t.judge),
8836 is_self: t.judge == self_idx + 1,
8837 body: t.body.clone(),
8838 });
8839 }
8840 turns
8841 }
8842}
8843
8844fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8846 agent::has_session(spec.kind, seat, sessions)
8847}
8848
8849fn next_untried_in_roster<'a>(
8870 roster: &'a [AgentSpec],
8871 start: usize,
8872 tried: &BTreeSet<String>,
8873) -> Option<&'a AgentSpec> {
8874 roster
8875 .get(start + 1..)?
8876 .iter()
8877 .find(|s| !tried.contains(&s.id))
8878}
8879
8880fn pick_successor<'a>(
8901 roster: &'a [AgentSpec],
8902 start: usize,
8903 tried: &BTreeSet<String>,
8904 carried: Option<&BTreeSet<String>>,
8905 others: &BTreeSet<String>,
8906) -> Option<&'a AgentSpec> {
8907 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8908 roster
8909 .get(start + 1..)
8910 .and_then(|tail| {
8911 tail.iter()
8912 .filter(free)
8913 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8914 })
8915 .or_else(|| {
8916 carried
8917 .is_some()
8918 .then(|| roster.iter().find(free))
8919 .flatten()
8920 })
8921 .or_else(|| match carried {
8922 Some(c) => next_for_seat(roster, start, tried, c),
8923 None => next_untried_in_roster(roster, start, tried),
8924 })
8925}
8926
8927fn next_for_seat<'a>(
8937 roster: &'a [AgentSpec],
8938 start: usize,
8939 round_tried: &BTreeSet<String>,
8940 carried_failed: &BTreeSet<String>,
8941) -> Option<&'a AgentSpec> {
8942 roster
8943 .get(start + 1..)?
8944 .iter()
8945 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8946 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8947}
8948
8949fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8955 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8956 return spec;
8957 };
8958 let ok = |id: &str| !h.failed.contains(id);
8959 if let Some(last) = h.last_ok.as_deref()
8960 && ok(last)
8961 && let Some(s) = roster.iter().find(|s| s.id == last)
8962 {
8963 return s.clone();
8964 }
8965 if ok(&spec.id) {
8966 return spec;
8967 }
8968 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8969 roster
8970 .get(from + 1..)
8971 .into_iter()
8972 .flatten()
8973 .chain(roster.iter())
8974 .find(|s| ok(&s.id))
8975 .cloned()
8976 .unwrap_or(spec)
8977}
8978
8979pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8983 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8984}
8985
8986const TIMED_OUT: &str = "timed out";
8989
8990impl FailClass {
8991 fn of(out: &AgentOutcome) -> Option<Self> {
8993 match out {
8994 AgentOutcome::Ok(_) => None,
8995 AgentOutcome::Quota(_) => Some(Self::Quota),
8996 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8997 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8998 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8999 AgentOutcome::ResourceBlocked(_) => None,
9002 }
9003 }
9004
9005 fn stem_word(&self) -> &'static str {
9007 match self {
9008 Self::Quota => "quota",
9009 _ => "handover",
9010 }
9011 }
9012}
9013
9014fn failure_signature(msg: &str) -> String {
9018 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
9019 if let Some(code) = line
9022 .strip_prefix("process exited with no usable output (exit code ")
9023 .and_then(|s| s.strip_suffix(')'))
9024 .and_then(|s| s.parse::<i32>().ok())
9025 {
9026 line = format!("exited with some({code}) and no usable output");
9027 } else if matches!(
9028 line.as_str(),
9029 "process exited with no usable output (no exit code reported)"
9030 | "process exited with no usable output (exit code unavailable)"
9031 ) {
9032 line = "exited with none and no usable output".to_owned();
9033 }
9034 let mut out = Vec::new();
9035 for word in line.split_whitespace() {
9036 if word.contains('/') || word.contains('\\') {
9037 out.push("<path>".to_owned());
9038 continue;
9039 }
9040 let mut w = String::new();
9041 let mut in_digits = false;
9042 for c in word.chars() {
9043 if c.is_ascii_digit() {
9044 if !in_digits {
9045 w.push('#');
9046 }
9047 in_digits = true;
9048 } else {
9049 in_digits = false;
9050 w.push(c);
9051 }
9052 }
9053 out.push(w);
9054 }
9055 out.join(" ").chars().take(120).collect()
9056}
9057
9058fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9065 match cur {
9066 FailClass::Quota | FailClass::Timeout => true,
9067 FailClass::Other(_) => prev != Some(cur),
9068 }
9069}
9070
9071fn fail_reason(out: &AgentOutcome) -> String {
9073 match out {
9074 AgentOutcome::Ok(_) => String::new(),
9075 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9076 AgentOutcome::Dropped(o) => format!(
9077 "the CLI dropped the stream ({})",
9078 o.dropped
9079 .as_ref()
9080 .map(|d| d.why.as_str())
9081 .unwrap_or("it ended without delivering its answer")
9082 ),
9083 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9084 e.lines().next().unwrap_or("").chars().take(160).collect()
9085 }
9086 }
9087}
9088
9089pub(crate) fn record_handover(
9092 state: &mut RunState,
9093 node: &str,
9094 seat: &str,
9095 from: &str,
9096 to: &str,
9097 class: &FailClass,
9098 reason: &str,
9099) {
9100 let message = if *class == FailClass::Quota {
9101 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9102 } else {
9103 format!("{seat}: handed over {from} -> {to} ({reason})")
9104 };
9105 state.event(node, message);
9106 state.handovers.push(Handover {
9107 at: Timestamp::now(),
9108 node: node.to_owned(),
9109 seat: seat.to_owned(),
9110 from: from.to_owned(),
9111 to: to.to_owned(),
9112 reason: reason.to_owned(),
9113 });
9114}
9115
9116fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9130 commands.iter().any(|c| c.exit_code.is_none())
9131}
9132
9133fn verified_noop_claim(
9146 usable: bool,
9147 commands: &[agent::CommandEvidence],
9148 text: &str,
9149) -> Option<String> {
9150 (usable && !has_unconfirmed_command(commands))
9151 .then(|| verdict::verified_noop(text))
9152 .flatten()
9153}
9154
9155fn short(commit: &str) -> String {
9156 commit.chars().take(7).collect()
9157}
9158
9159fn make_executable(path: &Path) -> Result<()> {
9160 #[cfg(unix)]
9161 {
9162 use std::os::unix::fs::PermissionsExt as _;
9163 let mut perms = std::fs::metadata(path)?.permissions();
9164 perms.set_mode(0o755);
9165 std::fs::set_permissions(path, perms)?;
9166 }
9167 #[cfg(not(unix))]
9168 {
9169 let _ = path;
9170 }
9171 Ok(())
9172}
9173
9174struct WaveCtx<'a> {
9181 run: &'a str,
9184 node: &'a str,
9186 prompts: &'a Prompts,
9187 cache: Option<&'a Path>,
9189 round: Option<usize>,
9192 carry_seats: bool,
9196 pauses: Option<(&'a Pause, &'a Pause)>,
9199}
9200
9201fn repository_wait(
9203 state: &RunState,
9204 pauses: Option<(&Pause, &Pause)>,
9205) -> crate::repo_lock::WaitPolicy {
9206 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9207 let inherited = crate::repo_lock::WaitPolicy::current();
9208 crate::repo_lock::WaitPolicy::new(
9209 Duration::from_secs(state.config.graph.timeout_resource_wait),
9210 move || {
9211 inherited.cancelled()
9212 || pauses
9213 .as_ref()
9214 .is_some_and(|(a, b)| a.parked() || b.parked())
9215 },
9216 )
9217}
9218
9219fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9220 if error.is::<crate::repo_lock::ResourceBlocked>() {
9221 state.resource_block = Some(crate::run::ResourceBlock {
9222 node: "repository".into(),
9223 kind: if error.to_string().contains("disk pressure") {
9224 "disk pressure"
9225 } else {
9226 "repository coordination"
9227 }
9228 .into(),
9229 detail: error.to_string(),
9230 at: Timestamp::now(),
9231 });
9232 state.event("resource", error.to_string());
9233 }
9234}
9235
9236async fn coordinate_operation(
9237 state: &mut RunState,
9238 cwd: &Path,
9239 wait: &crate::repo_lock::WaitPolicy,
9240) -> Result<Option<crate::repo_lock::SeatLease>> {
9241 let result =
9242 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9243 .await;
9244 if let Err(e) = &result {
9245 record_repository_block(state, e);
9246 }
9247 result
9248}
9249
9250async fn run_one(
9252 job: SeatJob,
9253 sem: Arc<Semaphore>,
9254 ctx: &WaveCtx<'_>,
9255 state: &mut RunState,
9256 attempt: usize,
9257) -> (SeatState, AgentOutcome) {
9258 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9259 .await
9260 .pop()
9261 .expect("one job in, one result out");
9262 (seat, out)
9263}
9264
9265async fn wave(
9271 jobs: Vec<SeatJob>,
9272 sem: Arc<Semaphore>,
9273 ctx: &WaveCtx<'_>,
9274 state: &mut RunState,
9275 attempt: usize,
9276) -> Vec<(usize, SeatState, AgentOutcome)> {
9277 let WaveCtx {
9278 run,
9279 node,
9280 prompts,
9281 cache,
9282 round,
9283 carry_seats: _,
9284 pauses,
9285 } = *ctx;
9286 for job in &jobs {
9287 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9288 }
9289 if let Err(e) = state.save() {
9290 tracing::warn!("could not persist in-progress seats: {e:#}");
9295 }
9296 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9312 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9316 let cache_guard = if let Some(cache_dir) = cache {
9317 if jobs_had_a_writer {
9318 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9319 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9320 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9321 {
9322 Ok(Some(g)) => Some(g),
9323 Ok(None) => {
9326 state.resource_block = Some(crate::run::ResourceBlock {
9327 node: node.to_owned(),
9328 kind: "shared build cache".to_owned(),
9329 detail: "the run is parking".to_owned(),
9330 at: Timestamp::now(),
9331 });
9332 return block_wave(
9333 jobs,
9334 state,
9335 node,
9336 attempt,
9337 "parked while waiting for the shared build cache",
9338 );
9339 }
9340 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9343 Err(e) => {
9344 let why = format!("{e:#}");
9348 state.resource_block = Some(crate::run::ResourceBlock {
9349 node: node.to_owned(),
9350 kind: "shared build cache".to_owned(),
9351 detail: why.clone(),
9352 at: Timestamp::now(),
9353 });
9354 state.event(
9355 "resource",
9356 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9357 );
9358 return block_wave(jobs, state, node, attempt, &why);
9359 }
9360 }
9361 } else {
9362 None
9363 }
9364 } else {
9365 None
9366 };
9367 state.seats_spawning(node, attempt);
9370 if let Err(e) = state.save() {
9371 tracing::warn!("could not persist the start of a wave: {e:#}");
9372 }
9373 let slot_limit = resource_limit
9377 + jobs
9378 .iter()
9379 .map(|j| j.timeout)
9380 .max()
9381 .unwrap_or_default()
9382 .saturating_mul(jobs.len() as u32);
9383 let mut set = tokio::task::JoinSet::new();
9384 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9387 let overlay = prompts.overlay(node);
9388 let repository_wait = repository_wait(state, pauses);
9389 for (i, mut job) in jobs.into_iter().enumerate() {
9390 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9391 if cache.is_some() {
9392 job.prompt.push('\n');
9393 job.prompt
9394 .push_str(&prompt::build_cache_note(node, job.allow_write));
9395 }
9396 let sem = Arc::clone(&sem);
9397 let run = run.to_owned();
9398 let node = node.to_owned();
9399 let attachments = if node == "implement" {
9402 state.attachments.clone()
9403 } else {
9404 Vec::new()
9405 };
9406 let cache = cache
9417 .filter(|_| job.allow_write && cache_guard.is_some())
9418 .map(Path::to_path_buf);
9419 let spawned_tx = spawned_tx.clone();
9420 let repository_wait = repository_wait.clone();
9421 let coordination = state.config.coordination.clone();
9422 let fetched_base = state.fetched_base.clone();
9423 set.spawn(async move {
9424 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9428 Ok(p) => p,
9429 Err(_) => {
9430 return (
9431 i,
9432 job.seat,
9433 AgentOutcome::ResourceBlocked(format!(
9434 "no parallel slot became free within {}s",
9435 slot_limit.as_secs()
9436 )),
9437 false,
9438 );
9439 }
9440 };
9441 spawned_tx.send(job.seat.key.clone()).ok();
9442 let mut seat = job.seat;
9443 let out = repository_wait
9444 .scope(agent::invoke(
9445 &job.spec,
9446 &mut seat,
9447 &Invocation {
9448 scope: crate::agent::InvocationScope::Repository,
9449 coordination: coordination.as_ref(),
9450 fetched_base: fetched_base.as_ref(),
9451 cwd: &job.cwd,
9452 prompt: &job.prompt,
9453 timeout: job.timeout,
9454 allow_write: job.allow_write,
9455 unsandboxed: false,
9456 sessions: job.sessions,
9457 artifacts: &job.artifacts,
9458 stem: &job.stem,
9459 run: &run,
9460 node: &node,
9461 cache_dir: cache.as_deref(),
9462 attachments: &attachments,
9463 writable: &[],
9464 },
9465 ))
9466 .await;
9467 let interrupted = out
9468 .as_ref()
9469 .err()
9470 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9471 let out = match out {
9472 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9473 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9474 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9482 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9483 Ok(o) => AgentOutcome::Failed(format!(
9484 "process exited with no usable output ({})",
9485 crate::notices::exit_status(o.exit_code)
9486 )),
9487 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9488 AgentOutcome::ResourceBlocked(e.to_string())
9489 }
9490 Err(e) => AgentOutcome::Failed(e.to_string()),
9491 };
9492 (i, seat, out, interrupted)
9493 });
9494 }
9495 drop(spawned_tx);
9496 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9497 loop {
9498 let joined = tokio::select! {
9499 Some(key) = spawned_rx.recv() => {
9500 state.seat_spawned(&key);
9502 if let Err(e) = state.save() {
9503 tracing::warn!("could not persist a seat's start: {e:#}");
9504 }
9505 continue;
9506 }
9507 joined = set.join_next() => joined,
9508 };
9509 let Some(joined) = joined else {
9510 break;
9511 };
9512 let (i, seat, out, interrupted) = match joined {
9513 Ok(v) => v,
9514 Err(e) => {
9518 tracing::error!("agent task panicked: {e}");
9519 continue;
9520 }
9521 };
9522 state.seat_finished(&seat.key);
9523 if interrupted
9524 && node == "implement"
9525 && let Some(label) = state
9526 .candidates
9527 .iter()
9528 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9529 .map(|candidate| candidate.label)
9530 {
9531 state.candidate_preservation.insert(
9532 label,
9533 "interrupted implementation awaiting persistence".into(),
9534 );
9535 state.interrupted_implementations.insert(label);
9536 }
9537 if let AgentOutcome::ResourceBlocked(why) = &out
9538 && state.resource_block.is_none()
9539 {
9540 state.resource_block = Some(crate::run::ResourceBlock {
9541 node: node.to_owned(),
9542 kind: if why.contains("disk pressure") {
9543 "disk pressure"
9544 } else if why.contains("repository coordination") {
9545 "repository coordination"
9546 } else {
9547 "parallel slot"
9548 }
9549 .to_owned(),
9550 detail: why.clone(),
9551 at: Timestamp::now(),
9552 });
9553 let activity = if interrupted {
9554 "agent execution interrupted"
9555 } else {
9556 "no agent was started"
9557 };
9558 state.event("resource", format!("{node}: {activity} - {why}"));
9559 }
9560 record_jobs(state, node, round, &seat.key, &out);
9561 if let Err(e) = state.save() {
9562 tracing::warn!("could not persist a seat's completion: {e:#}");
9563 }
9564 if collected.len() <= i {
9565 collected.resize_with(i + 1, || None);
9566 }
9567 collected[i] = Some((i, seat, out));
9568 }
9569 if state
9575 .active
9576 .values()
9577 .any(|a| a.node == node && a.attempt == attempt)
9578 {
9579 state
9580 .active
9581 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9582 if let Err(e) = state.save() {
9583 tracing::warn!("could not persist the end of a wave: {e:#}");
9584 }
9585 }
9586 if let Some(cache_dir) = cache
9593 && jobs_had_a_writer
9594 {
9595 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9596 }
9597 if let Some(guard) = cache_guard {
9598 guard.release();
9599 }
9600 collected.into_iter().flatten().collect()
9601}
9602
9603fn record_jobs(
9614 state: &mut RunState,
9615 node: &str,
9616 round: Option<usize>,
9617 seat: &str,
9618 out: &AgentOutcome,
9619) {
9620 let commands: &[agent::CommandEvidence] = match out {
9621 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9622 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9623 };
9624 let checked_at = Timestamp::now();
9625 for c in commands {
9626 state.jobs.push(JobRecord {
9627 node: node.to_owned(),
9628 round,
9629 seat: seat.to_owned(),
9630 id: c.id.clone(),
9631 description: c.description.clone(),
9632 checked_at,
9633 status: match c.exit_code {
9634 Some(0) => JobStatus::Completed,
9635 Some(_) => JobStatus::Failed,
9636 None => JobStatus::Unknown,
9637 },
9638 exit_code: c.exit_code,
9639 result_summary: c.result_summary.clone(),
9640 source: c.source.clone(),
9641 });
9642 }
9643}
9644
9645fn round_is_clean(
9666 blocking: usize,
9667 e2e_ok: bool,
9668 answered: usize,
9669 expected: usize,
9670 quota_missing: usize,
9671 policy: IncompleteReviewPolicy,
9672) -> bool {
9673 if blocking != 0 || !e2e_ok {
9674 return false;
9675 }
9676 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9677 return true;
9678 }
9679 answered > 0 && expected - answered <= quota_missing
9680}
9681
9682fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9706 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9707 return Some(RunStatus::Gating);
9708 }
9709 let last = reviews.last()?;
9710 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9711 if reviews.len() < max_rounds && !stagnant {
9712 return None;
9713 }
9714 if last.incomplete() && last.blocking == 0 {
9715 return Some(RunStatus::Blocked);
9716 }
9717 if last.e2e_status() == E2eStatus::ResourceBlocked {
9718 return None;
9719 }
9720 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9721 RunStatus::Gating
9722 } else {
9723 RunStatus::Blocked
9724 })
9725}
9726
9727pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9742 if nudged {
9743 (full / 4).max(Duration::from_secs(120)).min(full)
9744 } else {
9745 full
9746 }
9747}
9748
9749#[allow(clippy::too_many_arguments)]
9778async fn ask_json_wave<T>(
9779 jobs: Vec<SeatJob>,
9780 sem: Arc<Semaphore>,
9781 retries: usize,
9782 roster: &[AgentSpec],
9783 ctx: &WaveCtx<'_>,
9784 losses: &mut Vec<QuotaLoss>,
9785 state: &mut RunState,
9786 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9787) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9788where
9789 T: serde::de::DeserializeOwned + Send + 'static,
9790{
9791 ask_wave_with(
9792 jobs,
9793 sem,
9794 retries,
9795 roster,
9796 ctx,
9797 losses,
9798 state,
9799 &|text: &str| {
9800 let v = verdict::extract_json::<T>(text)?;
9801 validate(&v)?;
9802 Ok(v)
9803 },
9804 )
9805 .await
9806}
9807
9808#[allow(clippy::too_many_arguments)]
9817async fn ask_wave_with<T>(
9818 jobs: Vec<SeatJob>,
9819 sem: Arc<Semaphore>,
9820 retries: usize,
9821 roster: &[AgentSpec],
9822 ctx: &WaveCtx<'_>,
9823 losses: &mut Vec<QuotaLoss>,
9824 state: &mut RunState,
9825 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9826) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9827where
9828 T: Send + 'static,
9829{
9830 let n = jobs.len();
9831 let originals: Vec<SeatJob> = jobs;
9832 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9833 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9834 let mut nudges: Vec<usize> = vec![0; n];
9841 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9845 let mut tried: Vec<BTreeSet<String>> = specs
9846 .iter()
9847 .map(|s| BTreeSet::from([s.id.clone()]))
9848 .collect();
9849 let starts: Vec<usize> = specs
9850 .iter()
9851 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9852 .collect();
9853 let carry = ctx.carry_seats && !roster.is_empty();
9854 let carried: Vec<BTreeSet<String>> = originals
9857 .iter()
9858 .map(|j| {
9859 state
9860 .seat_history
9861 .get(&j.seat.key)
9862 .filter(|_| carry)
9863 .map(|h| h.failed.clone())
9864 .unwrap_or_default()
9865 })
9866 .collect();
9867 let mut prev: Vec<Option<FailClass>> = originals
9868 .iter()
9869 .map(|j| {
9870 state
9871 .seat_history
9872 .get(&j.seat.key)
9873 .filter(|_| carry)
9874 .and_then(|h| h.last_fail.clone())
9875 })
9876 .collect();
9877 let next_agent =
9878 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9879 let others: BTreeSet<String> = specs
9880 .iter()
9881 .enumerate()
9882 .filter(|(j, _)| *j != i)
9883 .map(|(_, s)| s.id.clone())
9884 .collect();
9885 pick_successor(
9886 roster,
9887 starts[i],
9888 tried,
9889 carry.then(|| &carried[i]),
9890 &others,
9891 )
9892 .cloned()
9893 };
9894 let mut fresh: Vec<Option<String>> = vec![None; n];
9895 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9896 let mut pending: Vec<usize> = (0..n).collect();
9897
9898 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9903 for round in 0..max_rounds {
9904 if pending.is_empty() {
9905 break;
9906 }
9907 let mut batch = Vec::with_capacity(pending.len());
9908 let mut renudged: Vec<&str> = Vec::new();
9909 for &i in &pending {
9910 let src = &originals[i];
9911 let full: &str = match &src.handover {
9916 Some(h) if specs[i].id != src.spec.id => h,
9917 _ => &src.prompt,
9918 };
9919 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9922 (
9923 full.to_owned(),
9924 src.timeout,
9925 format!("{}-{word}-{}", src.stem, specs[i].id),
9926 )
9927 } else if nudges[i] == 0 {
9928 (full.to_owned(), src.timeout, src.stem.clone())
9929 } else {
9930 renudged.push(src.seat.key.as_str());
9931 let why = done[i]
9932 .as_ref()
9933 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9934 .unwrap_or_else(|| "no parsable answer".to_owned());
9935 let nudge = prompt::nudge(&why);
9936 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9937 let prompt = if nudged {
9938 nudge
9939 } else {
9940 format!("{full}\n\n---\n\n{nudge}")
9941 };
9942 (
9943 prompt,
9944 retry_budget(src.timeout, nudged),
9945 format!("{}-retry{}", src.stem, nudges[i]),
9946 )
9947 };
9948 batch.push(SeatJob {
9949 spec: specs[i].clone(),
9950 seat: seats[i].clone(),
9951 cwd: src.cwd.clone(),
9952 prompt,
9953 timeout,
9954 allow_write: src.allow_write,
9955 sessions: src.sessions,
9956 artifacts: src.artifacts.clone(),
9957 stem,
9958 handover: None,
9959 });
9960 }
9961
9962 if !renudged.is_empty() {
9963 state.event(
9964 ctx.node,
9965 format!("retry {round}: re-asking {}", renudged.join(", ")),
9966 );
9967 }
9968 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9969 let mut still = Vec::new();
9970 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9971 seats[i] = seat;
9972 if let AgentOutcome::ResourceBlocked(why) = &out {
9975 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9976 continue;
9977 }
9978 let class = FailClass::of(&out);
9979 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9982 && nudges[i] < retries
9983 && !roster.is_empty();
9984 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9985 let next = should_hand_over(prev[i].as_ref(), &cur)
9986 .then(|| next_agent(i, &tried[i], &specs))
9987 .flatten();
9988 if carry {
9989 let h = state
9990 .seat_history
9991 .entry(originals[i].seat.key.clone())
9992 .or_default();
9993 h.failed.insert(specs[i].id.clone());
9994 h.last_fail = Some(cur.clone());
9995 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9996 h.last_ok = None;
9997 }
9998 if let Err(e) = state.save() {
10001 tracing::warn!("could not persist a seat's failure history: {e:#}");
10002 }
10003 }
10004 if let Some(next) = next {
10005 record_handover(
10006 state,
10007 ctx.node,
10008 &originals[i].seat.key,
10009 &specs[i].id,
10010 &next.id,
10011 &cur,
10012 &fail_reason(&out),
10013 );
10014 tried[i].insert(next.id.clone());
10015 prev[i] = Some(cur.clone());
10016 seats[i] =
10017 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
10018 specs[i] = next;
10019 fresh[i] = Some(cur.stem_word().to_owned());
10020 nudges[i] = 0;
10021 done[i] = Some(Err(anyhow::anyhow!(
10022 "handed over after: {}",
10023 fail_reason(&out)
10024 )));
10025 still.push(i);
10026 continue;
10027 }
10028 }
10029 if carry
10030 && !nudge_first
10031 && let Some(cur) = class.clone()
10032 {
10033 let h = state
10035 .seat_history
10036 .entry(originals[i].seat.key.clone())
10037 .or_default();
10038 h.failed.insert(specs[i].id.clone());
10039 h.last_fail = Some(cur);
10040 }
10041 let parsed = match out {
10042 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10043 AgentOutcome::Quota(o) => {
10044 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10045 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10046 }
10047 AgentOutcome::Dropped(o) => {
10051 let why = o
10052 .dropped
10053 .as_ref()
10054 .map(|d| d.why.as_str())
10055 .unwrap_or("the CLI ended the stream without delivering its answer");
10056 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10057 }
10058 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10059 Err(anyhow::anyhow!(e))
10060 }
10061 };
10062 let quota = class == Some(FailClass::Quota);
10063 let failed = parsed.is_err();
10064 done[i] = Some(parsed);
10065 if carry && !failed {
10066 let h = state
10067 .seat_history
10068 .entry(originals[i].seat.key.clone())
10069 .or_default();
10070 h.failed.remove(&specs[i].id);
10071 h.last_ok = Some(specs[i].id.clone());
10072 h.last_fail = None;
10073 }
10074 let agent_failure = class.is_some()
10082 && !nudge_first
10083 && !roster.is_empty()
10084 && next_agent(i, &tried[i], &specs).is_some();
10085 if failed && !quota && !agent_failure && nudges[i] < retries {
10086 nudges[i] += 1;
10087 still.push(i);
10088 }
10089 }
10090 pending = still;
10091 }
10092
10093 for (i, q) in last_quota.into_iter().enumerate() {
10096 if let Some(reset) = q {
10097 losses.push(QuotaLoss {
10098 seat: originals[i].seat.key.clone(),
10099 node: ctx.node.to_owned(),
10100 at: Timestamp::now(),
10101 reset,
10102 });
10103 }
10104 }
10105
10106 seats
10107 .into_iter()
10108 .zip(done)
10109 .zip(nudges)
10110 .map(|((seat, res), attempts)| {
10111 (
10112 seat,
10113 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10114 attempts,
10115 )
10116 })
10117 .collect()
10118}
10119
10120fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10122 results
10123 .iter()
10124 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10125}
10126
10127fn block_wave(
10131 jobs: Vec<SeatJob>,
10132 state: &mut RunState,
10133 node: &str,
10134 attempt: usize,
10135 why: &str,
10136) -> Vec<(usize, SeatState, AgentOutcome)> {
10137 state.resource_wait = None;
10138 state
10139 .active
10140 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10141 if let Err(e) = state.save() {
10142 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10143 }
10144 jobs.into_iter()
10145 .enumerate()
10146 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10147 .collect()
10148}
10149
10150async fn acquire_cache_lease(
10163 state: &mut RunState,
10164 cache_dir: &Path,
10165 owner: &crate::cache::Owner,
10166 budget: Duration,
10167 context: &str,
10168 cancel: &(dyn Fn() -> bool + Sync),
10169) -> Result<Option<crate::cache::Guard>> {
10170 let home = crate::run::home();
10171 let started = Instant::now();
10172 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10173 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10174 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10175 Err(e) => {
10176 state.event(
10177 "verify",
10178 format!("{context}: could not check the shared build cache: {e:#}"),
10179 );
10180 if let Err(e2) = state.save() {
10181 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10182 }
10183 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10184 }
10185 };
10186 state.event(
10187 "verify",
10188 format!(
10189 "{context}: waiting for the shared build cache at {} ({})",
10190 cache_dir.display(),
10191 busy.describe()
10192 ),
10193 );
10194 state.resource_wait = Some(crate::run::ResourceWait {
10197 node: context.to_owned(),
10198 kind: "shared build cache".to_owned(),
10199 holder: busy.describe(),
10200 since: Timestamp::now(),
10201 limit_secs: budget.as_secs(),
10202 });
10203 if let Err(e) = state.save() {
10204 tracing::warn!("could not persist a cache wait: {e:#}");
10205 }
10206 let remaining = budget.saturating_sub(started.elapsed());
10207 let waited = crate::cache::wait_for_unless(
10208 &home,
10209 cache_dir,
10210 owner,
10211 remaining,
10212 Duration::from_secs(5),
10213 cancel,
10214 )
10215 .await;
10216 state.resource_wait = None;
10217 match waited {
10218 Ok(g) => Ok(g),
10219 Err(e) => {
10220 state.event("verify", format!("{context}: {e:#}"));
10221 if let Err(e2) = state.save() {
10222 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10223 }
10224 Err(e)
10225 }
10226 }
10227}
10228
10229#[derive(Debug)]
10234struct CacheCheckError(String);
10235
10236impl std::fmt::Display for CacheCheckError {
10237 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10238 f.write_str(&self.0)
10239 }
10240}
10241
10242impl std::error::Error for CacheCheckError {}
10243
10244#[allow(clippy::too_many_arguments)]
10265async fn with_cache_lease<'s, F, Fut>(
10266 state: &'s mut RunState,
10267 cache_dir: Option<&Path>,
10268 node: &str,
10269 seat: &str,
10270 worktree: &Path,
10271 head: &str,
10272 budget: Duration,
10273 context: &str,
10274 wait: &crate::repo_lock::WaitPolicy,
10275 body: F,
10276) -> Result<(Vec<CommandOutcome>, bool)>
10277where
10278 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10279 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10280{
10281 let started = Instant::now();
10284 let home = crate::run::home();
10285 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10286 let guard = if let Some(cache_dir) = cache_dir {
10287 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10288 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10289 wait.cancelled()
10290 })
10291 .await
10292 .and_then(|g| {
10293 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10294 }) {
10295 Ok(g) => Some(g),
10296 Err(e) => {
10297 return Ok((
10298 vec![CommandOutcome {
10299 command: "(waiting for the shared build cache)".into(),
10300 code: None,
10301 output_tail: e.to_string(),
10302 duration_ms: started.elapsed().as_millis() as u64,
10303 resource_blocked: true,
10304 }],
10305 false,
10306 ));
10307 }
10308 }
10309 } else {
10310 None
10311 };
10312 let _coordination = coordinate_operation(state, worktree, wait).await?;
10313 if let Some(cache_dir) = cache_dir {
10314 let identity = crate::cache::Identity::new(worktree, head);
10315 if let Err(e) = crate::cache::ensure_fresh(
10316 &home,
10317 cache_dir,
10318 &identity,
10319 &state.config.verify.cargo_manifests(),
10320 ) {
10321 state.event(
10322 "verify",
10323 format!(
10324 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10325 worktree.display(),
10326 short(head)
10327 ),
10328 );
10329 if let Some(guard) = guard {
10330 guard.release();
10331 }
10332 return Ok((
10333 vec![CommandOutcome {
10334 command: "(confirming the shared build cache is fresh)".into(),
10335 code: None,
10336 output_tail: e.to_string(),
10337 duration_ms: started.elapsed().as_millis() as u64,
10338 resource_blocked: true,
10339 }],
10340 false,
10341 ));
10342 }
10343 }
10344 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10345 if !timed_out_pids.is_empty() {
10346 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10347 }
10348 drop(_coordination);
10351 if let Some(guard) = guard {
10352 guard.release();
10353 }
10354 Ok((outcomes, retried))
10355}
10356
10357async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10369 wait_for_pids_with(
10370 pids,
10371 crate::proc::pid_alive,
10372 LEASE_RELEASE_POLL,
10373 LEASE_RELEASE_MAX_WAIT,
10374 )
10375 .await;
10376}
10377
10378async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10384 pids: &[u32],
10385 alive: F,
10386 poll: Duration,
10387 max_wait: Duration,
10388) {
10389 let deadline = Instant::now() + max_wait;
10390 loop {
10391 if pids.iter().all(|&pid| !alive(pid)) {
10392 return;
10393 }
10394 if Instant::now() >= deadline {
10395 return;
10396 }
10397 tokio::time::sleep(poll).await;
10398 }
10399}
10400
10401fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10408 outcomes.iter().any(|o| o.resource_blocked)
10409}
10410
10411enum GateFix {
10413 Retry,
10415 Stop,
10418 Defer,
10421}
10422
10423fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10431 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10432 red.peek().is_some()
10433 && red.all(|o| {
10434 !o.resource_blocked
10435 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10436 && !o.output_tail.trim().is_empty()
10437 })
10438}
10439
10440fn e2e_outcome_label(o: &CommandOutcome) -> String {
10444 if o.ok() {
10445 return "pass".to_owned();
10446 }
10447 let reason = if o.build_failed() {
10448 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10449 } else {
10450 format!("FAIL ({:?})", o.code)
10451 };
10452 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10453}
10454
10455async fn run_e2e_with_retry(
10463 state: &mut RunState,
10464 shell: &[String],
10465 commands: &[String],
10466 worktree: &Path,
10467 timeout: Duration,
10468 context: &str,
10469) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10470 let (mut e2e, mut timed_out_pids) = run_commands(
10471 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10472 )
10473 .await;
10474 for o in &e2e {
10475 state.event(
10476 "verify",
10477 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10478 );
10479 }
10480 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10484 if verify_retried {
10485 state.event(
10486 "verify",
10487 format!(
10488 "{context}: verify could not build/link, not a test result — retrying once \
10489 before concluding"
10490 ),
10491 );
10492 let retried = run_commands(
10493 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10494 )
10495 .await;
10496 e2e = retried.0;
10497 timed_out_pids.extend(retried.1);
10500 for o in &e2e {
10501 state.event(
10502 "verify",
10503 format!(
10504 "{context}: retry `{}` -> {}",
10505 o.command,
10506 e2e_outcome_label(o)
10507 ),
10508 );
10509 }
10510 }
10511 (e2e, verify_retried, timed_out_pids)
10512}
10513
10514#[allow(clippy::too_many_arguments)]
10530async fn run_commands(
10531 state: &mut RunState,
10532 node: &str,
10533 task: &str,
10534 attempt: usize,
10535 shell: &[String],
10536 commands: &[String],
10537 cwd: &Path,
10538 timeout: Duration,
10539) -> (Vec<CommandOutcome>, Vec<u32>) {
10540 if commands.is_empty() {
10541 return (Vec::new(), Vec::new());
10546 }
10547 let mut out = Vec::new();
10548 let mut timed_out_pids = Vec::new();
10549 let total = commands.len();
10550 for (idx, command) in commands.iter().enumerate() {
10551 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10552 if let Err(e) = state.save() {
10553 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10554 }
10555 let started = Instant::now();
10556 let mut cmd = tokio::process::Command::new(&shell[0]);
10557 cmd.quiet();
10558 cmd.args(&shell[1..])
10559 .arg(command)
10560 .current_dir(cwd)
10561 .stdin(std::process::Stdio::null())
10562 .stdout(std::process::Stdio::piped())
10563 .stderr(std::process::Stdio::piped())
10564 .kill_on_drop(true);
10565 let mut outputs = Vec::new();
10566 let protected = (|| -> Result<()> {
10567 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10568 outputs.push(guard);
10569 }
10570 if let Some(cache) = state.config.cache_dir() {
10571 outputs.push(crate::retention::OutputLease::cache(
10572 &crate::run::home(),
10573 &cache,
10574 false,
10575 )?);
10576 }
10577 Ok(())
10578 })();
10579 if let Err(e) = protected {
10580 out.push(CommandOutcome {
10581 command: command.clone(),
10582 code: None,
10583 output_tail: format!("output ownership unavailable: {e:#}"),
10584 duration_ms: 0,
10585 resource_blocked: true,
10586 });
10587 continue;
10588 }
10589 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10590 let (code, body) = match spawned {
10591 Ok(mut running) => {
10592 let pid = running.child().id();
10593 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10594 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10595 let mut paths = vec![cwd.to_owned()];
10596 if let Some(cache) = state.config.cache_dir() {
10597 paths.push(cache);
10598 }
10599 let mut pressure = None;
10600 let status = tokio::select! {
10601 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10602 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10603 pressure = Some(reason); None
10604 }
10605 };
10606 let (code, failure) = match status {
10607 Some(Ok(Ok(status))) => (
10608 status.code(),
10609 status
10610 .code()
10611 .is_none()
10612 .then(|| format!("command terminated: {status}")),
10613 ),
10614 Some(Ok(Err(error))) => {
10615 (None, Some(format!("failed waiting for command: {error}")))
10616 }
10617 Some(Err(_)) => {
10618 if !running.coordinated()
10622 && let Some(pid) = pid
10623 {
10624 timed_out_pids.push(pid);
10625 }
10626 (
10627 None,
10628 Some(format!("timed out after {}s", timeout.as_secs())),
10629 )
10630 }
10631 None => (None, None),
10632 };
10633 running.finish().await;
10634 if let Some(reason) = pressure {
10635 state.resource_block = Some(crate::run::ResourceBlock {
10636 node: node.to_owned(),
10637 kind: "disk pressure".to_owned(),
10638 detail: reason.clone(),
10639 at: Timestamp::now(),
10640 });
10641 out.push(CommandOutcome {
10642 command: command.clone(),
10643 code: None,
10644 output_tail: reason,
10645 duration_ms: started.elapsed().as_millis() as u64,
10646 resource_blocked: true,
10647 });
10648 break;
10649 }
10650 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10651 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10652 if let Some(failure) = failure {
10653 body.push('\n');
10654 body.push_str(&failure);
10655 }
10656 (code, body)
10657 }
10658 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10659 };
10660 out.push(CommandOutcome {
10661 command: command.clone(),
10662 code,
10663 output_tail: tail(&body, OUTPUT_TAIL),
10664 duration_ms: started.elapsed().as_millis() as u64,
10665 resource_blocked: false,
10666 });
10667 }
10668 state.task_finished(task);
10669 if let Err(e) = state.save() {
10670 tracing::warn!("could not persist the end of {task}: {e:#}");
10671 }
10672 (out, timed_out_pids)
10673}
10674
10675fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10687 let repo = repo.display();
10688 match style {
10689 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10690 MergeStyle::Squash => {
10691 let subject = message
10694 .lines()
10695 .next()
10696 .unwrap_or(branch)
10697 .replace(['\\', '"', '$', '`'], "");
10698 format!(
10699 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10700 )
10701 }
10702 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10703 }
10704}
10705
10706const PR_TITLE_MAX: usize = 240;
10717
10718fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10729 let line = queue::first_line(text)?;
10730 let chars: Vec<char> = line.chars().collect();
10731 let (kept, mark) = if chars.len() <= max {
10732 (line.to_owned(), "")
10733 } else if let Some(end) = sentence_end(&chars, max) {
10734 (chars[..end].iter().collect::<String>(), "")
10735 } else {
10736 let room = max.saturating_sub(3);
10737 let cut = if chars[room].is_whitespace() {
10740 room
10741 } else {
10742 chars[..room]
10743 .iter()
10744 .rposition(|c| c.is_whitespace())
10745 .unwrap_or(room)
10746 };
10747 let head: String = chars[..cut].iter().collect();
10748 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10749 (head.to_owned(), "...")
10750 };
10751 if kept.is_empty() {
10752 return None;
10753 }
10754 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10755 return None;
10756 }
10757 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10758 return None;
10759 }
10760 Some(format!("{kept}{mark}"))
10761}
10762
10763fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10767 const MIN: usize = 20;
10768 for i in MIN..max.min(chars.len()) {
10769 if !matches!(chars[i], '.' | '!' | '?') {
10770 continue;
10771 }
10772 let Some(&next) = chars.get(i + 1) else {
10773 continue;
10774 };
10775 if !next.is_whitespace() {
10776 continue;
10777 }
10778 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10779 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10780 continue;
10781 }
10782 let word: String = chars[..i]
10783 .iter()
10784 .rev()
10785 .take_while(|c| !c.is_whitespace())
10786 .collect::<Vec<_>>()
10787 .into_iter()
10788 .rev()
10789 .collect();
10790 let word = word.to_ascii_lowercase();
10791 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10792 continue;
10793 }
10794 let end = chars[..i]
10795 .iter()
10796 .rposition(|c| !c.is_whitespace())
10797 .map_or(i, |p| p + 1);
10798 return Some(end);
10799 }
10800 None
10801}
10802
10803struct PrMessage {
10808 title: String,
10809 body: String,
10810}
10811
10812impl PrMessage {
10813 fn commit_message(&self) -> String {
10817 format!("{}\n\n{}", self.title, self.body)
10818 }
10819}
10820
10821fn title_marker(line: &str) -> Option<&str> {
10823 let line = line.trim();
10824 let head = line.get(..6)?;
10825 head.eq_ignore_ascii_case("title:")
10826 .then(|| line[6..].trim())
10827}
10828
10829fn summary_title(summary: &str) -> Option<String> {
10834 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10835 let raw = title_marker(first)?;
10836 if raw.is_empty() {
10837 return None;
10838 }
10839 let title = queue::title_from(raw, PR_TITLE_MAX);
10840 let lower = title.to_ascii_lowercase();
10841 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10842 return None;
10843 }
10844 Some(title)
10845}
10846
10847const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10849
10850pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10852
10853fn is_review_run(state: &RunState) -> bool {
10857 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10858}
10859
10860fn review_title(state: &RunState) -> Option<String> {
10865 english_subject(
10866 state.reviewed_commits.as_ref()?.first()?,
10867 state.config.graph.github_english(),
10868 )
10869}
10870
10871fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10874 let raw = raw.trim();
10875 if raw.is_empty() {
10876 return None;
10877 }
10878 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10879 return None;
10880 }
10881 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10882 return None;
10883 }
10884 let title = queue::title_from(raw, PR_TITLE_MAX);
10885 let lower = title.to_ascii_lowercase();
10886 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10887 return None;
10888 }
10889 Some(title)
10890}
10891
10892#[derive(Debug, Clone, PartialEq, Eq)]
10895struct BranchFacts {
10896 commits: Vec<(String, String)>,
10898 stat: String,
10900}
10901
10902const STAT_FILE_LINES: usize = 25;
10904const COMMITS_MAX_CHARS: usize = 20_000;
10906
10907async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10910 let commits = git::commit_log(repo, base, branch).await.ok()?;
10911 if commits.is_empty() {
10912 return None;
10913 }
10914 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10915 let lines: Vec<&str> = stat.lines().collect();
10916 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10917 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10918 let more = format!(" ... {omitted} more file(s)");
10919 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10920 kept.push(&more);
10921 kept.push(lines[lines.len() - 1]);
10922 kept.join("\n")
10923 } else {
10924 lines.join("\n")
10925 };
10926 Some(BranchFacts { commits, stat })
10927}
10928
10929fn markdown_safe(text: &str) -> String {
10932 text.replace("</details>", "</details>")
10933 .replace("\x60\x60\x60", "~~~")
10934}
10935
10936fn neutral_title(state: &RunState, winner: char) -> String {
10937 format!(
10938 "chore: land candidate {} of run {}",
10939 winner.to_ascii_uppercase(),
10940 state.id
10941 )
10942}
10943
10944pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10950 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10951 ""
10952 } else {
10953 pr_title
10954 }
10955}
10956
10957pub fn landing_subject_source(state: &RunState) -> String {
10961 if is_review_run(state) {
10962 let winner = state.candidates.first().map_or('A', |c| c.label);
10963 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10964 }
10965 state.instruction.clone()
10966}
10967
10968fn summary_without_title(summary: &str) -> String {
10971 let mut lines = summary.trim().lines().peekable();
10972 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10973 lines.next();
10974 }
10975 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10976}
10977
10978#[cfg(test)]
10990fn pr_message(state: &RunState, winner: char) -> PrMessage {
10991 pr_message_with(state, winner, None)
10992}
10993
10994#[cfg(test)]
10997fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10998 let raw = pr_message_raw(
10999 state,
11000 winner,
11001 facts,
11002 crate::pr_outcome::FinalHead::Unreported,
11003 ChangeNote::NotAsked,
11004 );
11005 let id = crate::scrub::Identity::current();
11006 PrMessage {
11007 title: crate::scrub::scrub(&raw.title, &id),
11008 body: crate::scrub::scrub(&raw.body, &id),
11009 }
11010}
11011
11012#[derive(Debug, Clone, Copy)]
11014enum ChangeNote<'a> {
11015 NotAsked,
11017 Summary(&'a crate::run::ChangeSummary),
11019 Unavailable(&'a str),
11021}
11022
11023fn pr_message_raw(
11024 state: &RunState,
11025 winner: char,
11026 facts: Option<&BranchFacts>,
11027 head: crate::pr_outcome::FinalHead<'_>,
11028 change: ChangeNote<'_>,
11029) -> PrMessage {
11030 if !matches!(change, ChangeNote::NotAsked) {
11031 let (title, mut body) = match change {
11032 ChangeNote::Summary(c) => (
11033 c.title.clone(),
11034 format!("## Summary\n\n{}\n\n", c.body.trim()),
11035 ),
11036 ChangeNote::Unavailable(why) => (
11037 neutral_title(state, winner),
11038 format!(
11039 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11040 ),
11041 ),
11042 ChangeNote::NotAsked => unreachable!(),
11043 };
11044 body.push_str(&public_pr_evidence(state, head));
11045 body.push_str(&format!(
11046 "\n---\nmagi:run/{} magi:candidate-{}\n",
11047 state.id,
11048 winner.to_ascii_lowercase()
11049 ));
11050 return PrMessage { title, body };
11051 }
11052 pr_preview(state, winner, facts, head)
11053}
11054
11055fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11058 let mut body = crate::pr_outcome::review_rounds(state);
11059 body.push_str(&crate::pr_outcome::final_verification(state, head));
11060 let open = state.last_round_findings();
11061 if !open.is_empty() {
11062 body.push_str("## Open review findings\n\n");
11063 for f in open {
11064 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11065 }
11066 body.push('\n');
11067 }
11068 if let Some(c) = state.change_summary.as_ref()
11069 && head == crate::pr_outcome::FinalHead::At(&c.head)
11070 {
11071 body.push_str(&format!(
11072 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11073 c.base, c.head
11074 ));
11075 }
11076 body
11077}
11078
11079fn pr_preview(
11081 state: &RunState,
11082 winner: char,
11083 facts: Option<&BranchFacts>,
11084 head: crate::pr_outcome::FinalHead<'_>,
11085) -> PrMessage {
11086 let summary = state
11087 .candidates
11088 .iter()
11089 .find(|c| c.label == winner)
11090 .map(|c| c.summary.as_str())
11091 .unwrap_or_default();
11092 let review = is_review_run(state);
11095 let title = if review {
11096 facts
11097 .and_then(|f| {
11098 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11099 })
11100 .or_else(|| review_title(state))
11101 .or_else(|| {
11102 state
11103 .candidates
11104 .iter()
11105 .find(|c| c.label == winner)
11106 .filter(|c| !c.branch.starts_with("magi/"))
11107 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11108 })
11109 .unwrap_or_else(|| neutral_title(state, winner))
11110 } else {
11111 summary_title(summary).unwrap_or_else(|| {
11112 english_title(
11113 &state.instruction,
11114 PR_TITLE_MAX,
11115 state.config.graph.github_english(),
11116 )
11117 .unwrap_or_else(|| neutral_title(state, winner))
11118 })
11119 };
11120
11121 let mut body = String::new();
11122 let what = summary_without_title(summary);
11123 if !what.is_empty() {
11124 body.push_str("## Summary\n\n");
11125 body.push_str(&what);
11126 body.push_str("\n\n");
11127 }
11128
11129 body.push_str(&crate::pr_outcome::review_rounds(state));
11131 body.push_str(&crate::pr_outcome::final_verification(state, head));
11132 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11133
11134 let open = state.open_findings();
11135 if !open.is_empty() {
11136 body.push_str("## Open review findings\n\n");
11137 for f in &open {
11138 body.push_str(&format!(
11139 "- `{}` [{:?}] {}\n",
11140 f.id, f.severity, "Context retained in local review records."
11141 ));
11142 }
11143 body.push('\n');
11144 }
11145
11146 if let Some(fix) = fix
11147 && !fix.rejected.is_empty()
11148 {
11149 body.push_str("## Declined by the fixer\n\n");
11150 for r in &fix.rejected {
11151 body.push_str(&format!(
11152 "- `{}`: {}\n",
11153 r.id, "Reason retained in local review records."
11154 ));
11155 }
11156 body.push('\n');
11157 }
11158
11159 if review {
11160 body.push_str("## Commits under review\n\n");
11162 if let Some(facts) = facts {
11163 let mut left = COMMITS_MAX_CHARS;
11164 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11165 let mut entry = format!("- {}\n", markdown_safe(subject));
11166 for l in markdown_safe(text).lines() {
11167 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11168 }
11169 if left == 0 {
11170 body.push_str(&format!(
11171 "- ... {} more commit(s)\n",
11172 facts.commits.len() - i
11173 ));
11174 break;
11175 }
11176 if entry.len() > left {
11177 let mut end = left;
11180 while !entry.is_char_boundary(end) {
11181 end -= 1;
11182 }
11183 entry.truncate(end);
11184 entry.push_str("\n ... (truncated)\n");
11185 left = 0;
11186 } else {
11187 left -= entry.len();
11188 }
11189 body.push_str(&entry);
11190 }
11191 if !facts.stat.trim().is_empty() {
11192 body.push_str(&format!(
11193 "\n## Diff stat\n\n```\n{}\n```\n",
11194 markdown_safe(facts.stat.trim())
11195 ));
11196 }
11197 } else {
11198 match &state.reviewed_commits {
11199 Some(subjects) => {
11200 for s in subjects {
11201 body.push_str(&format!("- {}\n", s.trim()));
11202 }
11203 }
11204 None => {
11205 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11208 body.push_str(rest.trim());
11209 body.push('\n');
11210 }
11211 }
11212 }
11213 } else {
11214 let task = state.instruction.trim();
11215 let task = if task.is_empty() {
11216 "(empty task)"
11217 } else {
11218 task
11219 };
11220 body.push_str(&format!(
11221 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11222 task.replace("</details>", "</details>")
11223 ));
11224 }
11225
11226 body.push_str(&format!(
11227 "\n---\nmagi:run/{} magi:candidate-{}\n",
11228 state.id,
11229 winner.to_ascii_lowercase()
11230 ));
11231
11232 PrMessage { title, body }
11233}
11234
11235fn seeded_instruction(state: &RunState) -> String {
11239 match refs::describe(&state.seeds) {
11240 Some(facts) => format!(
11241 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11242 Candidates start from the unmerged branch named above, when there \
11243 is one, and carry any unmerged commit named by sha as a \
11244 cherry-pick. Check that this is what the task meant before \
11245 building on it.",
11246 state.instruction
11247 ),
11248 None => state.instruction.clone(),
11249 }
11250}
11251
11252async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11256 let base = &state.base_branch;
11257 let mut against = base.clone();
11258 if mode == MergeMode::Pr {
11259 let remote = &state.config.merge.remote;
11263 let tracking = format!("{remote}/{base}");
11264 let fetched = git::fetch(repo, remote, base).await;
11265 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11266 against = tracking;
11267 } else {
11268 return false;
11269 }
11270 }
11271 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11272}
11273
11274fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11277 let mut detail = format!(
11278 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11279 nothing to open a pull request for"
11280 );
11281 match refs::describe(&state.seeds) {
11282 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11283 None => detail.push_str(
11284 "\nThe task names no existing branch or commit; if it means to land work \
11285 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11286 ),
11287 }
11288 detail
11289}
11290
11291#[derive(Debug, PartialEq, Eq)]
11294enum PrPlan {
11295 Create,
11296 Adopt { url: String, title: String },
11297 Stop(String),
11298}
11299
11300fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11303 match found {
11304 Ok(land::OpenPr::None) => PrPlan::Create,
11305 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11306 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11307 "several open pull requests exist for this branch, not picking one: {}",
11308 urls.join(" ")
11309 )),
11310 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11311 }
11312}
11313
11314pub(crate) async fn gh_pr_create(
11316 cwd: &Path,
11317 base: &str,
11318 head: &str,
11319 title: &str,
11320 body: &str,
11321) -> Result<String> {
11322 let out = tokio::process::Command::new("gh")
11323 .args([
11324 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11325 ])
11326 .current_dir(cwd)
11327 .quiet()
11328 .stdin(std::process::Stdio::null())
11329 .output()
11330 .await
11331 .context("spawn gh")?;
11332 if out.status.success() {
11333 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11334 } else {
11335 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11336 }
11337}
11338
11339async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11341 match std::fs::symlink_metadata(path) {
11342 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11343 git::worktree_remove_clean(repo, path).await?;
11346 return Ok(());
11347 }
11348 Err(e) => return Err(e.into()),
11349 Ok(_) => {}
11350 }
11351 if !git::is_clean(path).await? {
11352 bail!(
11353 "worktree has uncommitted work; refusing removal: {}",
11354 path.display()
11355 );
11356 }
11357 if !git::worktree_remove_clean(repo, path).await? {
11358 bail!("worktree removal refused: {}", path.display());
11359 }
11360 Ok(())
11361}
11362
11363pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11372 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11373 bail!("refusing fold: {reason}");
11374 }
11375 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11376 bail!("candidate preservation incomplete; refusing cleanup");
11377 }
11378 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11382 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11383 let _driver = crate::driver::maintenance(home, &state.id)?;
11384 let mut output_guards = Vec::new();
11385 for entry in std::fs::read_dir(state.worktree_root())
11386 .into_iter()
11387 .flatten()
11388 .flatten()
11389 {
11390 if entry.path().join(".git").is_file() {
11391 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11392 }
11393 }
11394 let repo = state.repo.clone();
11395 let wait = repository_wait(state, None);
11398 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11399 Ok(lease) => lease,
11400 Err(e) => {
11401 state.save_under(home)?;
11402 return Err(e);
11403 }
11404 };
11405 let _source_outputs = if state.status == RunStatus::Merged
11409 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11410 {
11411 verify_merged_fold_sources(state, drop_winner, home).await?
11412 } else {
11413 Vec::new()
11414 };
11415 let root = state.worktree_root();
11416 let winner = state.tally.as_ref().map(|t| t.winner);
11417 let mut removed = Vec::new();
11418
11419 for i in 0..state.candidates.len() {
11420 let c = state.candidates[i].clone();
11421 let is_winner = Some(c.label) == winner;
11422 if is_winner && !drop_winner {
11423 continue;
11424 }
11425 let existed = c.worktree.exists();
11426 remove_preserved_worktree(&repo, &c.worktree).await?;
11429 if existed {
11430 removed.push(c.worktree.to_string_lossy().into_owned());
11431 }
11432 let handed_over = state.released_branches.contains(&c.branch);
11435 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11436 if !git::branch_delete(&repo, &c.branch).await? {
11437 bail!("candidate branch removal refused");
11438 }
11439 removed.push(c.branch.clone());
11440 }
11441 state.candidates[i].folded = true;
11442 }
11443
11444 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11445 let path = name.path();
11446 let keep = !drop_winner
11447 && winner.is_some_and(|w| {
11448 path.file_name()
11449 .is_some_and(|n| n == format!("cand-{w}").as_str())
11450 });
11451 if keep {
11452 continue;
11453 }
11454 remove_preserved_worktree(&repo, &path).await?;
11455 removed.push(path.to_string_lossy().into_owned());
11456 }
11457
11458 remove_if_empty(&root);
11467
11468 if state.enabled_worktree_config && drop_winner {
11469 git::release_worktree_config(&repo).await.ok();
11473 state.enabled_worktree_config = false;
11474 }
11475 state.save_under(home)?;
11476 Ok(removed)
11477}
11478
11479async fn verify_merged_fold_sources(
11480 state: &RunState,
11481 drop_winner: bool,
11482 home: &Path,
11483) -> Result<Vec<std::fs::File>> {
11484 let winner = state.tally.as_ref().map(|t| t.winner);
11485 let mut heads = BTreeSet::new();
11486 for candidate in &state.candidates {
11487 if Some(candidate.label) == winner && !drop_winner {
11488 continue;
11489 }
11490 match std::fs::symlink_metadata(&candidate.worktree) {
11491 Ok(_) => anyhow::ensure!(
11492 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11493 "candidate recovery source is outside the inventoried run root: {}",
11494 candidate.worktree.display()
11495 ),
11496 Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11497 Err(error) => {
11498 return Err(error).context(format!(
11499 "cannot inspect candidate recovery source: {}",
11500 candidate.worktree.display()
11501 ));
11502 }
11503 }
11504 if git::branch_exists(&state.repo, &candidate.branch).await? {
11505 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11506 heads.extend(
11507 git::git(
11508 &state.repo,
11509 &["reflog", "show", "--format=%H", &candidate.branch],
11510 )
11511 .await?
11512 .lines()
11513 .map(str::to_owned),
11514 );
11515 }
11516 }
11517 let mut cargo_guards = Vec::new();
11518 let entries = match std::fs::read_dir(state.worktree_root()) {
11519 Ok(entries) => Some(entries),
11520 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11521 Err(error) => return Err(error.into()),
11522 };
11523 for entry in entries.into_iter().flatten() {
11524 let path = entry?.path();
11525 let metadata = std::fs::symlink_metadata(&path)?;
11526 anyhow::ensure!(
11527 metadata.is_dir() && !metadata.file_type().is_symlink(),
11528 "non-directory or symlink recovery source retained: {}",
11529 path.display()
11530 );
11531 if !drop_winner
11532 && winner.is_some_and(|w| {
11533 path.file_name()
11534 .is_some_and(|name| name == format!("cand-{w}").as_str())
11535 })
11536 {
11537 continue;
11538 }
11539 anyhow::ensure!(
11540 path.join(".git").is_file(),
11541 "unregistered recovery source retained: {}",
11542 path.display()
11543 );
11544 anyhow::ensure!(
11545 git::git(
11546 &path,
11547 &[
11548 "status",
11549 "--porcelain",
11550 "--untracked-files=all",
11551 "--ignored",
11552 "--",
11553 ".",
11554 ":(exclude)target"
11555 ]
11556 )
11557 .await?
11558 .is_empty(),
11559 "source changes or ignored recovery files retained: {}",
11560 path.display()
11561 );
11562 match std::fs::symlink_metadata(path.join("target")) {
11563 Ok(_) => {
11564 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11565 }
11566 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11567 Err(error) => return Err(error.into()),
11568 }
11569 for checkpoint in [
11570 "rebase-merge",
11571 "rebase-apply",
11572 "sequencer",
11573 "CHERRY_PICK_HEAD",
11574 "MERGE_HEAD",
11575 "REBASE_HEAD",
11576 "REVERT_HEAD",
11577 "BISECT_LOG",
11578 "AUTO_MERGE",
11579 "MERGE_MSG",
11580 ] {
11581 let location = git::git(
11582 &path,
11583 &[
11584 "rev-parse",
11585 "--path-format=absolute",
11586 "--git-path",
11587 checkpoint,
11588 ],
11589 )
11590 .await?;
11591 anyhow::ensure!(
11592 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11593 if e.kind() == std::io::ErrorKind::NotFound),
11594 "Git recovery checkpoint retained: {checkpoint}"
11595 );
11596 }
11597 heads.insert(git::rev_parse(&path, "HEAD").await?);
11598 heads.extend(
11599 git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11600 .await?
11601 .lines()
11602 .map(str::to_owned),
11603 );
11604 if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11605 heads.insert(original);
11606 }
11607 }
11608 if !heads.is_empty() {
11609 let reference = format!(
11613 "refs/remotes/{}/{}",
11614 state.config.merge.remote, state.base_branch
11615 );
11616 let base = git::rev_parse(&state.repo, &reference)
11617 .await
11618 .context("cannot prove merged sources against the fetched base")?;
11619 let mut squash_winner = None;
11620 let mut checked_squash = false;
11621 for head in heads {
11622 if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11623 .await?
11624 .ok()
11625 {
11626 continue;
11627 }
11628 if !checked_squash {
11629 squash_winner = proven_squash_winner(state, &base).await?;
11630 checked_squash = true;
11631 }
11632 let incorporated = if let Some(winner) = &squash_winner {
11633 git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11634 .await?
11635 .ok()
11636 } else {
11637 false
11638 };
11639 anyhow::ensure!(
11640 incorporated,
11641 "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11642 );
11643 }
11644 }
11645 Ok(cargo_guards)
11646}
11647
11648async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11654 let Some(winner) = state.winner() else {
11655 return Ok(None);
11656 };
11657 let head = git::rev_parse(&state.repo, &winner.branch).await?;
11658 if !git::git_raw(
11659 &state.repo,
11660 &["merge-base", "--is-ancestor", &state.base_commit, &head],
11661 )
11662 .await?
11663 .ok()
11664 {
11665 return Ok(None);
11666 }
11667 let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11668 if patch.is_empty() {
11669 return Ok(None);
11670 }
11671 let range = format!("{}..{base}", state.base_commit);
11672 let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11673 for commit in commits.lines() {
11674 let parent = format!("{commit}^");
11675 if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11676 return Ok(Some(head));
11677 }
11678 }
11679 Ok(None)
11680}
11681
11682async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11683 git::git(
11684 repo,
11685 &[
11686 "-c",
11687 "core.quotePath=true",
11688 "diff",
11689 "--binary",
11690 "--full-index",
11691 "--no-renames",
11692 "--no-ext-diff",
11693 "--no-textconv",
11694 "--no-color",
11695 "--ignore-submodules=none",
11696 "--submodule=short",
11697 from,
11698 to,
11699 "--",
11700 ],
11701 )
11702 .await
11703}
11704
11705fn remove_if_empty(dir: &Path) {
11716 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11717 std::fs::remove_dir(dir).ok();
11718 }
11719}
11720
11721pub fn worst_open(state: &RunState) -> Option<Severity> {
11723 state
11724 .reviews
11725 .last()?
11726 .reviews
11727 .iter()
11728 .flat_map(|r| r.findings.iter())
11729 .map(|f| f.severity)
11730 .max()
11731}
11732
11733async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11737 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11738 git::acquire_worktree_config(repo).await?;
11739 state.enabled_worktree_config = true;
11740 }
11741 Ok(())
11742}
11743
11744#[cfg(test)]
11745mod tests {
11746 #[tokio::test]
11747 async fn launcher_base_fetch_requires_remote_success() {
11748 let tmp = tempfile::tempdir().unwrap();
11749 let repo = tmp.path().join("repo");
11750 let remote = tmp.path().join("remote");
11751 std::fs::create_dir(&repo).unwrap();
11752 std::fs::create_dir(&remote).unwrap();
11753 git::git(&remote, &["init", "--bare", "-b", "main"])
11754 .await
11755 .unwrap();
11756 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11757 git::git(
11758 &repo,
11759 &[
11760 "-c",
11761 "user.name=Fixture",
11762 "-c",
11763 "user.email=fixture@example.invalid",
11764 "commit",
11765 "--allow-empty",
11766 "-m",
11767 "fixture",
11768 ],
11769 )
11770 .await
11771 .unwrap();
11772 git::git(
11773 &repo,
11774 &["remote", "add", "origin", remote.to_str().unwrap()],
11775 )
11776 .await
11777 .unwrap();
11778 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11779 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11780 assert_eq!(
11781 resolve_base(&repo, "main", "origin", None)
11782 .await
11783 .unwrap()
11784 .commit,
11785 head
11786 );
11787 #[cfg(unix)]
11788 {
11789 let policy: crate::config::Coordination = toml::from_str(
11790 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11791 ).unwrap();
11792 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11793 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11794 let held = crate::repo_lock::SeatLease::operation_with(
11795 &repo,
11796 Some(&policy),
11797 &crate::repo_lock::WaitPolicy::default(),
11798 )
11799 .await
11800 .unwrap();
11801 assert!(held.is_some());
11802 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11803 tokio::pin!(fetch);
11804 assert!(
11805 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11806 .await
11807 .is_err()
11808 );
11809 assert!(
11810 !repo.join(".git/FETCH_HEAD").exists(),
11811 "no fetch before admission"
11812 );
11813 drop(held);
11814 let fetched = fetch.await.unwrap();
11815 assert_eq!(fetched.commit, head);
11816 assert_eq!(fetched.remote, "origin");
11817 assert_eq!(fetched.branch, "main");
11818 }
11819 git::git(
11821 &repo,
11822 &[
11823 "remote",
11824 "set-url",
11825 "origin",
11826 tmp.path().join("missing").to_str().unwrap(),
11827 ],
11828 )
11829 .await
11830 .unwrap();
11831 let error = resolve_base(&repo, "main", "origin", None)
11832 .await
11833 .unwrap_err()
11834 .to_string();
11835 assert!(error.contains("cannot read origin/main"));
11836 assert!(error.contains("refusing to branch off"));
11837 }
11838
11839 #[test]
11840 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11841 let leaked = vec!["chore(deps): update a crate".to_owned()];
11842 let own = "fix(daemon): apply a chosen action";
11843 assert!(should_retitle("", own, &leaked));
11844 assert!(should_retitle(
11845 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11846 own,
11847 &leaked
11848 ));
11849 assert!(should_retitle(
11850 "chore: land candidate A of run 1",
11851 own,
11852 &leaked
11853 ));
11854 assert!(should_retitle(
11855 "magi: candidate A (uncommitted work)",
11856 own,
11857 &leaked
11858 ));
11859 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11860 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11861 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11862 }
11863
11864 #[test]
11865 fn pr_merge_plan_creates_adopts_or_stops() {
11866 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11867 assert_eq!(
11868 pr_merge_plan(Ok(land::OpenPr::One {
11869 url: "u".into(),
11870 title: "t".into()
11871 })),
11872 PrPlan::Adopt {
11873 url: "u".into(),
11874 title: "t".into()
11875 }
11876 );
11877 let PrPlan::Stop(many) =
11878 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11879 else {
11880 panic!("many must stop");
11881 };
11882 assert!(many.contains('a') && many.contains('b'));
11883 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11884 panic!("a failed lookup must stop");
11885 };
11886 assert!(err.contains("bad token"));
11887 }
11888
11889 use super::*;
11890 use crate::run::GateStatus;
11891 use std::collections::BTreeMap;
11892 use std::time::Duration;
11893
11894 fn conductor() -> AgentSpec {
11895 AgentSpec {
11896 id: "conductor".to_owned(),
11897 kind: crate::config::AgentKind::Command,
11898 model: None,
11899 command: vec!["true".to_owned()],
11900 extra_args: Vec::new(),
11901 env: BTreeMap::new(),
11902 prompt_delivery: None,
11903 }
11904 }
11905
11906 fn spec(id: &str) -> AgentSpec {
11907 AgentSpec {
11908 id: id.to_owned(),
11909 kind: crate::config::AgentKind::Command,
11910 model: None,
11911 command: vec!["true".to_owned()],
11912 extra_args: Vec::new(),
11913 env: BTreeMap::new(),
11914 prompt_delivery: None,
11915 }
11916 }
11917
11918 fn ids(xs: &[&str]) -> BTreeSet<String> {
11919 xs.iter().map(|s| (*s).to_owned()).collect()
11920 }
11921
11922 #[test]
11923 fn pick_successor_skips_an_agent_another_seat_holds() {
11924 let roster = [spec("a"), spec("b"), spec("c")];
11926 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11927 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11928 }
11929
11930 #[test]
11931 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11932 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11934 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11935 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11936 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11937 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11938 }
11939
11940 #[test]
11941 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11942 let roster = [spec("a"), spec("b")];
11943 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11944 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11945 let carried = ids(&["b"]);
11946 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11947 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11948 }
11949
11950 #[test]
11951 fn pick_successor_returns_none_without_any_untried_successor() {
11952 let roster = [spec("a"), spec("b")];
11953 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11954 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11955 let carried = ids(&["a"]);
11956 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11957 }
11958
11959 #[test]
11960 fn pick_successor_rescue_avoids_another_seats_occupant() {
11961 let roster = [spec("a"), spec("b"), spec("c")];
11963 let carried = ids(&["b", "c"]);
11964 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11965 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11966 }
11967
11968 #[test]
11969 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11970 let roster = [spec("a"), spec("b"), spec("c")];
11971 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11972 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11973 }
11974
11975 #[test]
11976 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11977 let roster = [spec("a"), spec("b"), spec("c")];
11978 let failed = ids(&["a", "b", "c"]);
11979 let mut tried = ids(&["b"]);
11981 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11982 assert_eq!(first.id, "a");
11983 tried.insert(first.id.clone());
11984 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11985 assert_eq!(second.id, "c");
11986 tried.insert(second.id.clone());
11987 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11988 }
11989
11990 #[test]
11991 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11992 let roster = [spec("a"), spec("b")];
11993 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11994 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11995 }
11996
11997 #[test]
11998 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11999 let roster = [spec("a"), spec("b"), spec("c")];
12000 let h = SeatHistory {
12001 failed: ids(&["a"]),
12002 last_ok: Some("b".to_owned()),
12003 last_fail: None,
12004 };
12005 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12006 let gone = SeatHistory {
12008 last_ok: Some("zzz".to_owned()),
12009 ..h.clone()
12010 };
12011 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12012 let failed = SeatHistory {
12013 failed: ids(&["a", "b"]),
12014 last_ok: Some("b".to_owned()),
12015 last_fail: None,
12016 };
12017 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12018 }
12019
12020 #[test]
12021 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12022 let first = SeatState::new("review-1", "alpha", 7);
12023 let next = handover_seat("review-1", "gamma", 7);
12024 assert_ne!(first.claude_session, next.claude_session);
12025 }
12026
12027 #[test]
12028 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12029 let mut state = state_with_summary("x", "y");
12030 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12031 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12032 assert_ne!(a.claude_session, b.claude_session);
12033 }
12034
12035 #[test]
12036 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12037 let roster = [spec("a"), spec("b")];
12038 let h = SeatHistory {
12039 failed: ids(&["a", "b"]),
12040 ..SeatHistory::default()
12041 };
12042 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12043 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12044 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12045 }
12046
12047 #[test]
12054 fn failure_signature_ignores_numbers_and_paths() {
12055 assert_eq!(
12056 failure_signature("exited with Some(2) and no usable output"),
12057 failure_signature("exited with Some(137) and no usable output")
12058 );
12059 assert_eq!(
12060 failure_signature("process exited with no usable output (exit code 2)"),
12061 failure_signature("process exited with no usable output (exit code 137)")
12062 );
12063 for (old, new) in [
12064 (
12065 "exited with Some(2) and no usable output",
12066 "process exited with no usable output (exit code 2)",
12067 ),
12068 (
12069 "exited with None and no usable output",
12070 "process exited with no usable output (exit code unavailable)",
12071 ),
12072 ] {
12073 let previous = FailClass::Other(failure_signature(old));
12074 let current = FailClass::Other(failure_signature(new));
12075 assert_eq!(previous, current, "resumption retains the failure class");
12076 assert!(!should_hand_over(Some(&previous), ¤t));
12077 }
12078 assert_eq!(
12079 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12080 failure_signature("cannot open /var/x.txt: denied")
12081 );
12082 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12083 }
12084
12085 #[test]
12086 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12087 let other = FailClass::Other("x".into());
12088 assert!(should_hand_over(None, &FailClass::Quota));
12089 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12090 assert!(should_hand_over(
12091 Some(&FailClass::Timeout),
12092 &FailClass::Timeout
12093 ));
12094 assert!(should_hand_over(None, &other));
12095 assert!(!should_hand_over(Some(&other), &other));
12096 assert!(should_hand_over(
12097 Some(&other),
12098 &FailClass::Other("y".into())
12099 ));
12100 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12102 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12103 }
12104
12105 #[test]
12106 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12107 let a = SeatState::new("judge-1", "alpha", 7);
12108 let b = handover_seat("judge-1", "beta", 7);
12109 assert_ne!(a.claude_session, b.claude_session);
12110 assert_eq!(b.turns, 0);
12111 }
12112
12113 #[test]
12114 fn a_timeout_is_classified_apart_from_other_failures() {
12115 assert_eq!(
12116 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12117 Some(FailClass::Timeout)
12118 );
12119 assert!(matches!(
12120 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12121 Some(FailClass::Other(_))
12122 ));
12123 }
12124
12125 #[test]
12126 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12127 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12128 let tried = BTreeSet::from(["beta".to_owned()]);
12129 let next = next_untried_in_roster(&roster, 1, &tried);
12132 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12133 }
12134
12135 #[test]
12136 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12137 let roster = vec![spec("alpha"), spec("beta")];
12138 let tried = BTreeSet::from(["beta".to_owned()]);
12139 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12143 }
12144
12145 #[test]
12146 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12147 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12148 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12149 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12153 }
12154
12155 #[test]
12156 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12157 let roster = vec![spec("a"), spec("a"), spec("b")];
12158 let tried = BTreeSet::from(["a".to_owned()]);
12159 let next = next_untried_in_roster(&roster, 0, &tried);
12160 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12161 }
12162
12163 #[test]
12164 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12165 let roster = vec![spec("a"), spec("b")];
12166 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12167 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12168 }
12169
12170 #[test]
12171 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12172 let dir = tempfile::tempdir().unwrap();
12173 let bay = dir.path().join("ffff");
12174
12175 remove_if_empty(&bay);
12177 assert!(!bay.exists());
12178
12179 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12182 remove_if_empty(&bay);
12183 assert!(bay.exists(), "non-empty directory must survive");
12184
12185 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12187 remove_if_empty(&bay);
12188 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12189 }
12190
12191 #[test]
12200 fn a_full_panel_that_found_nothing_is_clean() {
12201 assert!(round_is_clean(
12202 0,
12203 true,
12204 2,
12205 2,
12206 0,
12207 IncompleteReviewPolicy::Block
12208 ));
12209 }
12210
12211 #[test]
12212 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12213 assert!(!round_is_clean(
12214 0,
12215 true,
12216 1,
12217 2,
12218 0,
12219 IncompleteReviewPolicy::Block
12220 ));
12221 }
12222
12223 #[test]
12224 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12225 assert!(!round_is_clean(
12226 1,
12227 true,
12228 1,
12229 2,
12230 0,
12231 IncompleteReviewPolicy::Warn
12232 ));
12233 }
12234
12235 #[test]
12236 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12237 assert!(round_is_clean(
12238 0,
12239 true,
12240 1,
12241 2,
12242 0,
12243 IncompleteReviewPolicy::Warn
12244 ));
12245 }
12246
12247 #[test]
12248 fn a_full_panel_with_an_open_finding_is_not_clean() {
12249 assert!(!round_is_clean(
12250 1,
12251 true,
12252 2,
12253 2,
12254 0,
12255 IncompleteReviewPolicy::Block
12256 ));
12257 }
12258
12259 #[test]
12260 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12261 assert!(!round_is_clean(
12262 0,
12263 false,
12264 2,
12265 2,
12266 0,
12267 IncompleteReviewPolicy::Block
12268 ));
12269 }
12270
12271 #[test]
12278 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12279 assert!(round_is_clean(
12282 0,
12283 true,
12284 1,
12285 2,
12286 1,
12287 IncompleteReviewPolicy::Block
12288 ));
12289 }
12290
12291 #[test]
12292 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12293 assert!(!round_is_clean(
12296 0,
12297 true,
12298 1,
12299 2,
12300 0,
12301 IncompleteReviewPolicy::Block
12302 ));
12303 }
12304
12305 #[test]
12306 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12307 assert!(!round_is_clean(
12308 1,
12309 true,
12310 1,
12311 2,
12312 1,
12313 IncompleteReviewPolicy::Block
12314 ));
12315 assert!(!round_is_clean(
12316 0,
12317 false,
12318 1,
12319 2,
12320 1,
12321 IncompleteReviewPolicy::Block
12322 ));
12323 }
12324
12325 #[test]
12326 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12327 assert!(!round_is_clean(
12331 0,
12332 true,
12333 0,
12334 2,
12335 2,
12336 IncompleteReviewPolicy::Block
12337 ));
12338 }
12339
12340 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12341 CommandOutcome {
12342 command: "test".to_owned(),
12343 code,
12344 output_tail: String::new(),
12345 duration_ms: 0,
12346 resource_blocked,
12347 }
12348 }
12349
12350 #[test]
12351 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12352 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12353 assert!(
12354 !verify_inconclusive(&[outcome(Some(1), false)]),
12355 "an ordinary failure is still evidence about the patch"
12356 );
12357 assert!(verify_inconclusive(&[outcome(None, true)]));
12358 assert!(
12359 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12360 "one inconclusive outcome taints the whole batch"
12361 );
12362 assert!(!verify_inconclusive(&[]));
12363 }
12364
12365 #[tokio::test]
12366 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12367 let calls = std::sync::atomic::AtomicUsize::new(0);
12371 let started = Instant::now();
12372 wait_for_pids_with(
12373 &[123],
12374 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12375 Duration::from_millis(5),
12376 Duration::from_secs(5),
12377 )
12378 .await;
12379 assert!(
12380 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12381 "must keep checking rather than deciding on the first answer"
12382 );
12383 assert!(
12384 started.elapsed() < Duration::from_secs(1),
12385 "must return the moment it is confirmed dead, not wait out the ceiling"
12386 );
12387 }
12388
12389 #[tokio::test]
12390 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12391 let started = Instant::now();
12392 wait_for_pids_with(
12393 &[123],
12394 |_| true, Duration::from_millis(5),
12396 Duration::from_millis(30),
12397 )
12398 .await;
12399 let elapsed = started.elapsed();
12400 assert!(
12401 elapsed >= Duration::from_millis(30),
12402 "must not give up before its own ceiling: {elapsed:?}"
12403 );
12404 assert!(
12405 elapsed < Duration::from_secs(1),
12406 "must not wait past its own ceiling either: {elapsed:?}"
12407 );
12408 }
12409
12410 #[tokio::test]
12411 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12412 let started = Instant::now();
12413 wait_for_pids_with(
12414 &[],
12415 |_| true,
12416 Duration::from_secs(5),
12417 Duration::from_secs(5),
12418 )
12419 .await;
12420 assert!(
12421 started.elapsed() < Duration::from_millis(200),
12422 "an empty pid list has nothing to confirm"
12423 );
12424 }
12425
12426 fn review_round(
12432 clean: bool,
12433 blocking: usize,
12434 answered: usize,
12435 expected: usize,
12436 progressed: bool,
12437 e2e_ok: bool,
12438 ) -> ReviewRound {
12439 ReviewRound {
12440 round: 1,
12441 head: "h".to_owned(),
12442 verified_head: None,
12443 verified_at: None,
12444 reviews: Vec::new(),
12445 e2e: vec![CommandOutcome {
12446 command: "test".to_owned(),
12447 code: Some(if e2e_ok { 0 } else { 1 }),
12448 output_tail: String::new(),
12449 duration_ms: 0,
12450 resource_blocked: false,
12451 }],
12452 verify_retried: false,
12453 e2e_deferred: false,
12454 e2e_defer_reason: None,
12455 fix: None,
12456 blocking,
12457 answered,
12458 expected,
12459 clean,
12460 progressed,
12461 vote_split: false,
12462 reconsideration: Vec::new(),
12463 verdict: None,
12464 }
12465 }
12466
12467 #[test]
12468 fn review_conclusion_is_none_when_nothing_has_run() {
12469 assert_eq!(review_conclusion(&[], 3), None);
12470 }
12471
12472 #[test]
12473 fn review_conclusion_is_none_while_rounds_remain() {
12474 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12475 assert_eq!(review_conclusion(&rounds, 3), None);
12476 }
12477
12478 #[test]
12479 fn review_conclusion_is_gating_once_a_round_is_clean() {
12480 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12481 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12482 }
12483
12484 #[test]
12485 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12486 let rounds = vec![
12487 review_round(false, 1, 2, 2, true, true),
12488 review_round(false, 1, 2, 2, true, true),
12489 ];
12490 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12491 }
12492
12493 #[test]
12494 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12495 let rounds = vec![
12496 review_round(false, 1, 2, 2, true, true),
12497 review_round(false, 1, 2, 2, true, false),
12498 ];
12499 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12500 }
12501
12502 #[test]
12503 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12504 let mut blocked = review_round(false, 1, 2, 2, true, false);
12511 blocked.e2e[0].resource_blocked = true;
12512 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12513 assert_eq!(review_conclusion(&rounds, 2), None);
12514 }
12515
12516 #[test]
12517 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12518 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12520 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12521 }
12522
12523 #[test]
12524 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12525 let rounds = vec![
12526 review_round(false, 1, 2, 2, false, true),
12527 review_round(false, 1, 2, 2, false, true),
12528 ];
12529 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12530 }
12531
12532 fn secs(n: u64) -> Duration {
12533 Duration::from_secs(n)
12534 }
12535
12536 #[tokio::test]
12539 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12540 ask_test_home();
12541 let tmp = tempfile::tempdir().unwrap();
12542 let repo = tmp.path().join("repo");
12543 std::fs::create_dir_all(&repo).unwrap();
12544 init_repo(&repo);
12545 let g = |args: &[&str]| {
12546 let out = std::process::Command::new("git")
12547 .args(args)
12548 .current_dir(&repo)
12549 .quiet()
12550 .output()
12551 .unwrap();
12552 assert!(
12553 out.status.success(),
12554 "{}",
12555 String::from_utf8_lossy(&out.stderr)
12556 );
12557 };
12558 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12559 g(&["checkout", "-b", "historical"]);
12560 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12561 g(&["add", "."]);
12562 g(&["commit", "-m", "old alternative"]);
12563 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12564 g(&["checkout", "-b", "magi/task-selected", &base]);
12565 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12566 g(&["add", "."]);
12567 g(&["commit", "-m", "measurement infrastructure"]);
12568 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12569 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12570 g(&["add", "."]);
12571 g(&["commit", "-m", "dependent implementation"]);
12572 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12573 g(&["checkout", "main"]);
12574 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12575 g(&["add", "."]);
12576 g(&["commit", "-m", "unrelated base change"]);
12577 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12578 let origin = tmp.path().join("origin.git");
12579 g(&[
12580 "clone",
12581 "--bare",
12582 repo.to_str().unwrap(),
12583 origin.to_str().unwrap(),
12584 ]);
12585 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12586 g(&["fetch", "origin"]);
12587 let selection = crate::pr_recovery::Selection {
12588 url: "https://example.test/project/pull/1".into(),
12589 branch: "magi/task-selected".into(),
12590 head: head.clone(),
12591 source_head: None,
12592 source_run: None,
12593 source_proof: None,
12594 disposition: Some(crate::pr_recovery::Disposition::Review),
12595 instruction: "preserve measured behavior".into(),
12596 approved_at: jiff::Timestamp::now(),
12597 };
12598 let mut template = runner_at(RunStatus::Reviewing);
12599 template.state.repo = repo.clone();
12600 template.state.base_commit = current.clone();
12601 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12602 template.state.origin = Some(crate::run::Origin::queue("selected"));
12603 template.state.pr_head = Some(selection.branch.clone());
12604 template.state.pr_recovery = Some(selection.clone());
12605 let mut runner =
12606 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12607 .await
12608 .unwrap();
12609 let wt = runner.state.winner().unwrap().worktree.clone();
12610 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12611 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12612 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12613 assert!(runner.state.seeds.is_empty());
12614 assert_eq!(runner.state.instruction, "preserve measured behavior");
12615 runner.sync_to_base().await.unwrap();
12616 assert_eq!(
12617 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12618 "prerequisite"
12619 );
12620 assert_eq!(
12621 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12622 "measured implementation"
12623 );
12624 assert_eq!(
12625 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12626 "current main change"
12627 );
12628 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12629 assert_eq!(
12630 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12631 head
12632 );
12633
12634 let mut partial = runner_at(RunStatus::Reviewing);
12636 partial.state.repo = repo.clone();
12637 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12638 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12639 run: Some("exit 1".into()),
12640 ..Default::default()
12641 }];
12642 let selection = runner.state.pr_recovery.clone().unwrap();
12643 partial.state.pr_recovery = Some(selection.clone());
12644 partial.state.pr_head = Some(selection.branch.clone());
12645 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12646 let id = partial.state.id.clone();
12647 let error = match Runner::open_selected(
12648 &repo,
12649 partial.state,
12650 partial.roles,
12651 1,
12652 runner.state.base_commit.clone(),
12653 selection,
12654 )
12655 .await
12656 {
12657 Ok(_) => panic!("setup must fail"),
12658 Err(e) => e,
12659 };
12660 assert_eq!(
12661 error
12662 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12663 .unwrap()
12664 .run,
12665 id
12666 );
12667 let failed = RunState::load(&id).unwrap();
12668 assert_eq!(failed.status, RunStatus::Blocked);
12669 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12670 assert!(Runner::resume(&id).is_err());
12671 runner.state.status = RunStatus::Ready;
12672 runner.state.driver_exited = true;
12673 runner.state.save().unwrap();
12674 let mut retry = runner_at(RunStatus::Reviewing);
12675 retry.state.repo = repo.clone();
12676 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12677 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12678 reselected.approved_at = jiff::Timestamp::now();
12679 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12680 retry.state.pr_head = Some(reselected.branch.clone());
12681 retry.state.pr_recovery = Some(reselected.clone());
12682 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12683 .await
12684 .unwrap();
12685 let restarted = Runner::open_selected(
12686 &repo,
12687 retry.state,
12688 retry.roles,
12689 1,
12690 runner.state.base_commit.clone(),
12691 reselected,
12692 )
12693 .await
12694 .unwrap();
12695 assert_eq!(
12696 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12697 .await
12698 .unwrap(),
12699 head
12700 );
12701 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12702 assert_eq!(
12703 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12704 head
12705 );
12706 }
12707
12708 #[tokio::test]
12709 #[cfg(unix)]
12710 async fn fallback_and_interrupted_restoration_require_coordination() {
12711 use std::os::fd::AsRawFd as _;
12712 ask_test_home();
12713 let tmp = tempfile::tempdir().unwrap();
12714 let mut runner = registered_runner(tmp.path()).await;
12715 runner.state.config.graph.timeout_resource_wait = 1;
12716 let winner = runner.state.candidates[0].clone();
12717 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12718 let leftover = winner.worktree.join("leftover");
12719 std::fs::write(&leftover, "untrusted edits").unwrap();
12720 let next = tmp.path().join("next-fixer");
12721 let mut failed = spec("first");
12722 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12723 let mut successor = spec("second");
12724 successor.command = vec![
12725 "sh".into(),
12726 "-c".into(),
12727 format!("touch '{}'; echo done", next.display()),
12728 ];
12729 let attempts = vec![
12730 (failed.clone(), "first".into()),
12731 (successor.clone(), "second".into()),
12732 ];
12733 let lock_path = runner.state.repo.join(".git/updater.lock");
12734 let external = std::fs::File::create(&lock_path).unwrap();
12735 assert_eq!(
12736 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12737 0
12738 );
12739 let artifacts = tmp.path().join("artifacts");
12740 let build = |spec, seat| SeatJob {
12741 spec,
12742 seat,
12743 cwd: winner.worktree.clone(),
12744 prompt: "fixture".into(),
12745 timeout: Duration::from_secs(10),
12746 allow_write: true,
12747 sessions: false,
12748 artifacts: artifacts.clone(),
12749 stem: "fix".into(),
12750 handover: None,
12751 };
12752 let (_, _, out) = runner
12753 .ask_fixer_among(
12754 attempts.clone(),
12755 "graft",
12756 None,
12757 Some((&winner.worktree, &head)),
12758 build,
12759 )
12760 .await;
12761 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12762 assert!(leftover.exists(), "contention must prevent clean");
12763 assert!(!next.exists());
12764 assert!(runner.state.handovers.is_empty());
12765 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12767 from_run: "earlier".into(),
12768 generation: 1,
12769 mode: crate::recovery::Mode::VerifyRepair,
12770 reasons: Vec::new(),
12771 reviewed_head: None,
12772 rounds: Vec::new(),
12773 open_findings: Vec::new(),
12774 evidence: None,
12775 criteria_inherited: true,
12776 repair: crate::recovery::Repair::Started {
12777 before: head.clone(),
12778 },
12779 });
12780 assert!(
12781 runner
12782 .recovery_repair(&winner)
12783 .await
12784 .unwrap_err()
12785 .is::<crate::repo_lock::ResourceBlocked>()
12786 );
12787 assert!(matches!(
12788 runner.state.recovery.as_ref().unwrap().repair,
12789 crate::recovery::Repair::Started { .. }
12790 ));
12791 assert!(leftover.exists());
12792 drop(external);
12793 runner.recovery_repair(&winner).await.unwrap();
12794 assert!(!leftover.exists());
12795 assert!(matches!(
12796 runner.state.recovery.as_ref().unwrap().repair,
12797 crate::recovery::Repair::Done { .. }
12798 ));
12799 std::fs::remove_file(&lock_path).unwrap();
12801 std::fs::create_dir(&lock_path).unwrap();
12802 std::fs::write(&leftover, "untrusted edits").unwrap();
12803 let (_, _, out) = runner
12804 .ask_fixer_among(
12805 attempts,
12806 "graft",
12807 None,
12808 Some((&winner.worktree, &head)),
12809 build,
12810 )
12811 .await;
12812 assert!(matches!(out, AgentOutcome::Failed(_)));
12813 assert!(leftover.exists());
12814 assert!(!next.exists());
12815 assert_eq!(
12816 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12817 head
12818 );
12819 }
12820
12821 #[cfg(unix)]
12822 async fn registered_runner(root: &Path) -> Runner {
12823 ask_test_home();
12824 let repo = root.join("repo");
12825 std::fs::create_dir_all(&repo).unwrap();
12826 init_repo(&repo);
12827 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12828 let mut runner = runner_at(RunStatus::Prep);
12829 runner.state.repo = repo.clone();
12830 runner.state.config.coordination = crate::config::test_coordination();
12831 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12832 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12833 runner.state.config.graph.advise = false;
12834 runner.roles.implementers = vec![spec("implement")];
12835 runner.roles.judges = vec![spec("judge")];
12836 runner.prep().await.unwrap();
12837 runner
12838 }
12839
12840 #[tokio::test]
12841 #[cfg(unix)]
12842 async fn parked_implementer_persists_completed_work() {
12843 for registered in [false, true] {
12844 let tmp = tempfile::tempdir().unwrap();
12845 let mut runner = registered_runner(tmp.path()).await;
12846 if !registered {
12847 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12848 }
12849 let marker = tmp.path().join("started");
12850 let release = tmp.path().join("release");
12851 let mut implementer = spec("implement");
12852 implementer.command = vec![
12853 "sh".into(),
12854 "-c".into(),
12855 format!(
12856 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12857 marker.display(),
12858 release.display()
12859 ),
12860 ];
12861 runner.roles.implementers = vec![implementer.clone()];
12862 runner.state.config.agents = vec![implementer];
12863 let pause = runner.pause.clone();
12864 let (result, ()) = tokio::join!(runner.implement(), async {
12865 tokio::time::timeout(Duration::from_secs(10), async {
12866 while !marker.exists() {
12867 tokio::time::sleep(Duration::from_millis(10)).await;
12868 }
12869 })
12870 .await
12871 .unwrap();
12872 pause.park();
12873 std::fs::write(&release, "go").unwrap();
12874 });
12875 result.unwrap();
12876 let candidate = &runner.state.candidates[0];
12877 assert!(candidate.commits > 0, "{candidate:?}");
12878 assert!(candidate.summary.contains("saved-result"));
12879 assert_eq!(candidate.files, 1);
12880 let before = candidate.commits;
12881 runner.implement().await.unwrap();
12882 assert_eq!(runner.state.candidates[0].commits, before);
12883 }
12884 }
12885
12886 #[tokio::test]
12887 #[cfg(unix)]
12888 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12889 let tmp = tempfile::tempdir().unwrap();
12890 let mut runner = registered_runner(tmp.path()).await;
12891 let winner = runner.state.candidates[0].clone();
12892 let cache = tmp.path().join("cache");
12893 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12894 runner.state.config.graph.timeout_resource_wait = 0;
12895 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12896 let crate::cache::AcquireOutcome::Acquired(held) =
12897 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12898 else {
12899 panic!("cache must be free")
12900 };
12901 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12902 from_run: "earlier".into(),
12903 generation: 1,
12904 mode: crate::recovery::Mode::VerifyRepair,
12905 reasons: Vec::new(),
12906 reviewed_head: None,
12907 rounds: Vec::new(),
12908 open_findings: Vec::new(),
12909 evidence: None,
12910 criteria_inherited: true,
12911 repair: crate::recovery::Repair::NotStarted,
12912 });
12913 runner.recovery_repair(&winner).await.unwrap();
12914 assert!(matches!(
12915 runner.state.recovery.as_ref().unwrap().repair,
12916 crate::recovery::Repair::WaitingForRestore { .. }
12917 ));
12918 drop(held);
12919 runner.state.resource_block = None;
12920 runner.state.config.graph.timeout_resource_wait = 1;
12921 runner.recovery_repair(&winner).await.unwrap();
12922 assert!(matches!(
12923 runner.state.recovery.as_ref().unwrap().repair,
12924 crate::recovery::Repair::Done { .. }
12925 ));
12926 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12927 }
12928
12929 #[tokio::test]
12930 #[cfg(unix)]
12931 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12932 use std::os::fd::AsRawFd as _;
12933 for graft in [false, true] {
12934 let tmp = tempfile::tempdir().unwrap();
12935 let mut runner = registered_runner(tmp.path()).await;
12936 runner.state.config.graph.timeout_resource_wait = 1;
12937 let winner = runner.state.candidates[0].clone();
12938 let marker = tmp.path().join("fixer-started");
12939 let mut fixer = spec("fixer");
12940 fixer.command = vec![
12941 "sh".into(),
12942 "-c".into(),
12943 format!("echo started >> '{}'; echo done", marker.display()),
12944 ];
12945 runner.roles.fixer = Some(vec![fixer]);
12946 if graft {
12947 let mut loser = winner.clone();
12948 loser.label = 'B';
12949 loser.index = 1;
12950 runner.state.candidates.push(loser);
12951 runner.state.config.graph.graft_runner_up = true;
12952 runner.state.tally = Some(Tally {
12953 first_choice: BTreeMap::new(),
12954 borda: BTreeMap::new(),
12955 winner: winner.label,
12956 rankings: 1,
12957 unanimous_initial: true,
12958 deliberated: false,
12959 changed_votes: 0,
12960 unanimous_final: true,
12961 tie_break: None,
12962 judges: 1,
12963 present: 1,
12964 quorum: 1,
12965 met_quorum: true,
12966 uncontested: None,
12967 });
12968 runner.state.judgements.push(crate::run::Judgement {
12969 judge: 1,
12970 seat: "judge-1".into(),
12971 agent: "judge".into(),
12972 ranking: vec![winner.label, 'B'],
12973 reasons: BTreeMap::new(),
12974 confidence: None,
12975 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12976 order: vec![0, 1],
12977 failed: None,
12978 duration_ms: 0,
12979 });
12980 } else {
12981 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12982 from_run: "earlier".into(),
12983 generation: 1,
12984 mode: crate::recovery::Mode::VerifyRepair,
12985 reasons: Vec::new(),
12986 reviewed_head: None,
12987 rounds: Vec::new(),
12988 open_findings: Vec::new(),
12989 evidence: None,
12990 criteria_inherited: true,
12991 repair: crate::recovery::Repair::NotStarted,
12992 });
12993 }
12994 let reader = crate::repo_lock::SeatLease::for_checkout(
12995 &winner.worktree,
12996 false,
12997 crate::config::test_coordination().as_ref(),
12998 )
12999 .await
13000 .unwrap();
13001 let pause = runner.pause.clone();
13002 let (result, ()) = tokio::join!(
13003 async {
13004 if graft {
13005 runner.graft().await
13006 } else {
13007 runner.recovery_repair(&winner).await
13008 }
13009 },
13010 async {
13011 tokio::time::sleep(Duration::from_millis(100)).await;
13012 pause.park();
13013 }
13014 );
13015 result.unwrap();
13016 assert!(!marker.exists(), "a blocked seat must not spawn");
13017 runner.state = RunState::load(&runner.state.id).unwrap();
13019 if graft {
13020 assert_eq!(
13021 runner.state.graft.as_ref().unwrap().outcome,
13022 GraftOutcome::WaitingForRestore
13023 );
13024 } else {
13025 assert!(matches!(
13026 runner.state.recovery.as_ref().unwrap().repair,
13027 crate::recovery::Repair::WaitingForRestore { .. }
13028 ));
13029 }
13030 drop(reader);
13031 runner.pause = Pause::new();
13032 runner.state.resource_block = None;
13033 let external =
13035 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13036 assert_eq!(
13037 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13038 0
13039 );
13040 let leftover = winner.worktree.join("leftover");
13041 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13042 let result = if graft {
13043 runner.graft().await
13044 } else {
13045 runner.recovery_repair(&winner).await
13046 };
13047 assert!(
13048 result
13049 .unwrap_err()
13050 .is::<crate::repo_lock::ResourceBlocked>()
13051 );
13052 assert!(leftover.exists());
13053 assert!(!marker.exists());
13054 drop(external);
13055 runner.state.resource_block = None;
13056 let result = if graft {
13057 runner.graft().await
13058 } else {
13059 runner.recovery_repair(&winner).await
13060 };
13061 result.unwrap();
13062 assert!(!leftover.exists(), "restore must precede retry");
13063 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13064 if graft {
13065 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13066 } else {
13067 assert!(matches!(
13068 runner.state.recovery.as_ref().unwrap().repair,
13069 crate::recovery::Repair::Done { .. }
13070 ));
13071 }
13072 }
13073 }
13074
13075 #[tokio::test]
13076 #[cfg(unix)]
13077 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13078 use std::os::fd::AsRawFd as _;
13079 let tmp = tempfile::tempdir().unwrap();
13080 let mut runner = registered_runner(tmp.path()).await;
13081 runner.state.config.graph.timeout_resource_wait = 1;
13082 let winner = runner.state.candidates[0].clone();
13083 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13084 runner.state.tally = Some(Tally {
13085 first_choice: BTreeMap::new(),
13086 borda: BTreeMap::new(),
13087 winner: winner.label,
13088 rankings: 0,
13089 unanimous_initial: false,
13090 deliberated: false,
13091 changed_votes: 0,
13092 unanimous_final: false,
13093 tie_break: None,
13094 judges: 0,
13095 present: 0,
13096 quorum: 0,
13097 met_quorum: true,
13098 uncontested: Some("fixture".into()),
13099 });
13100 runner.state.graft = Some(GraftRecord {
13101 pre: pre.clone(),
13102 post: None,
13103 lines: None,
13104 limit: None,
13105 agent: None,
13106 outcome: GraftOutcome::Running,
13107 requested: Vec::new(),
13108 adoption: None,
13109 adoption_error: None,
13110 });
13111 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13112 git::commit_all(&winner.worktree, "unfinished graft")
13113 .await
13114 .unwrap();
13115 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13116 let leftover = winner.worktree.join("leftover");
13117 std::fs::write(&leftover, "untrusted edits").unwrap();
13118 let path = runner.state.repo.join(".git/updater.lock");
13119 let external = std::fs::File::create(&path).unwrap();
13120 assert_eq!(
13121 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13122 0
13123 );
13124 assert!(
13125 runner
13126 .graft()
13127 .await
13128 .unwrap_err()
13129 .is::<crate::repo_lock::ResourceBlocked>()
13130 );
13131 assert!(matches!(
13132 runner.state.graft.as_ref().unwrap().outcome,
13133 GraftOutcome::Running
13134 ));
13135 assert_eq!(
13136 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13137 unfinished
13138 );
13139 assert!(leftover.exists());
13140 drop(external);
13141 std::fs::remove_file(&path).unwrap();
13142 std::fs::create_dir(&path).unwrap();
13143 assert!(runner.graft().await.is_err());
13144 assert!(matches!(
13145 runner.state.graft.as_ref().unwrap().outcome,
13146 GraftOutcome::Running
13147 ));
13148 assert_eq!(
13149 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13150 unfinished
13151 );
13152 assert!(leftover.exists());
13153 std::fs::remove_dir(&path).unwrap();
13154 runner.graft().await.unwrap();
13155 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13156 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13157 assert!(!leftover.exists());
13158 }
13159
13160 #[tokio::test]
13161 #[cfg(unix)]
13162 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13163 use std::os::fd::AsRawFd as _;
13164 let tmp = tempfile::tempdir().unwrap();
13165 let mut runner = registered_runner(tmp.path()).await;
13166 runner.state.prep_complete = Some(false);
13167 runner.state.config.graph.timeout_resource_wait = 1;
13168 runner.state.save().unwrap();
13169 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13170 assert_eq!(
13171 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13172 0
13173 );
13174 runner.execute().await.unwrap();
13175 let disk = RunState::load(&runner.state.id).unwrap();
13176 assert!(disk.driver_exited);
13177 assert_eq!(disk.driver_pid, None);
13178 assert_eq!(runner.state.status, RunStatus::Blocked);
13179 assert_eq!(
13180 runner.state.resource_block.as_ref().unwrap().kind,
13181 "repository coordination"
13182 );
13183 assert!(runner.state.prep_failure.is_none());
13184 assert!(runner.state.seats.is_empty());
13185 runner.pause.park();
13186 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13189 .await
13190 .unwrap()
13191 .unwrap();
13192 assert!(runner.state.parked);
13193 assert!(runner.state.prep_failure.is_none());
13194 let disk = RunState::load(&runner.state.id).unwrap();
13195 assert!(disk.driver_exited);
13196 assert_eq!(disk.driver_pid, None);
13197 }
13198
13199 #[tokio::test]
13200 #[cfg(unix)]
13201 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13202 use std::os::fd::AsRawFd as _;
13203 let tmp = tempfile::tempdir().unwrap();
13204 let mut runner = registered_runner(tmp.path()).await;
13205 runner.state.prep_complete = Some(false);
13206 runner.state.config.graph.timeout_resource_wait = 30;
13207 runner.state.save().unwrap();
13208 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13209 assert_eq!(
13210 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13211 0
13212 );
13213 assert!(
13214 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13215 .await
13216 .is_err()
13217 );
13218 let disk = RunState::load(&runner.state.id).unwrap();
13219 assert!(disk.driver_exited);
13220 assert_eq!(disk.driver_pid, None);
13221 assert!(disk.status.resumable());
13222 assert!(runner.state.driver_exited);
13223 assert_eq!(runner.state.driver_pid, None);
13224 runner.state.save().unwrap();
13225 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13226 }
13227
13228 #[tokio::test]
13229 #[cfg(unix)]
13230 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13231 let tmp = tempfile::tempdir().unwrap();
13232 let mut runner = registered_runner(tmp.path()).await;
13233 let winner = runner.state.candidates[0].clone();
13234 let cache = tmp.path().join("cache");
13235 let started = tmp.path().join("pre-gate-started");
13236 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13237 runner.state.config.verify.gate =
13238 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13239 let home = crate::run::home();
13240 let owner =
13241 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13242 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13243 crate::cache::AcquireOutcome::Acquired(g) => g,
13244 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13245 };
13246 let mut gate = Box::pin(runner.run_gate(&winner));
13247 assert!(
13248 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13249 .await
13250 .is_err()
13251 );
13252 assert!(!started.exists(), "pre-gate must wait for the cache too");
13253 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13257 let editor = crate::repo_lock::SeatLease::checkout_with(
13258 &winner.worktree,
13259 true,
13260 crate::config::test_coordination().as_ref(),
13261 &wait,
13262 )
13263 .await
13264 .unwrap();
13265 drop(editor);
13266 guard.release();
13267 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13268 .await
13269 .unwrap()
13270 .unwrap();
13271 drop(gate);
13272 assert!(outcomes.iter().all(CommandOutcome::ok));
13273 assert!(started.exists());
13274 }
13275
13276 #[tokio::test]
13277 #[cfg(unix)]
13278 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13279 use std::os::fd::AsRawFd as _;
13280
13281 ask_test_home();
13282 let tmp = tempfile::tempdir().unwrap();
13283 let repo = tmp.path().join("repo");
13284 std::fs::create_dir_all(&repo).unwrap();
13285 init_repo(&repo);
13286 let common = repo.join(".git");
13287 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13288 let mut runner = runner_at(RunStatus::Prep);
13289 runner.state.repo = repo.clone();
13290 runner.state.config.coordination = crate::config::test_coordination();
13291 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13292 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13293 runner.state.config.graph.advise = false;
13294 runner.roles.implementers = vec![spec("implement")];
13295 runner.roles.judges = vec![spec("judge")];
13296 runner.prep().await.unwrap();
13297 let winner = runner.state.candidates[0].clone();
13298 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13301 let completed = tmp.path().join("seat-completed");
13302 runner.roles.implementers[0].command = vec![
13303 "sh".into(),
13304 "-c".into(),
13305 format!("touch '{}'; echo completed", completed.display()),
13306 ];
13307 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13308 assert_eq!(
13309 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13310 0
13311 );
13312 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13313 let mut implement = Box::pin(runner.implement());
13314 tokio::select! {
13315 result = &mut implement => panic!("rescue must wait: {result:?}"),
13316 result = tokio::time::timeout(Duration::from_secs(10), async {
13317 while !completed.exists() {
13318 tokio::time::sleep(Duration::from_millis(20)).await;
13319 }
13320 }) => result.unwrap(),
13321 }
13322 assert!(
13323 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13324 .await
13325 .is_err()
13326 );
13327 assert_eq!(
13328 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13329 before
13330 );
13331 assert_eq!(
13332 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13333 0
13334 );
13335 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13336 .await
13337 .unwrap()
13338 .unwrap();
13339 drop(implement);
13340 assert_ne!(
13341 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13342 before
13343 );
13344 assert!(git::is_clean(&winner.worktree).await.unwrap());
13345
13346 let started = tmp.path().join("pre-gate-started");
13347 let proceed = tmp.path().join("pre-gate-proceed");
13348 let checked = tmp.path().join("gate-ran");
13349 runner.state.config.verify.pre_gate = vec![format!(
13350 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13351 started.display(),
13352 proceed.display()
13353 )];
13354 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13355 assert_eq!(
13356 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13357 0
13358 );
13359 let mut gate = Box::pin(runner.run_gate(&winner));
13360 assert!(
13361 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13362 .await
13363 .is_err()
13364 );
13365 assert!(!started.exists());
13366 assert!(!checked.exists());
13367 assert!(git::is_clean(&winner.worktree).await.unwrap());
13368 assert_eq!(
13369 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13370 0
13371 );
13372 tokio::select! {
13373 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13374 result = tokio::time::timeout(Duration::from_secs(10), async {
13375 while !started.exists() {
13376 tokio::time::sleep(Duration::from_millis(20)).await;
13377 }
13378 }) => result.unwrap(),
13379 }
13380 assert_eq!(
13381 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13382 -1,
13383 "mechanical edits must hold the real lock"
13384 );
13385 std::fs::write(proceed, "ready").unwrap();
13386 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13387 .await
13388 .unwrap()
13389 .unwrap();
13390 drop(gate);
13391 assert!(outcomes.iter().all(CommandOutcome::ok));
13392 assert!(checked.exists());
13393 assert!(runner.state.pre_gate_commit.is_some());
13394 assert!(git::is_clean(&winner.worktree).await.unwrap());
13395 assert_eq!(
13396 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13397 0,
13398 "gate completion must release coordination"
13399 );
13400 drop(outsider);
13401 std::fs::remove_file(common.join("updater.lock")).unwrap();
13402 std::fs::create_dir(common.join("updater.lock")).unwrap();
13403 std::fs::remove_file(&started).unwrap();
13404 std::fs::remove_file(&checked).unwrap();
13405 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13406 let error = runner.run_gate(&winner).await.unwrap_err();
13407 assert!(
13408 error
13409 .to_string()
13410 .contains("acquire mandatory repository lock")
13411 );
13412 assert!(
13413 !started.exists(),
13414 "failed acquisition must prevent pre-gate"
13415 );
13416 assert!(!checked.exists(), "failed acquisition must prevent gate");
13417 assert!(git::is_clean(&winner.worktree).await.unwrap());
13418 assert_eq!(
13419 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13420 head
13421 );
13422 }
13423
13424 #[tokio::test]
13425 #[cfg(unix)]
13426 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13427 use std::os::fd::AsRawFd as _;
13428 use std::time::Duration;
13429
13430 ask_test_home();
13431 let tmp = tempfile::tempdir().unwrap();
13432 let repo = tmp.path().join("repo");
13433 std::fs::create_dir_all(&repo).unwrap();
13434 init_repo(&repo);
13435 let common = repo.join(".git");
13436 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13437 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13438 assert_eq!(
13439 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13440 0
13441 );
13442 let mut runner = runner_at(RunStatus::Prep);
13443 runner.state.repo = repo.clone();
13444 runner.state.config.coordination = crate::config::test_coordination();
13445 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13446 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13447 runner.state.config.graph.advise = true;
13448 runner.state.config.graph.advisors = 1;
13449 runner.roles.implementers = vec![spec("implement")];
13450 runner.roles.judges = vec![spec("judge")];
13451 let root = runner.state.worktree_root();
13452 let started = tmp.path().join("setup-started");
13453 let proceed = tmp.path().join("setup-proceed");
13454 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13455 run: Some(format!(
13456 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13457 started.display(),
13458 proceed.display()
13459 )),
13460 ..Default::default()
13461 }];
13462 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13463 let config = git::git(&repo, &["config", "--local", "--list"])
13464 .await
13465 .unwrap();
13466 let mut prep = Box::pin(runner.prep());
13467 assert!(
13468 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13469 .await
13470 .is_err()
13471 );
13472 assert!(
13473 !root.exists(),
13474 "contention must prevent even checkout creation"
13475 );
13476 assert!(!started.exists(), "contention must prevent setup");
13477 assert_eq!(
13478 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13479 branches
13480 );
13481 assert_eq!(
13482 git::git(&repo, &["config", "--local", "--list"])
13483 .await
13484 .unwrap(),
13485 config
13486 );
13487 drop(lock);
13488 tokio::select! {
13489 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13490 result = tokio::time::timeout(Duration::from_secs(10), async {
13491 while !started.exists() {
13492 tokio::time::sleep(Duration::from_millis(20)).await;
13493 }
13494 }) => result.unwrap(),
13495 }
13496 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13497 assert_eq!(
13498 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13499 -1
13500 );
13501 assert!(
13502 tokio::time::timeout(
13503 Duration::from_millis(100),
13504 crate::repo_lock::SeatLease::for_checkout(
13505 &repo,
13506 true,
13507 crate::config::test_coordination().as_ref()
13508 )
13509 )
13510 .await
13511 .is_err(),
13512 "preparation must also exclude local editors"
13513 );
13514 std::fs::write(proceed, "ready").unwrap();
13515 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13516 .await
13517 .unwrap()
13518 .unwrap();
13519 drop(prep);
13520 assert_eq!(runner.state.prep_complete, Some(true));
13521 assert!(root.join("judge-1").exists());
13522 assert!(root.join("advisor-1").exists());
13523 assert_eq!(
13524 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13525 0
13526 );
13527 }
13528
13529 #[tokio::test]
13530 #[cfg(unix)]
13531 async fn preparation_lock_failure_prevents_repository_changes() {
13532 ask_test_home();
13533 let tmp = tempfile::tempdir().unwrap();
13534 let repo = tmp.path().join("repo");
13535 std::fs::create_dir_all(&repo).unwrap();
13536 init_repo(&repo);
13537 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13538 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13541 let mut runner = runner_at(RunStatus::Prep);
13542 runner.state.repo = repo.clone();
13543 runner.state.config.coordination = crate::config::test_coordination();
13544 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13545 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13546 runner.roles.implementers = vec![spec("implement")];
13547 let error = runner.prep().await.unwrap_err();
13548 assert!(
13549 error
13550 .to_string()
13551 .contains("acquire mandatory repository lock")
13552 );
13553 assert!(!runner.state.worktree_root().exists());
13554 assert!(runner.state.candidates.is_empty());
13555 assert!(!repo.join(".git/config.worktree").exists());
13556 assert_eq!(
13557 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13558 "* main"
13559 );
13560 }
13561
13562 #[tokio::test]
13563 async fn preparation_records_creation_intent_before_git_can_fail() {
13564 ask_test_home();
13565 let tmp = tempfile::tempdir().unwrap();
13566 let repo = tmp.path().join("repo");
13567 std::fs::create_dir_all(&repo).unwrap();
13568 init_repo(&repo);
13569 let mut runner = runner_at(RunStatus::Prep);
13570 runner.state.repo = repo.clone();
13571 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13572 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13573 runner.state.config.blind.commit_msg_hook = false;
13574 runner.roles.implementers = vec![spec("one")];
13575 let label = blind::assign_labels(1, runner.state.seed)[0];
13576 let branch = runner.state.branch_for(label);
13577 git::git(&repo, &["branch", &branch]).await.unwrap();
13580 assert!(runner.prep().await.is_err());
13581 let saved = RunState::load(&runner.state.id).unwrap();
13582 assert!(saved.candidates.is_empty());
13583 let event = saved
13584 .events
13585 .iter()
13586 .find(|e| e.node == "prep-intent")
13587 .unwrap();
13588 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13589 assert_eq!(intent["branch"], branch);
13590 assert_eq!(intent["start"], runner.state.base_commit);
13591 assert_eq!(
13592 intent["worktree"],
13593 runner
13594 .state
13595 .worktree_root()
13596 .join(format!("cand-{label}"))
13597 .to_string_lossy()
13598 .as_ref()
13599 );
13600 assert_eq!(intent["picks"], serde_json::json!([]));
13601 }
13602
13603 #[tokio::test]
13604 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13605 ask_test_home();
13606 let tmp = tempfile::tempdir().unwrap();
13607 let repo = tmp.path().join("repo");
13608 std::fs::create_dir_all(&repo).unwrap();
13609 init_repo(&repo);
13610 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13611 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13612 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13613 git::git(&repo, &["add", "."]).await.unwrap();
13614 git::git(&repo, &["commit", "-m", "required seed"])
13615 .await
13616 .unwrap();
13617 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13618 git::git(&repo, &["checkout", "main"]).await.unwrap();
13619 let mut runner = runner_at(RunStatus::Prep);
13620 runner.state.repo = repo.clone();
13621 runner.state.base_commit = base.clone();
13622 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13623 runner.state.config.graph.advise = false;
13624 runner.state.config.blind.commit_msg_hook = false;
13625 runner.state.seeds = vec![refs::Seed {
13626 token: seed.clone(),
13627 sha: seed.clone(),
13628 kind: refs::SeedKind::Unmerged,
13629 branch: false,
13630 detail: String::new(),
13631 }];
13632 runner.roles.implementers = vec![spec("one"), spec("two")];
13633 runner.roles.judges = vec![spec("judge")];
13634 let label = blind::assign_labels(2, runner.state.seed)[0];
13635 let branch = runner.state.branch_for(label);
13636 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13637 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13638 .await
13639 .unwrap();
13640 runner.state.candidates.push(Candidate {
13641 index: 0,
13642 label,
13643 agent: "one".into(),
13644 branch,
13645 worktree,
13646 summary: String::new(),
13647 stat: String::new(),
13648 files: 0,
13649 commits: 0,
13650 empty: false,
13651 failed: None,
13652 verified_noop: None,
13653 duration_ms: 0,
13654 folded: false,
13655 });
13656 runner.state.save().unwrap();
13657 runner.state = RunState::load(&runner.state.id).unwrap();
13659 assert_eq!(runner.state.prep_complete, Some(false));
13660 runner.prep().await.unwrap();
13661 assert_eq!(runner.state.prep_complete, Some(true));
13662 assert_eq!(runner.state.candidates.len(), 2);
13663 for candidate in &runner.state.candidates {
13664 assert_eq!(
13665 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13666 git::tree_of(&repo, &seed).await.unwrap()
13667 );
13668 assert_eq!(
13669 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13670 "required commit"
13671 );
13672 }
13673 assert!(runner.state.worktree_root().join("judge-1").exists());
13674 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13675 runner.prep().await.unwrap();
13676 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13677
13678 runner.state.prep_complete = Some(false);
13679 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13680 run: Some(format!(
13681 "test -f '{}'",
13682 tmp.path().join("dependency-ready").display()
13683 )),
13684 ..Default::default()
13685 }];
13686 assert!(runner.prep().await.is_err());
13687 assert!(runner.state.prep_failure.is_some());
13688 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13689 runner.state = RunState::load(&runner.state.id).unwrap();
13690 assert!(
13691 runner.prep().await.is_err(),
13692 "correcting the cause alone must not auto-retry"
13693 );
13694 let failed_at = runner.state.events.last().unwrap().at;
13695 assert!(
13696 runner.retry_preparation(failed_at).is_err(),
13697 "old approvals cannot unlock the hold"
13698 );
13699 let mut task = crate::queue::Task::new(
13700 "retry".into(),
13701 "retry setup".into(),
13702 repo.clone(),
13703 crate::queue::Source::Human,
13704 );
13705 task.start(runner.state.id.clone());
13706 task.hold_machine(Some("setup failed".into()));
13707 task.release_operator("cli", None);
13708 runner
13709 .retry_preparation(task.resumption.as_ref().unwrap().at)
13710 .unwrap();
13711 runner.prep().await.unwrap();
13712 assert!(runner.state.prep_failure.is_none());
13713 assert_eq!(runner.state.prep_complete, Some(true));
13714 assert_eq!(runner.state.candidates.len(), 2);
13715 assert!(runner.state.worktree_root().join("judge-1").exists());
13716 for candidate in &runner.state.candidates {
13717 assert_eq!(
13718 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13719 git::tree_of(&repo, &seed).await.unwrap()
13720 );
13721 }
13722 }
13723
13724 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13725 let mut heads = Vec::new();
13726 for candidate in candidates {
13727 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13728 }
13729 heads
13730 }
13731
13732 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13733 ask_test_home();
13734 let tmp = tempfile::tempdir().unwrap();
13735 let repo = tmp.path().join("repo");
13736 std::fs::create_dir(&repo).unwrap();
13737 init_repo(&repo);
13738 let mut runner = runner_at(RunStatus::Reviewing);
13739 runner.state.repo = repo.clone();
13740 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13741 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13742 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13743 let branch = format!("candidate-{label}");
13744 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13745 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13746 .await
13747 .unwrap();
13748 if label == 'B' {
13749 std::fs::write(worktree.join("change"), "winner").unwrap();
13750 git::rescue_commit(&worktree, "candidate change")
13751 .await
13752 .unwrap();
13753 }
13754 runner.state.candidates.push(Candidate {
13755 index,
13756 label,
13757 agent: "alpha".into(),
13758 branch,
13759 worktree,
13760 summary: "completed reply".into(),
13761 stat: String::new(),
13762 files: usize::from(label == 'B'),
13763 commits: usize::from(label == 'B'),
13764 empty: label == 'A',
13765 failed: None,
13766 verified_noop: None,
13767 duration_ms: 1,
13768 folded: false,
13769 });
13770 }
13771 runner.state.judge_skipped = true;
13772 runner.state.tally = Some(Tally {
13773 first_choice: BTreeMap::new(),
13774 borda: BTreeMap::new(),
13775 winner: 'B',
13776 rankings: 0,
13777 unanimous_initial: false,
13778 deliberated: false,
13779 changed_votes: 0,
13780 unanimous_final: false,
13781 tie_break: None,
13782 judges: 0,
13783 present: 0,
13784 quorum: 0,
13785 met_quorum: true,
13786 uncontested: Some("only B produced a change".into()),
13787 });
13788 runner.state.save().unwrap();
13789 (tmp, runner)
13790 }
13791
13792 #[tokio::test]
13793 #[cfg(unix)]
13794 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13795 for (solo, failed_checkpoint) in
13796 [(true, false), (false, false), (true, true), (false, true)]
13797 {
13798 let (_tmp, mut runner) = preservation_resume_fixture().await;
13799 if solo {
13800 runner.state.candidates.retain(|c| c.label == 'A');
13801 }
13802 runner.state.candidates[0].empty = false;
13803 runner.state.candidates[0].summary.clear();
13804 runner.state.tally = None;
13805 runner.state.judge_skipped = false;
13806 runner.state.status = RunStatus::Implementing;
13807 runner.state.config.disk.min_free_bytes = u64::MAX;
13808 let mut implementer = spec("alpha");
13809 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13810 runner.roles.implementers = vec![implementer];
13811 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13812 if failed_checkpoint {
13813 std::fs::create_dir_all(&patch).unwrap();
13814 }
13815 runner.state.save().unwrap();
13816 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13817 runner.state = RunState::load(&runner.state.id).unwrap();
13818 assert_eq!(runner.state.status, RunStatus::Blocked);
13819 assert!(runner.state.interrupted_implementations.contains(&'A'));
13820 assert_eq!(runner.state.candidates[0].commits, 0);
13821 assert!(!runner.state.candidates[0].empty);
13822 assert!(runner.state.candidates[0].failed.is_none());
13823 if failed_checkpoint {
13824 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13825 std::fs::remove_dir(patch).unwrap();
13826 }
13827 runner.state.resource_block = None;
13828 runner.state.config.disk.min_free_bytes = 0;
13829 runner.roles.implementers[0].command = vec![
13830 "sh".into(),
13831 "-c".into(),
13832 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13833 ];
13834 runner.implement().await.unwrap();
13835 assert!(runner.state.interrupted_implementations.is_empty());
13836 assert!(runner.state.candidate_preservation.is_empty());
13837 assert_eq!(runner.state.candidates[0].commits, 1);
13838 assert!(!runner.state.candidates[0].empty);
13839 assert!(
13840 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13841 .unwrap()
13842 .contains("retried empty interrupted seat")
13843 );
13844 if !solo {
13845 assert_eq!(runner.state.candidates[1].commits, 1);
13846 }
13847 }
13848 }
13849
13850 #[tokio::test]
13851 #[cfg(unix)]
13852 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13853 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13854 let (_tmp, mut runner) = preservation_resume_fixture().await;
13855 runner.state.candidates.retain(|c| c.label == 'A');
13856 runner.state.candidates[0].empty = false;
13857 runner.state.candidates[0].summary.clear();
13858 runner.state.tally = None;
13859 runner.state.judge_skipped = false;
13860 runner.state.status = RunStatus::Implementing;
13861 runner.state.config.disk.min_free_bytes = u64::MAX;
13864 let wt = runner.state.candidates[0].worktree.clone();
13865 std::fs::write(
13866 wt.join("interrupted-source"),
13867 "retain implementation work\n",
13868 )
13869 .unwrap();
13870 if committed {
13871 git::rescue_commit(&wt, "implementation checkpoint")
13872 .await
13873 .unwrap();
13874 }
13875 let lock = git::git(
13876 &wt,
13877 &[
13878 "rev-parse",
13879 "--path-format=absolute",
13880 "--git-path",
13881 "index.lock",
13882 ],
13883 )
13884 .await
13885 .unwrap();
13886 if block_rescue {
13887 std::fs::write(&lock, "fixture-owned lock").unwrap();
13888 }
13889 let mut implementer = spec("alpha");
13890 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13891 runner.roles.implementers = vec![implementer];
13892 runner.state.save().unwrap();
13893 let result = runner.implement().await;
13894 assert_eq!(result.is_err(), block_rescue);
13895 assert_eq!(
13896 runner.state.resource_block.as_ref().unwrap().kind,
13897 "disk pressure"
13898 );
13899 assert!(
13900 runner
13901 .state
13902 .events
13903 .iter()
13904 .any(|e| e.message.contains("agent execution interrupted"))
13905 );
13906 assert!(
13907 runner
13908 .state
13909 .dir()
13910 .join("artifacts/candidate-A-recovery/staged.patch")
13911 .exists()
13912 );
13913 if block_rescue {
13914 let saved = RunState::load(&runner.state.id).unwrap();
13915 assert!(saved.candidate_preservation.contains_key(&'A'));
13916 assert!(!saved.candidates[0].empty);
13917 assert_eq!(
13918 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13919 "retain implementation work\n"
13920 );
13921 std::fs::remove_file(lock).unwrap();
13922 runner.state = saved;
13923 } else {
13924 assert_eq!(runner.state.status, RunStatus::Blocked);
13925 assert_eq!(runner.state.candidates[0].commits, 1);
13926 }
13927 runner.roles.implementers.clear();
13930 runner.state.config.disk.min_free_bytes = 0;
13931 runner.state.resource_block = None;
13932 runner.implement().await.unwrap();
13933 assert!(runner.state.candidate_preservation.is_empty());
13934 assert_eq!(runner.state.candidates[0].commits, 1);
13935 assert!(!runner.state.candidates[0].empty);
13936 assert!(git::is_clean(&wt).await.unwrap());
13937 assert!(
13938 crate::run::read_artifact(&runner.state, "cand-A.patch")
13939 .unwrap()
13940 .contains("retain implementation work")
13941 );
13942 }
13943 }
13944
13945 #[tokio::test]
13946 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13947 let (_tmp, mut runner) = preservation_resume_fixture().await;
13948 runner.implement().await.unwrap();
13950 assert!(runner.state.candidate_preservation.is_empty());
13951 assert!(
13952 !runner
13953 .state
13954 .dir()
13955 .join("artifacts/candidate-A-recovery")
13956 .exists()
13957 );
13958 runner.fold_losers().await.unwrap();
13959 assert!(!runner.state.candidates[0].worktree.exists());
13960 for schema in [14, crate::run::SCHEMA] {
13961 runner.state.schema = schema;
13962 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13963 runner.state.save().unwrap();
13964 runner.state = RunState::load(&runner.state.id).unwrap();
13965 runner.implement().await.unwrap();
13966 assert!(runner.state.candidate_preservation.is_empty());
13967 assert!(runner.state.candidates[0].folded);
13968 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13969 }
13970 }
13971
13972 #[tokio::test]
13973 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13974 let (_tmp, mut runner) = preservation_resume_fixture().await;
13975 let wt = runner.state.candidates[0].worktree.clone();
13976 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13977 runner.state.schema = 14;
13978 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13979 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13980 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13981 for _ in 0..2 {
13982 assert!(runner.implement().await.is_err());
13983 assert_eq!(runner.state.status, RunStatus::Blocked);
13984 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13985 assert!(
13986 runner.state.candidates[0].empty,
13987 "old verdict must remain unusable"
13988 );
13989 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13990 assert!(runner.state.judge_skipped);
13991 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13992 assert!(
13993 crate::run::read_artifact(&runner.state, "cand-A.patch")
13994 .unwrap()
13995 .contains("owned edits")
13996 );
13997 assert!(wt.exists());
13998 assert_eq!(
13999 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14000 .await
14001 .unwrap(),
14002 1
14003 );
14004 runner.state = RunState::load(&runner.state.id).unwrap();
14005 }
14006 }
14007
14008 #[tokio::test]
14009 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14010 let (_tmp, mut runner) = preservation_resume_fixture().await;
14011 let wt = runner.state.candidates[0].worktree.clone();
14012 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14013 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14014 let index_lock = PathBuf::from(
14015 git::git(
14016 &wt,
14017 &[
14018 "rev-parse",
14019 "--path-format=absolute",
14020 "--git-path",
14021 "index.lock",
14022 ],
14023 )
14024 .await
14025 .unwrap(),
14026 );
14027 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14028 runner.state.schema = 14;
14029 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14030 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14031 assert!(runner.implement().await.is_err());
14032 assert!(runner.after_implement().is_err());
14033 assert!(runner.fold_losers().await.is_err());
14034 assert_eq!(
14035 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14036 "failed seat edits"
14037 );
14038 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14039 runner.state = RunState::load(&runner.state.id).unwrap();
14040 std::fs::remove_file(index_lock).unwrap();
14041 assert!(runner.implement().await.is_err());
14043 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14044 assert!(runner.state.candidates[0].empty);
14045 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14046 assert!(
14047 crate::run::read_artifact(&runner.state, "cand-A.patch")
14048 .unwrap()
14049 .contains("failed seat edits")
14050 );
14051 assert!(wt.exists());
14052 }
14053
14054 #[tokio::test]
14055 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14056 let (_tmp, mut runner) = preservation_resume_fixture().await;
14057 let repo = runner.state.repo.clone();
14058 let common = git::git(
14059 &repo,
14060 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14061 )
14062 .await
14063 .unwrap();
14064 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14065 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14066 assert!(runner.fold_losers().await.is_err());
14067 assert!(!runner.state.candidates[0].worktree.exists());
14068 assert!(!runner.state.candidates[0].folded);
14069 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14070 runner.state = RunState::load(&runner.state.id).unwrap();
14071 std::fs::remove_file(branch_lock).unwrap();
14072 let judge = runner.state.worktree_root().join("judge-1");
14073 git::worktree_add_detached(&repo, &judge, "main")
14074 .await
14075 .unwrap();
14076 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14077 runner.roles.judges = vec![spec("judge")];
14078 assert!(runner.fold_losers().await.is_err());
14079 runner.state = RunState::load(&runner.state.id).unwrap();
14080 assert!(
14081 runner.state.candidates[0].folded,
14082 "candidate progress saved before judge failure"
14083 );
14084 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14085 assert!(judge.join("uncommitted").exists());
14086 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14087 runner.fold_losers().await.unwrap();
14088 assert!(!judge.exists());
14089 runner.fold_losers().await.unwrap();
14090 }
14091
14092 #[tokio::test]
14093 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14094 let (_tmp, mut runner) = preservation_resume_fixture().await;
14095 runner.state.candidates.retain(|c| c.label == 'B');
14096 runner.state.tally = None;
14097 runner.state.judge_skipped = false;
14098 runner.state.prep_complete = Some(true);
14099 runner.state.config.graph.advise = false;
14100 runner.state.config.graph.selected_implementer = Some("alpha".into());
14101 let wt = runner.state.candidates[0].worktree.clone();
14102 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14103 runner
14104 .state
14105 .candidate_preservation
14106 .insert('B', "awaiting persistence".into());
14107 let index_lock = git::git(
14108 &wt,
14109 &[
14110 "rev-parse",
14111 "--path-format=absolute",
14112 "--git-path",
14113 "index.lock",
14114 ],
14115 )
14116 .await
14117 .unwrap();
14118 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14119 runner.state.save().unwrap();
14120 assert!(runner.execute_graph().await.is_err());
14121 assert_eq!(runner.state.status, RunStatus::Blocked);
14122 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14123 std::fs::remove_file(index_lock).unwrap();
14124 for _ in 0..2 {
14127 runner.state = RunState::load(&runner.state.id).unwrap();
14128 runner.pause.park();
14129 runner.execute_graph().await.unwrap();
14130 assert_eq!(runner.state.status, RunStatus::Implementing);
14131 assert!(runner.state.candidate_preservation.is_empty());
14132 assert_eq!(runner.state.candidates[0].commits, 2);
14133 assert!(runner.state.tally.is_none());
14134 }
14135 runner.implement().await.unwrap();
14136 assert_eq!(runner.state.status, RunStatus::Judging);
14137 runner.state.status = RunStatus::Blocked;
14139 runner.state.candidates[0].failed = Some("implementation failed".into());
14140 runner.state.save().unwrap();
14141 let error = runner.execute_graph().await.unwrap_err();
14142 assert!(
14143 error
14144 .to_string()
14145 .contains("selected implementation is held")
14146 );
14147 }
14148
14149 #[tokio::test]
14150 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14151 let (_tmp, mut runner) = preservation_resume_fixture().await;
14152 let repo = runner.state.repo.clone();
14153 let candidate = runner.state.candidates[0].clone();
14154 let registered = git::worktree_holding(&repo, &candidate.branch)
14157 .await
14158 .unwrap()
14159 .unwrap();
14160 assert_eq!(
14161 registered.canonicalize().unwrap(),
14162 candidate.worktree.canonicalize().unwrap()
14163 );
14164 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14165 assert_eq!(
14166 git::worktree_holding(&repo, &candidate.branch)
14167 .await
14168 .unwrap(),
14169 Some(registered)
14170 );
14171 assert!(
14172 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14173 "the stale registration initially keeps the branch checked out"
14174 );
14175 runner.fold_losers().await.unwrap();
14176 assert!(runner.state.candidates[0].folded);
14177 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14178 assert_eq!(
14179 git::worktree_holding(&repo, &candidate.branch)
14180 .await
14181 .unwrap(),
14182 None
14183 );
14184 runner.state = RunState::load(&runner.state.id).unwrap();
14185 runner.fold_losers().await.unwrap();
14186 }
14187
14188 #[tokio::test]
14189 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14190 let (_tmp, mut runner) = preservation_resume_fixture().await;
14191 let repo = runner.state.repo.clone();
14192 let candidate = runner.state.candidates[0].clone();
14193 let winner = runner.state.candidates[1].clone();
14194 let registered = git::worktree_holding(&repo, &candidate.branch)
14195 .await
14196 .unwrap()
14197 .unwrap();
14198 assert_eq!(
14199 registered.canonicalize().unwrap(),
14200 candidate.worktree.canonicalize().unwrap()
14201 );
14202 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14203 assert_eq!(
14204 git::worktree_holding(&repo, &candidate.branch)
14205 .await
14206 .unwrap(),
14207 Some(registered)
14208 );
14209 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14210
14211 for _ in 0..2 {
14212 fold_run(&mut runner.state, false, &crate::run::home())
14213 .await
14214 .unwrap();
14215 runner.state = RunState::load(&runner.state.id).unwrap();
14216 assert!(runner.state.candidates[0].folded);
14217 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14218 assert_eq!(
14219 git::worktree_holding(&repo, &candidate.branch)
14220 .await
14221 .unwrap(),
14222 None
14223 );
14224 assert!(!runner.state.candidates[1].folded);
14225 assert!(winner.worktree.exists());
14226 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14227 }
14228 }
14229
14230 #[tokio::test]
14231 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14232 ask_test_home();
14233 let tmp = tempfile::tempdir().unwrap();
14234 let repo = tmp.path().join("repo");
14235 std::fs::create_dir(&repo).unwrap();
14236 init_repo(&repo);
14237 let mut runner = runner_at(RunStatus::Implementing);
14238 runner.state.repo = repo.clone();
14239 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14240 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14241 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14242 let branch = format!("candidate-{label}");
14243 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14244 git::worktree_add_branch(&repo, &wt, &branch, "main")
14245 .await
14246 .unwrap();
14247 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14248 git::git(&wt, &["add", "README.md"]).await.unwrap();
14249 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14250 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14251 runner.state.candidates.push(Candidate {
14252 index,
14253 label,
14254 agent: "alpha".to_owned(),
14255 branch,
14256 worktree: wt,
14257 summary: "completed implementation".to_owned(),
14258 stat: String::new(),
14259 files: 0,
14260 commits: 0,
14261 empty: false,
14262 failed: None,
14263 verified_noop: None,
14264 duration_ms: 42,
14265 folded: false,
14266 });
14267 runner
14268 .state
14269 .candidate_preservation
14270 .insert(label, "awaiting persistence".to_owned());
14271 }
14272 let wt = runner.state.candidates[0].worktree.clone();
14273 let index_lock = git::git(
14274 &wt,
14275 &[
14276 "rev-parse",
14277 "--path-format=absolute",
14278 "--git-path",
14279 "index.lock",
14280 ],
14281 )
14282 .await
14283 .unwrap();
14284 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14285 runner.state.save().unwrap();
14286 assert!(runner.persist_candidates().await.is_err());
14287 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14288 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14289 assert_eq!(
14290 runner.state.candidates[1].commits, 1,
14291 "other results are still collected"
14292 );
14293 assert!(
14294 !runner.state.candidates[0].empty,
14295 "failed rescue is not no change"
14296 );
14297 assert_eq!(
14298 runner.state.candidates[0].summary,
14299 "completed implementation"
14300 );
14301 assert!(runner.after_implement().is_err());
14302 let home = tmp.path().join("fold-home");
14303 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14304 assert!(wt.join("new-file").exists());
14305 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14306 assert!(recovery.join("staged.patch").exists());
14307 assert!(recovery.join("unstaged.patch").exists());
14308 assert_eq!(
14309 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14310 "untracked edits\n"
14311 );
14312 runner.state.begin_pass();
14313 runner.state.end_pass_parked(ParkCause::Upgrade {
14314 from: "1.0".into(),
14315 to: Some("1.1".into()),
14316 });
14317 runner.state.save().unwrap();
14318 runner.state = RunState::load(&runner.state.id).unwrap();
14319 assert!(matches!(
14320 runner.state.passes.last().unwrap().end,
14321 Some(crate::run::PassEnd::Parked {
14322 cause: ParkCause::Upgrade { .. }
14323 })
14324 ));
14325 std::fs::remove_file(index_lock).unwrap();
14326 runner.implement().await.unwrap();
14328 assert!(runner.state.candidate_preservation.is_empty());
14329 assert!(
14330 runner
14331 .state
14332 .candidates
14333 .iter()
14334 .all(|c| c.commits == 1 && !c.empty)
14335 );
14336 assert!(git::is_clean(&wt).await.unwrap());
14337 assert!(
14338 crate::run::read_artifact(&runner.state, "cand-A.patch")
14339 .unwrap()
14340 .contains("later tracked edits")
14341 );
14342 }
14343
14344 #[tokio::test]
14345 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14346 ask_test_home();
14347 let tmp = tempfile::tempdir().unwrap();
14348 let repo = tmp.path().join("repo");
14349 std::fs::create_dir(&repo).unwrap();
14350 init_repo(&repo);
14351 let wt = tmp.path().join("candidate");
14352 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14353 .await
14354 .unwrap();
14355 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14356 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14357 assert!(wt.join("uncommitted").exists());
14358 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14359 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14360 let mut runner = runner_at(RunStatus::Implementing);
14361 runner.state.repo = repo.clone();
14362 runner.state.base_commit = "missing-base".to_owned();
14363 runner.state.candidates.push(Candidate {
14364 index: 0,
14365 label: 'A',
14366 agent: "alpha".into(),
14367 branch: "candidate".into(),
14368 worktree: wt.clone(),
14369 summary: "completed".into(),
14370 stat: String::new(),
14371 files: 0,
14372 commits: 0,
14373 empty: false,
14374 failed: None,
14375 verified_noop: None,
14376 duration_ms: 1,
14377 folded: false,
14378 });
14379 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14380 assert!(
14381 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14382 .await
14383 .is_err()
14384 );
14385 assert!(!runner.state.candidates[0].folded);
14386 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14387 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14388 runner
14389 .state
14390 .candidate_preservation
14391 .insert('A', "pending".into());
14392 assert!(runner.persist_candidates().await.is_err());
14393 assert!(!runner.state.candidates[0].empty);
14394 assert!(!runner.state.candidates[0].folded);
14395 assert!(wt.exists());
14396 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14397 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14398 std::fs::create_dir_all(&patch_path).unwrap();
14399 assert!(
14400 runner.persist_candidates().await.is_err(),
14401 "artifact failure stays pending"
14402 );
14403 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14404 assert!(!runner.state.candidates[0].empty);
14405 std::fs::remove_dir(patch_path).unwrap();
14406 runner.persist_candidates().await.unwrap();
14407 assert!(
14408 runner.state.candidates[0].empty,
14409 "only successful inspection confirms no change"
14410 );
14411 }
14412
14413 fn init_repo(dir: &Path) {
14414 let run = |args: &[&str]| {
14415 let out = std::process::Command::new("git")
14416 .args(args)
14417 .current_dir(dir)
14418 .quiet()
14419 .output()
14420 .expect("spawn git");
14421 assert!(
14422 out.status.success(),
14423 "git {args:?} failed: {}",
14424 String::from_utf8_lossy(&out.stderr)
14425 );
14426 };
14427 run(&["init", "-b", "main"]);
14428 run(&["config", "user.name", "magi test"]);
14429 run(&["config", "user.email", "magi@example.com"]);
14430 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14431 run(&["add", "-A"]);
14432 run(&["commit", "-m", "init"]);
14433 }
14434
14435 fn ask_test_home() {
14443 crate::run::pin_test_home();
14444 }
14445
14446 fn runner_at(status: RunStatus) -> Runner {
14449 let mut state = RunState::new(
14450 PathBuf::from("/nonexistent/repo"),
14451 "main".to_owned(),
14452 "deadbeef".to_owned(),
14453 "task".to_owned(),
14454 Config::default(),
14455 );
14456 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14459 state.id = format!(
14460 "20000101-000000-{:04x}",
14461 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14462 );
14463 state.status = status;
14464 Runner {
14465 state,
14466 roles: ResolvedRoles {
14467 implementers: Vec::new(),
14468 judges: Vec::new(),
14469 reviewers: Vec::new(),
14470 fixer: None,
14471 conductor: conductor(),
14472 implementer_roster: Vec::new(),
14473 judge_roster: Vec::new(),
14474 reviewer_roster: Vec::new(),
14475 },
14476 sem: Arc::new(Semaphore::new(1)),
14477 pause: Pause::new(),
14478 interrupt: Pause::new(),
14479 }
14480 }
14481
14482 #[test]
14483 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14484 crate::run::pin_test_home();
14485 let mut runner = runner_at(RunStatus::Gating);
14486 runner.state.save().unwrap();
14487 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14490 assert!(runner.claim_driver().is_err());
14491 assert!(runner.state.driver_exited);
14492 assert_eq!(runner.state.driver_pid, None);
14493 let disk = RunState::load(&runner.state.id).unwrap();
14494 assert!(disk.driver_exited);
14495 assert_eq!(disk.driver_pid, None);
14496 assert_eq!(disk.status, RunStatus::Gating);
14497 }
14498
14499 #[tokio::test]
14500 async fn an_early_graph_error_releases_its_driver() {
14501 crate::run::pin_test_home();
14502 let mut runner = runner_at(RunStatus::Blocked);
14503 runner.state.config.graph.selected_implementer = Some("selected".into());
14504 runner.state.save().unwrap();
14505 assert!(runner.execute().await.is_err());
14506 let disk = RunState::load(&runner.state.id).unwrap();
14507 assert!(disk.driver_exited);
14508 assert_eq!(disk.driver_pid, None);
14509 assert_eq!(disk.status, RunStatus::Blocked);
14510 }
14511
14512 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14513 crate::run::MergeRepair {
14514 question: "repair-question".into(),
14515 head: "old-head".into(),
14516 pr: "https://github.com/o/r/pull/42".into(),
14517 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14518 gate: vec![outcome(Some(0), false)],
14519 gate_head: Some("old-head".into()),
14520 gate_fixes: Vec::new(),
14521 pre_gate: Vec::new(),
14522 phase,
14523 failure: None,
14524 }
14525 }
14526
14527 #[tokio::test]
14528 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14529 crate::run::pin_test_home();
14530 let mut runner = runner_at(RunStatus::Landing);
14531 runner.state.merge_repairs.push(merge_repair_request(
14532 crate::run::MergeRepairPhase::Requested,
14533 ));
14534 runner.state.save().unwrap();
14535 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14536 assert!(
14537 runner
14538 .execute()
14539 .await
14540 .unwrap_err()
14541 .to_string()
14542 .contains("already running")
14543 );
14544 assert_eq!(
14545 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14546 crate::run::MergeRepairPhase::Requested
14547 );
14548 drop(claim);
14549 let mut disk = runner.state.clone();
14550 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14551 disk.save().unwrap();
14552 assert!(
14553 runner
14554 .execute()
14555 .await
14556 .unwrap_err()
14557 .to_string()
14558 .contains("decision changed")
14559 );
14560 assert_eq!(
14561 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14562 crate::run::MergeRepairPhase::Complete
14563 );
14564 }
14565
14566 #[test]
14567 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14568 crate::run::pin_test_home();
14569 let mut runner = runner_at(RunStatus::Landing);
14570 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14571 runner.state.reviews = request.reviews.clone();
14572 runner.state.gate = request.gate.clone();
14573 runner.state.gate_ran = true;
14574 runner.state.land_approval = Some(crate::run::LandApproval {
14575 question: "old".into(),
14576 head: "old-head".into(),
14577 });
14578 runner.state.merge_repairs.push(request);
14579 runner.begin_merge_repair_review(0).unwrap();
14580 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14581 assert!(!runner.state.gate_status().ok());
14582 assert!(runner.state.land_approval.is_none());
14583 let disk = RunState::load(&runner.state.id).unwrap();
14584 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14585 assert!(disk.merge_repairs[0].gate[0].ok());
14586 assert_eq!(
14587 disk.merge_repairs[0].phase,
14588 crate::run::MergeRepairPhase::Reviewing
14589 );
14590 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14591 assert!(disk.reviews.is_empty());
14592 }
14593
14594 #[tokio::test]
14595 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14596 crate::run::pin_test_home();
14597 for phase in [
14598 crate::run::MergeRepairPhase::Fixing,
14599 crate::run::MergeRepairPhase::Requested,
14600 ] {
14601 let mut runner = runner_at(RunStatus::Landing);
14602 runner.state.config.graph.land_rounds = 0;
14603 runner.state.merge_repairs.push(merge_repair_request(phase));
14604 runner.state.save().unwrap();
14605 assert!(!runner.repair_merge_review().await.unwrap());
14606 assert_eq!(runner.state.status, RunStatus::Blocked);
14607 assert_eq!(
14608 runner.state.merge_repairs[0].phase,
14609 crate::run::MergeRepairPhase::Failed
14610 );
14611 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14612 assert!(
14613 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14614 "interrupted"
14615 } else {
14616 "budget exhausted"
14617 })
14618 );
14619 assert!(runner.state.merge.is_none());
14620 }
14621 }
14622
14623 #[test]
14627 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14628 crate::run::pin_test_home();
14629 let mut runner = runner_at(RunStatus::Implementing);
14630 let interrupt = Pause::new();
14631 runner.watch_interrupt(interrupt.clone());
14632
14633 interrupt.park_because("task a1b2 asked to run first");
14634
14635 assert!(runner.park_here().expect("park_here"));
14636 assert!(runner.state.parked);
14637 let last = runner.state.events.last().expect("a park event");
14638 assert_eq!(last.node, "park");
14639 assert!(
14640 last.message.contains("task a1b2 asked to run first"),
14641 "expected the interrupt reason in {:?}",
14642 last.message
14643 );
14644 }
14645
14646 #[test]
14649 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14650 crate::run::pin_test_home();
14651 let mut runner = runner_at(RunStatus::Implementing);
14652 runner.state.begin_pass();
14653 let shutdown = Pause::new();
14654 runner.on_pause(shutdown.clone());
14655 let upgrade = ParkCause::Upgrade {
14656 from: "0.121.0".to_owned(),
14657 to: Some("0.121.1".to_owned()),
14658 };
14659 shutdown.park_for(upgrade.clone());
14660 shutdown.park_for(ParkCause::Operator);
14661
14662 assert!(runner.park_here().expect("park_here"));
14663 assert_eq!(
14664 runner.state.passes[0].end,
14665 Some(crate::run::PassEnd::Parked { cause: upgrade })
14666 );
14667 let last = runner.state.events.last().expect("a park event");
14668 assert!(last.message.contains("0.121.0"), "{}", last.message);
14669 }
14670
14671 #[test]
14679 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14680 crate::run::pin_test_home();
14681 let mut runner = runner_at(RunStatus::Implementing);
14682 let shutdown = Pause::new();
14683 runner.on_pause(shutdown.clone());
14684 let interrupt = Pause::new();
14685 runner.watch_interrupt(interrupt.clone());
14686
14687 assert!(!runner.park_here().expect("park_here"));
14689 assert!(!runner.state.parked);
14690
14691 interrupt.park_because("test");
14693 assert!(!shutdown.parked());
14694 assert!(runner.park_here().expect("park_here"));
14695 }
14696
14697 #[tokio::test]
14711 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14712 crate::run::pin_test_home();
14713 let mut runner = runner_at(RunStatus::Implementing);
14714 let interrupt = Pause::new();
14715 runner.watch_interrupt(interrupt.clone());
14716
14717 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14718 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14719
14720 let node = async move {
14724 started_tx.send(()).expect("send started");
14725 finish_rx.await.expect("recv finish");
14726 "node finished"
14727 };
14728
14729 let interrupter = async move {
14730 started_rx.await.expect("recv started");
14731 interrupt.park_because("higher-priority task waiting");
14733 tokio::task::yield_now().await;
14737 finish_tx.send(()).expect("send finish");
14738 };
14739
14740 let (node_result, ()) = tokio::join!(node, interrupter);
14741 assert_eq!(
14742 node_result, "node finished",
14743 "the in-flight call ran to completion"
14744 );
14745
14746 assert!(runner.park_here().expect("park_here"));
14749 assert!(runner.state.parked);
14750 }
14751
14752 #[test]
14758 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14759 crate::run::pin_test_home();
14760 let mut runner = runner_at(RunStatus::Judging);
14761 runner.state.config.agents = vec![conductor()];
14765 runner.state.candidates = vec![Candidate {
14766 index: 0,
14767 label: 'A',
14768 agent: "alpha".to_owned(),
14769 branch: "magi/x/A".to_owned(),
14770 worktree: PathBuf::from("/nonexistent/worktree"),
14771 summary: "did the thing".to_owned(),
14772 stat: "1 file changed".to_owned(),
14773 files: 1,
14774 commits: 1,
14775 empty: false,
14776 failed: None,
14777 verified_noop: None,
14778 duration_ms: 1234,
14779 folded: false,
14780 }];
14781 let run_id = runner.state.id.clone();
14782
14783 let interrupt = Pause::new();
14784 runner.watch_interrupt(interrupt.clone());
14785 interrupt.park_because("task c3d4 asked to run first");
14786 assert!(runner.park_here().expect("park_here"));
14787
14788 let resumed = Runner::resume(&run_id).expect("resume");
14789 assert_eq!(resumed.state.candidates.len(), 1);
14790 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14791 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14792 assert_eq!(resumed.state.status, runner.state.status);
14793 assert!(
14794 resumed.state.parked,
14795 "still parked until `execute` actually walks the graph again"
14796 );
14797 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14798 }
14799
14800 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14802 let mut q = ask::Question::new(
14803 run.to_owned(),
14804 "implement".to_owned(),
14805 "impl-A".to_owned(),
14806 "Which storage backend should the cache use?".to_owned(),
14807 String::new(),
14808 vec!["SQLite".to_owned(), "Redis".to_owned()],
14809 );
14810 store.put(&mut q).unwrap();
14811 q
14812 }
14813
14814 #[test]
14815 fn a_failed_runs_open_question_is_abandoned() {
14816 ask_test_home();
14817 let store = ask::Questions::open();
14818 let mut runner = runner_at(RunStatus::Failed);
14819 let run = runner.state.id.clone();
14820 let q = ask_open_question(&store, &run);
14821
14822 runner.settle_questions();
14823
14824 let back = store.get(&q.id).unwrap();
14825 assert!(
14826 !back.status.open(),
14827 "the seat that asked died with the run; nobody is left to read an answer"
14828 );
14829 assert!(
14830 back.detail.contains(&run) && back.detail.contains("failed"),
14831 "the reason names what the run became, not just that it is gone: {}",
14832 back.detail
14833 );
14834 }
14835
14836 #[test]
14837 fn a_merged_runs_open_question_is_abandoned_too() {
14838 ask_test_home();
14839 let store = ask::Questions::open();
14840 for status in [RunStatus::Merged, RunStatus::Ready] {
14843 let mut runner = runner_at(status);
14844 let run = runner.state.id.clone();
14845 let q = ask_open_question(&store, &run);
14846
14847 runner.settle_questions();
14848
14849 let back = store.get(&q.id).unwrap();
14850 assert!(
14851 !back.status.open(),
14852 "{status:?} run's question must not outlive the run"
14853 );
14854 }
14855 }
14856
14857 #[test]
14858 fn a_still_resumable_runs_open_question_is_left_alone() {
14859 ask_test_home();
14860 let store = ask::Questions::open();
14861 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14867 let mut runner = runner_at(status);
14868 let run = runner.state.id.clone();
14869 let q = ask_open_question(&store, &run);
14870
14871 runner.settle_questions();
14872
14873 let back = store.get(&q.id).unwrap();
14874 assert!(
14875 back.status.open(),
14876 "{status:?} is still alive; the question must still be waiting"
14877 );
14878 }
14879 }
14880
14881 #[test]
14882 fn settle_questions_never_touches_an_already_answered_question() {
14883 ask_test_home();
14884 let store = ask::Questions::open();
14885 let mut runner = runner_at(RunStatus::Failed);
14886 let run = runner.state.id.clone();
14887 let mut q = ask_open_question(&store, &run);
14888 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14889 .unwrap();
14890 store.put(&mut q).unwrap();
14891
14892 runner.settle_questions();
14897 runner.settle_questions();
14898
14899 let back = store.get(&q.id).unwrap();
14900 assert_eq!(
14901 back.status,
14902 ask::QuestionStatus::Answered,
14903 "a real answer is a decision on record, never overwritten by a sweep"
14904 );
14905 }
14906
14907 #[tokio::test]
14918 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14919 crate::run::pin_test_home();
14920 let tmp = tempfile::tempdir().expect("tempdir");
14921 let repo = tmp.path().join("repo");
14922 std::fs::create_dir_all(&repo).unwrap();
14923 init_repo(&repo);
14924
14925 let mut config = Config::default();
14926 config.graph.worktree_root = Some(tmp.path().join("wt"));
14927
14928 let mut state = RunState::new(
14929 repo.clone(),
14930 "main".to_owned(),
14931 "deadbeef".to_owned(),
14932 "task".to_owned(),
14933 config,
14934 );
14935 let root = state.worktree_root();
14936 let wt_a = root.join("cand-A");
14937 let wt_b = root.join("cand-B");
14938 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14939 .await
14940 .expect("worktree A");
14941 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14942 .await
14943 .expect("worktree B");
14944
14945 state.candidates = vec![
14946 Candidate {
14947 index: 0,
14948 label: 'A',
14949 agent: "alpha".to_owned(),
14950 branch: "magi/x/A".to_owned(),
14951 worktree: wt_a.clone(),
14952 summary: String::new(),
14953 stat: String::new(),
14954 files: 0,
14955 commits: 0,
14956 empty: false,
14957 failed: None,
14958 verified_noop: None,
14959 duration_ms: 0,
14960 folded: false,
14961 },
14962 Candidate {
14963 index: 1,
14964 label: 'B',
14965 agent: "beta".to_owned(),
14966 branch: "magi/x/B".to_owned(),
14967 worktree: wt_b.clone(),
14968 summary: String::new(),
14969 stat: String::new(),
14970 files: 0,
14971 commits: 0,
14972 empty: false,
14973 failed: None,
14974 verified_noop: None,
14975 duration_ms: 0,
14976 folded: false,
14977 },
14978 ];
14979 state.tally = Some(Tally {
14980 first_choice: BTreeMap::from([('A', 1)]),
14981 borda: BTreeMap::new(),
14982 winner: 'A',
14983 rankings: 1,
14984 unanimous_initial: true,
14985 deliberated: false,
14986 changed_votes: 0,
14987 unanimous_final: true,
14988 tie_break: None,
14989 judges: 1,
14990 present: 1,
14991 quorum: 1,
14992 met_quorum: true,
14993 uncontested: None,
14994 });
14995 state.status = RunStatus::Ready;
14996
14997 fold_run(&mut state, false, &crate::run::home())
14998 .await
14999 .expect("fold_run");
15000
15001 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15002 assert!(
15003 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15004 "the unmerged winner's branch survives"
15005 );
15006 assert!(
15007 !state.candidates[0].folded,
15008 "the winner is not marked folded"
15009 );
15010
15011 assert!(!wt_b.exists(), "the loser's worktree is removed");
15012 assert!(
15013 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15014 "the loser's branch is removed"
15015 );
15016 assert!(state.candidates[1].folded, "the loser is marked folded");
15017 }
15018
15019 #[tokio::test]
15020 #[cfg(unix)]
15021 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15022 use std::os::fd::AsRawFd as _;
15023 use std::time::Duration;
15024
15025 let tmp = tempfile::tempdir().unwrap();
15026 let repo = tmp.path().join("repo");
15027 std::fs::create_dir_all(&repo).unwrap();
15028 init_repo(&repo);
15029 let common = repo.join(".git");
15030 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15031 let home = tmp.path().join("home");
15032 let mut config = Config::default();
15033 config.coordination = crate::config::test_coordination();
15034 config.graph.worktree_root = Some(tmp.path().join("wt"));
15035 config.graph.timeout_resource_wait = 1;
15036 let mut state = RunState::new(
15037 repo.clone(),
15038 "main".into(),
15039 "deadbeef".into(),
15040 "task".into(),
15041 config,
15042 );
15043 let root = state.worktree_root();
15044 let candidate = root.join("cand-A");
15045 let reviewer = root.join("review-1");
15046 let branch = state.branch_for('A');
15047 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15048 .await
15049 .unwrap();
15050 git::worktree_add_detached(&repo, &reviewer, "main")
15051 .await
15052 .unwrap();
15053 git::acquire_worktree_config(&repo).await.unwrap();
15054 state.enabled_worktree_config = true;
15055 state.status = RunStatus::Ready;
15056 state.candidates.push(Candidate {
15057 index: 0,
15058 label: 'A',
15059 agent: "implement".into(),
15060 branch: branch.clone(),
15061 worktree: candidate.clone(),
15062 summary: String::new(),
15063 stat: String::new(),
15064 files: 0,
15065 commits: 0,
15066 empty: false,
15067 failed: None,
15068 verified_noop: None,
15069 duration_ms: 0,
15070 folded: false,
15071 });
15072 state.save_under(&home).unwrap();
15073 let config_before = std::fs::read(common.join("config")).unwrap();
15074 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15075 .await
15076 .unwrap();
15077 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15080 assert_eq!(
15081 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15082 0
15083 );
15084 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15085 assert!(
15086 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15087 .await
15088 .is_err()
15089 );
15090 assert!(candidate.exists());
15091 assert!(reviewer.exists());
15092 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15093 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15094 assert_eq!(
15095 git::git(&repo, &["worktree", "list", "--porcelain"])
15096 .await
15097 .unwrap(),
15098 worktrees_before
15099 );
15100 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15101 .await
15102 .unwrap()
15103 .unwrap_err();
15104 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15105 drop(fold);
15106 assert!(candidate.exists() && reviewer.exists());
15107 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15108 assert_eq!(
15109 state.resource_block.as_ref().unwrap().kind,
15110 "repository coordination"
15111 );
15112 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15113 drop(external);
15114 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15115 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15116 .await
15117 .unwrap()
15118 .unwrap();
15119 drop(fold);
15120 assert!(removed.contains(&branch));
15121 assert!(!root.exists());
15122 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15123 assert!(state.candidates[0].folded);
15124 assert!(!state.enabled_worktree_config);
15125 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15126 let enabled = git::git_raw(
15127 &repo,
15128 &["config", "--local", "--get", "extensions.worktreeConfig"],
15129 )
15130 .await
15131 .unwrap();
15132 assert!(!enabled.ok(), "the last config reference was released");
15133 }
15134
15135 #[tokio::test]
15138 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15139 let tmp = tempfile::tempdir().expect("tempdir");
15140 let repo = tmp.path().join("repo");
15141 std::fs::create_dir_all(&repo).unwrap();
15142 init_repo(&repo);
15143 let home = tmp.path().join("home");
15144
15145 let mut config = Config::default();
15146 config.graph.worktree_root = Some(tmp.path().join("wt"));
15147 let mut state = RunState::new(
15148 repo.clone(),
15149 "main".to_owned(),
15150 "deadbeef".to_owned(),
15151 "task".to_owned(),
15152 config,
15153 );
15154 git::git(&repo, &["branch", "magi/x/A", "main"])
15156 .await
15157 .expect("branch");
15158 state.candidates = vec![Candidate {
15159 index: 0,
15160 label: 'A',
15161 agent: "alpha".to_owned(),
15162 branch: "magi/x/A".to_owned(),
15163 worktree: state.worktree_root().join("cand-A"),
15164 summary: String::new(),
15165 stat: String::new(),
15166 files: 0,
15167 commits: 0,
15168 empty: false,
15169 failed: None,
15170 verified_noop: None,
15171 duration_ms: 0,
15172 folded: true,
15173 }];
15174 state.released_to = Some("20260901-000000-new1".to_owned());
15175 state.released_branches = vec!["magi/x/A".to_owned()];
15176
15177 fold_run(&mut state, true, &home).await.expect("fold_run");
15178
15179 assert!(
15180 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15181 "the handed-over branch survives a fold"
15182 );
15183 }
15184
15185 #[tokio::test]
15189 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15190 let tmp = tempfile::tempdir().expect("tempdir");
15191 let repo = tmp.path().join("repo");
15192 std::fs::create_dir_all(&repo).unwrap();
15193 init_repo(&repo);
15194 let run = |args: &[&str]| {
15195 let out = std::process::Command::new("git")
15196 .quiet()
15197 .args(args)
15198 .current_dir(&repo)
15199 .output()
15200 .expect("spawn git");
15201 assert!(out.status.success(), "git {args:?}");
15202 };
15203 run(&["branch", "magi/x/A"]);
15204 run(&["checkout", "-q", "-b", "magi/x/B"]);
15205 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15206 run(&["add", "-A"]);
15207 run(&["commit", "-q", "-m", "work"]);
15208 run(&["checkout", "-q", "main"]);
15209
15210 let probe = RunState::new(
15214 repo.clone(),
15215 "main".to_owned(),
15216 "deadbeef".to_owned(),
15217 "task".to_owned(),
15218 Config::default(),
15219 );
15220 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15221 let bare = tmp.path().join("origin.git");
15222 let out = std::process::Command::new("git")
15223 .quiet()
15224 .args(["init", "-q", "--bare"])
15225 .arg(&bare)
15226 .output()
15227 .expect("spawn git");
15228 assert!(out.status.success(), "git init --bare");
15229 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15230 run(&["push", "-q", "origin", "main"]);
15231
15232 let mut state = RunState::new(
15233 repo.clone(),
15234 "main".to_owned(),
15235 "deadbeef".to_owned(),
15236 "task".to_owned(),
15237 Config::default(),
15238 );
15239 state.seeds = vec![refs::Seed {
15240 token: "magi/27b2/A".to_owned(),
15241 kind: refs::SeedKind::Unresolved,
15242 sha: String::new(),
15243 branch: true,
15244 detail: "no branch or commit named magi/27b2/A".to_owned(),
15245 }];
15246
15247 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15248 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15249 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15250 let detail = empty_candidate_detail(&state, "main");
15251 assert!(detail.starts_with("empty candidate"), "{detail}");
15252 assert!(detail.contains("magi/27b2/A"), "{detail}");
15253 }
15254
15255 #[tokio::test]
15264 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15265 let tmp = tempfile::tempdir().expect("tempdir");
15266 let repo = tmp.path().join("repo");
15267 std::fs::create_dir_all(&repo).unwrap();
15268 init_repo(&repo);
15269
15270 let mut config = Config::default();
15271 config.merge.mode = MergeMode::Local;
15272
15273 let mut state = RunState::new(
15274 repo.clone(),
15275 "main".to_owned(),
15276 "deadbeef".to_owned(),
15277 "task".to_owned(),
15278 config,
15279 );
15280 state.candidates = vec![Candidate {
15281 index: 0,
15282 label: 'A',
15283 agent: "alpha".to_owned(),
15284 branch: "does-not-exist".to_owned(),
15285 worktree: repo.clone(),
15286 summary: String::new(),
15287 stat: String::new(),
15288 files: 0,
15289 commits: 0,
15290 empty: false,
15291 failed: None,
15292 verified_noop: None,
15293 duration_ms: 0,
15294 folded: false,
15295 }];
15296 state.tally = Some(Tally {
15297 first_choice: BTreeMap::from([('A', 1)]),
15298 borda: BTreeMap::new(),
15299 winner: 'A',
15300 rankings: 1,
15301 unanimous_initial: true,
15302 deliberated: false,
15303 changed_votes: 0,
15304 unanimous_final: true,
15305 tie_break: None,
15306 judges: 0,
15307 present: 0,
15308 quorum: 0,
15309 met_quorum: true,
15310 uncontested: Some("only candidate A produced a change".to_owned()),
15311 });
15312 state.reviews = vec![ReviewRound {
15313 round: 1,
15314 head: "deadbeef".to_owned(),
15315 verified_head: None,
15316 verified_at: None,
15317 reviews: Vec::new(),
15318 e2e: Vec::new(),
15319 fix: None,
15320 blocking: 0,
15321 answered: 0,
15322 expected: 0,
15323 clean: true,
15324 verify_retried: false,
15325 e2e_deferred: false,
15326 e2e_defer_reason: None,
15327 progressed: false,
15328 vote_split: false,
15329 reconsideration: Vec::new(),
15330 verdict: None,
15331 }];
15332 state.gate = vec![CommandOutcome {
15333 command: "test".to_owned(),
15334 code: Some(0),
15335 output_tail: String::new(),
15336 duration_ms: 0,
15337 resource_blocked: false,
15338 }];
15339 state.gate_ran = true;
15340 state.status = RunStatus::Ready;
15345 state.merge = Some(MergeOutcome {
15346 mode: MergeMode::Local,
15347 ok: false,
15348 detail: "already concluded".to_owned(),
15349 empty: false,
15350 });
15351
15352 let mut runner = Runner {
15353 state,
15354 roles: ResolvedRoles {
15355 implementers: Vec::new(),
15356 judges: Vec::new(),
15357 reviewers: Vec::new(),
15358 fixer: None,
15359 conductor: conductor(),
15360 implementer_roster: Vec::new(),
15361 judge_roster: Vec::new(),
15362 reviewer_roster: Vec::new(),
15363 },
15364 sem: Arc::new(Semaphore::new(1)),
15365 pause: Pause::new(),
15366 interrupt: Pause::new(),
15367 };
15368
15369 runner.merge().await.expect("merge");
15370
15371 assert_eq!(
15372 runner.state.status,
15373 RunStatus::Ready,
15374 "a concluded run's status must not change on reentry"
15375 );
15376 assert_eq!(
15377 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15378 Some("already concluded"),
15379 "merge must not run again once the node already recorded an outcome"
15380 );
15381 }
15382
15383 #[tokio::test]
15392 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15393 let tmp = tempfile::tempdir().expect("tempdir");
15394 let repo = tmp.path().join("repo");
15395 std::fs::create_dir_all(&repo).unwrap();
15396 init_repo(&repo);
15397
15398 let mut config = Config::default();
15399 config.merge.mode = MergeMode::Local;
15400
15401 let mut state = RunState::new(
15402 repo.clone(),
15403 "main".to_owned(),
15404 "deadbeef".to_owned(),
15405 "task".to_owned(),
15406 config,
15407 );
15408 state.candidates = vec![Candidate {
15409 index: 0,
15410 label: 'A',
15411 agent: "alpha".to_owned(),
15412 branch: "does-not-exist".to_owned(),
15413 worktree: repo.clone(),
15414 summary: String::new(),
15415 stat: String::new(),
15416 files: 0,
15417 commits: 0,
15418 empty: false,
15419 failed: None,
15420 verified_noop: None,
15421 duration_ms: 0,
15422 folded: false,
15423 }];
15424 state.tally = Some(Tally {
15425 first_choice: BTreeMap::from([('A', 1)]),
15426 borda: BTreeMap::new(),
15427 winner: 'A',
15428 rankings: 1,
15429 unanimous_initial: true,
15430 deliberated: false,
15431 changed_votes: 0,
15432 unanimous_final: true,
15433 tie_break: None,
15434 judges: 0,
15435 present: 0,
15436 quorum: 0,
15437 met_quorum: true,
15438 uncontested: Some("only candidate A produced a change".to_owned()),
15439 });
15440 state.reviews = vec![ReviewRound {
15441 round: 1,
15442 head: "deadbeef".to_owned(),
15443 verified_head: None,
15444 verified_at: None,
15445 reviews: Vec::new(),
15446 e2e: Vec::new(),
15447 fix: None,
15448 blocking: 0,
15449 answered: 0,
15450 expected: 0,
15451 clean: true,
15452 verify_retried: false,
15453 e2e_deferred: false,
15454 e2e_defer_reason: None,
15455 progressed: false,
15456 vote_split: false,
15457 reconsideration: Vec::new(),
15458 verdict: None,
15459 }];
15460 state.gate = Vec::new();
15462 state.gate_ran = false;
15463 state.status = RunStatus::Gating;
15464
15465 let mut runner = Runner {
15466 state,
15467 roles: ResolvedRoles {
15468 implementers: Vec::new(),
15469 judges: Vec::new(),
15470 reviewers: Vec::new(),
15471 fixer: None,
15472 conductor: conductor(),
15473 implementer_roster: Vec::new(),
15474 judge_roster: Vec::new(),
15475 reviewer_roster: Vec::new(),
15476 },
15477 sem: Arc::new(Semaphore::new(1)),
15478 pause: Pause::new(),
15479 interrupt: Pause::new(),
15480 };
15481
15482 runner.merge().await.expect("merge");
15483
15484 assert!(
15485 runner.state.merge.is_none(),
15486 "an empty gate must never be read as a passing one: {:?}",
15487 runner.state.merge
15488 );
15489 }
15490
15491 #[tokio::test]
15498 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15499 let tmp = tempfile::tempdir().expect("tempdir");
15500 let repo = tmp.path().join("repo");
15501 std::fs::create_dir_all(&repo).unwrap();
15502 init_repo(&repo);
15503
15504 let config = Config::default();
15506
15507 let mut state = RunState::new(
15508 repo.clone(),
15509 "main".to_owned(),
15510 "deadbeef".to_owned(),
15511 "task".to_owned(),
15512 config,
15513 );
15514 state.candidates = vec![Candidate {
15515 index: 0,
15516 label: 'A',
15517 agent: "alpha".to_owned(),
15518 branch: "does-not-exist".to_owned(),
15519 worktree: repo.clone(),
15520 summary: String::new(),
15521 stat: String::new(),
15522 files: 0,
15523 commits: 0,
15524 empty: false,
15525 failed: None,
15526 verified_noop: None,
15527 duration_ms: 0,
15528 folded: false,
15529 }];
15530 state.tally = Some(Tally {
15531 first_choice: BTreeMap::from([('A', 1)]),
15532 borda: BTreeMap::new(),
15533 winner: 'A',
15534 rankings: 1,
15535 unanimous_initial: true,
15536 deliberated: false,
15537 changed_votes: 0,
15538 unanimous_final: true,
15539 tie_break: None,
15540 judges: 0,
15541 present: 0,
15542 quorum: 0,
15543 met_quorum: true,
15544 uncontested: Some("only candidate A produced a change".to_owned()),
15545 });
15546 state.reviews = vec![ReviewRound {
15547 round: 1,
15548 head: "deadbeef".to_owned(),
15549 verified_head: None,
15550 verified_at: None,
15551 reviews: Vec::new(),
15552 e2e: Vec::new(),
15553 fix: None,
15554 blocking: 0,
15555 answered: 0,
15556 expected: 0,
15557 clean: true,
15558 verify_retried: false,
15559 e2e_deferred: false,
15560 e2e_defer_reason: None,
15561 progressed: false,
15562 vote_split: false,
15563 reconsideration: Vec::new(),
15564 verdict: None,
15565 }];
15566
15567 let mut runner = Runner {
15568 state,
15569 roles: ResolvedRoles {
15570 implementers: Vec::new(),
15571 judges: Vec::new(),
15572 reviewers: Vec::new(),
15573 fixer: None,
15574 conductor: conductor(),
15575 implementer_roster: Vec::new(),
15576 judge_roster: Vec::new(),
15577 reviewer_roster: Vec::new(),
15578 },
15579 sem: Arc::new(Semaphore::new(1)),
15580 pause: Pause::new(),
15581 interrupt: Pause::new(),
15582 };
15583
15584 runner.gate().await.expect("gate");
15585 assert!(
15586 runner.state.gate_ran,
15587 "zero configured commands is still a real attempt, not an unrun gate"
15588 );
15589 assert!(runner.state.gate.is_empty());
15590 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15591 assert_ne!(
15592 runner.state.status,
15593 RunStatus::Blocked,
15594 "a gate with nothing to check must not read as failed"
15595 );
15596
15597 runner.merge().await.expect("merge");
15598 assert_eq!(
15599 runner.state.status,
15600 RunStatus::Ready,
15601 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15602 );
15603 }
15604
15605 #[tokio::test]
15616 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15617 crate::run::pin_test_home();
15618 let home = crate::run::home();
15619
15620 let tmp = tempfile::tempdir().expect("tempdir");
15621 let repo = tmp.path().join("repo");
15622 std::fs::create_dir_all(&repo).unwrap();
15623 init_repo(&repo);
15624 let cache_dir = tmp.path().join("target");
15627
15628 let mut config = Config::default();
15629 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15630 config.graph.timeout_verify = Some(2);
15633 config.graph.timeout_resource_wait = 2;
15634
15635 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15636 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15637 .expect("no io error acquiring directly")
15638 {
15639 crate::cache::AcquireOutcome::Acquired(g) => g,
15640 crate::cache::AcquireOutcome::Busy(b) => {
15641 panic!("expected the direct acquire to win the lease first: {b:?}")
15642 }
15643 };
15644
15645 let mut state = RunState::new(
15646 repo.clone(),
15647 "main".to_owned(),
15648 "deadbeef".to_owned(),
15649 "task".to_owned(),
15650 config,
15651 );
15652 state.candidates = vec![Candidate {
15653 index: 0,
15654 label: 'A',
15655 agent: "alpha".to_owned(),
15656 branch: "does-not-exist".to_owned(),
15657 worktree: repo.clone(),
15658 summary: String::new(),
15659 stat: String::new(),
15660 files: 0,
15661 commits: 0,
15662 empty: false,
15663 failed: None,
15664 verified_noop: None,
15665 duration_ms: 0,
15666 folded: false,
15667 }];
15668 state.tally = Some(Tally {
15669 first_choice: BTreeMap::from([('A', 1)]),
15670 borda: BTreeMap::new(),
15671 winner: 'A',
15672 rankings: 1,
15673 unanimous_initial: true,
15674 deliberated: false,
15675 changed_votes: 0,
15676 unanimous_final: true,
15677 tie_break: None,
15678 judges: 0,
15679 present: 0,
15680 quorum: 0,
15681 met_quorum: true,
15682 uncontested: Some("only candidate A produced a change".to_owned()),
15683 });
15684 state.reviews = vec![ReviewRound {
15685 round: 1,
15686 head: "deadbeef".to_owned(),
15687 verified_head: None,
15688 verified_at: None,
15689 reviews: Vec::new(),
15690 e2e: Vec::new(),
15691 fix: None,
15692 blocking: 0,
15693 answered: 0,
15694 expected: 0,
15695 clean: true,
15696 verify_retried: false,
15697 e2e_deferred: false,
15698 e2e_defer_reason: None,
15699 progressed: false,
15700 vote_split: false,
15701 reconsideration: Vec::new(),
15702 verdict: None,
15703 }];
15704
15705 let mut runner = Runner {
15706 state,
15707 roles: ResolvedRoles {
15708 implementers: Vec::new(),
15709 judges: Vec::new(),
15710 reviewers: Vec::new(),
15711 fixer: None,
15712 conductor: conductor(),
15713 implementer_roster: Vec::new(),
15714 judge_roster: Vec::new(),
15715 reviewer_roster: Vec::new(),
15716 },
15717 sem: Arc::new(Semaphore::new(1)),
15718 pause: Pause::new(),
15719 interrupt: Pause::new(),
15720 };
15721
15722 let started = std::time::Instant::now();
15723 runner.gate().await.expect("gate");
15724 assert!(
15725 started.elapsed() < Duration::from_secs(1),
15726 "a gate with nothing to run must never wait on a lease it never needed"
15727 );
15728 assert!(
15729 runner.state.gate_ran,
15730 "zero commands is still a real, immediate attempt"
15731 );
15732 assert!(runner.state.gate.is_empty());
15733 assert_ne!(
15734 runner.state.status,
15735 RunStatus::Blocked,
15736 "must not read as resource-blocked on a lease it never asked for"
15737 );
15738 }
15739
15740 #[cfg(unix)]
15741 #[tokio::test]
15742 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15743 crate::run::pin_test_home();
15744 let tmp = tempfile::tempdir().unwrap();
15745 let mut config = Config::default();
15746 config.disk.min_free_bytes = 0;
15747 let mut state = RunState::new(
15748 tmp.path().to_owned(),
15749 "main".into(),
15750 "base".into(),
15751 "verification evidence".into(),
15752 config,
15753 );
15754 let (outcomes, pids) = run_commands(
15755 &mut state,
15756 "gate",
15757 "gate",
15758 1,
15759 &["sh".into(), "-c".into()],
15760 &[
15761 "printf before-timeout; exec sleep 30".into(),
15762 "kill -TERM $$".into(),
15763 "exit 7".into(),
15764 "printf success".into(),
15765 ],
15766 tmp.path(),
15767 Duration::from_secs(1),
15768 )
15769 .await;
15770 wait_for_timed_out_children_to_die(&pids).await;
15771 assert_eq!(
15772 pids.len(),
15773 1,
15774 "unleased timeout retains its death-check PID"
15775 );
15776 assert_eq!(outcomes.len(), 4);
15777 assert_eq!(outcomes[0].code, None);
15778 assert!(outcomes[0].output_tail.contains("before-timeout"));
15779 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15780 assert_eq!(outcomes[1].code, None);
15781 assert!(outcomes[1].output_tail.contains("command terminated:"));
15782 assert!(!outcomes[1].output_tail.contains("timed out"));
15783 assert_eq!(outcomes[2].code, Some(7));
15784 assert!(!outcomes[2].output_tail.contains("timed out"));
15785 assert_eq!(outcomes[3].code, Some(0));
15786 assert_eq!(outcomes[3].output_tail, "success");
15787 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15788 }
15789
15790 #[tokio::test]
15800 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15801 crate::run::pin_test_home();
15802
15803 let tmp = tempfile::tempdir().expect("tempdir");
15804 let repo = tmp.path().join("repo");
15805 std::fs::create_dir_all(&repo).unwrap();
15806 init_repo(&repo);
15807
15808 let mut config = Config::default();
15809 config.verify.gate = vec![
15810 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15811 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15812 .to_owned(),
15813 ];
15814
15815 let mut state = RunState::new(
15816 repo.clone(),
15817 "main".to_owned(),
15818 "deadbeef".to_owned(),
15819 "task".to_owned(),
15820 config,
15821 );
15822 let run_id = state.id.clone();
15823 state.candidates = vec![Candidate {
15824 index: 0,
15825 label: 'A',
15826 agent: "alpha".to_owned(),
15827 branch: "does-not-exist".to_owned(),
15828 worktree: repo.clone(),
15829 summary: String::new(),
15830 stat: String::new(),
15831 files: 0,
15832 commits: 0,
15833 empty: false,
15834 failed: None,
15835 verified_noop: None,
15836 duration_ms: 0,
15837 folded: false,
15838 }];
15839 state.tally = Some(Tally {
15840 first_choice: BTreeMap::from([('A', 1)]),
15841 borda: BTreeMap::new(),
15842 winner: 'A',
15843 rankings: 1,
15844 unanimous_initial: true,
15845 deliberated: false,
15846 changed_votes: 0,
15847 unanimous_final: true,
15848 tie_break: None,
15849 judges: 0,
15850 present: 0,
15851 quorum: 0,
15852 met_quorum: true,
15853 uncontested: Some("only candidate A produced a change".to_owned()),
15854 });
15855 state.reviews = vec![ReviewRound {
15856 round: 1,
15857 head: "deadbeef".to_owned(),
15858 verified_head: None,
15859 verified_at: None,
15860 reviews: Vec::new(),
15861 e2e: Vec::new(),
15862 fix: None,
15863 blocking: 0,
15864 answered: 0,
15865 expected: 0,
15866 clean: true,
15867 verify_retried: false,
15868 e2e_deferred: false,
15869 e2e_defer_reason: None,
15870 progressed: false,
15871 vote_split: false,
15872 reconsideration: Vec::new(),
15873 verdict: None,
15874 }];
15875
15876 let mut runner = Runner {
15877 state,
15878 roles: ResolvedRoles {
15879 implementers: Vec::new(),
15880 judges: Vec::new(),
15881 reviewers: Vec::new(),
15882 fixer: None,
15883 conductor: conductor(),
15884 implementer_roster: Vec::new(),
15885 judge_roster: Vec::new(),
15886 reviewer_roster: Vec::new(),
15887 },
15888 sem: Arc::new(Semaphore::new(1)),
15889 pause: Pause::new(),
15890 interrupt: Pause::new(),
15891 };
15892
15893 let started_marker = repo.join("started.marker");
15894 let release_marker = repo.join("release.marker");
15895 let poller = tokio::spawn(async move {
15896 for _ in 0..100 {
15901 if started_marker.exists()
15902 && let Ok(s) = crate::run::RunState::load(&run_id)
15903 && let Some(a) = s.active.get("gate")
15904 {
15905 std::fs::write(&release_marker, b"go").expect("release marker");
15906 return Some(a.clone());
15907 }
15908 tokio::time::sleep(Duration::from_millis(50)).await;
15909 }
15910 None
15911 });
15912
15913 runner.gate().await.expect("gate");
15914 let captured = poller.await.expect("poller task");
15915 let captured = captured.expect(
15916 "the poller never saw a `gate` task entry in run.json while the command was \
15917 still blocked on its own release marker",
15918 );
15919
15920 assert_eq!(captured.task.as_deref(), Some("gate"));
15921 assert_eq!(captured.node, "gate");
15922 assert_eq!(captured.index, Some(1));
15923 assert_eq!(captured.total, Some(1));
15924 assert!(
15925 captured
15926 .command
15927 .as_deref()
15928 .is_some_and(|c| c.contains("started.marker")),
15929 "{captured:?}"
15930 );
15931
15932 assert!(
15933 runner.state.active.is_empty(),
15934 "the entry must be cleared once the command actually finished: {:?}",
15935 runner.state.active
15936 );
15937 assert!(runner.state.gate_ran);
15938 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15939 }
15940
15941 #[tokio::test]
15944 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15945 use crate::verdict::{Finding, ReviewVote, Severity};
15946 crate::run::pin_test_home();
15947 let tmp = tempfile::tempdir().expect("tempdir");
15948 let repo = tmp.path().join("repo");
15949 std::fs::create_dir_all(&repo).unwrap();
15950 init_repo(&repo);
15951
15952 for (severity, vote, expect) in [
15953 (Severity::Major, ReviewVote::Reject, true),
15954 (Severity::Minor, ReviewVote::Reject, false),
15955 (Severity::Major, ReviewVote::Approve, false),
15956 ] {
15957 let mut round = review_round(false, 1, 1, 1, false, true);
15958 round.reviews = vec![ReviewRecord {
15959 reviewer: 1,
15960 agent: "alpha".to_owned(),
15961 summary: String::new(),
15962 findings: vec![Finding {
15963 id: "R1-1-1".to_owned(),
15964 severity,
15965 file: None,
15966 line: None,
15967 title: "t".to_owned(),
15968 detail: String::new(),
15969 }],
15970 vote: Some(vote),
15971 failed: None,
15972 duration_ms: 0,
15973 attempts: 0,
15974 }];
15975 let mut state = RunState::new(
15976 repo.clone(),
15977 "main".to_owned(),
15978 "deadbeef".to_owned(),
15979 "task".to_owned(),
15980 Config::default(),
15981 );
15982 state.reviews = vec![round];
15983 let mut runner = Runner {
15984 state,
15985 roles: ResolvedRoles {
15986 implementers: Vec::new(),
15987 judges: Vec::new(),
15988 reviewers: Vec::new(),
15989 fixer: None,
15990 conductor: conductor(),
15991 implementer_roster: Vec::new(),
15992 judge_roster: Vec::new(),
15993 reviewer_roster: Vec::new(),
15994 },
15995 sem: Arc::new(Semaphore::new(1)),
15996 pause: Pause::new(),
15997 interrupt: Pause::new(),
15998 };
15999 let shell = runner.state.config.shell();
16000 runner
16001 .stop_reviewing("round budget spent", &shell, &repo)
16002 .await
16003 .expect("stop_reviewing");
16004 assert_eq!(runner.state.status, RunStatus::Gating);
16005 assert_eq!(
16006 runner.state.contested_handoff.is_some(),
16007 expect,
16008 "{severity:?} + {vote:?}"
16009 );
16010 }
16011 }
16012
16013 #[tokio::test]
16026 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16027 crate::run::pin_test_home();
16028 let home = crate::run::home();
16029
16030 let tmp = tempfile::tempdir().expect("tempdir");
16031 let repo = tmp.path().join("repo");
16032 std::fs::create_dir_all(&repo).unwrap();
16033 init_repo(&repo);
16034 let head = crate::git::rev_parse(&repo, "HEAD")
16035 .await
16036 .expect("rev-parse");
16037 let cache_dir = tmp.path().join("target");
16040
16041 let mut config = Config::default();
16042 config.verify.e2e = vec![format!(
16043 "CARGO_TARGET_DIR='{}' test -f README.md",
16044 cache_dir.display()
16045 )];
16046 config.graph.review_rounds = 1;
16047 config.graph.timeout_verify = Some(2);
16050 config.graph.timeout_resource_wait = 2;
16051
16052 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16053 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16054 .expect("no io error acquiring directly")
16055 {
16056 crate::cache::AcquireOutcome::Acquired(g) => g,
16057 crate::cache::AcquireOutcome::Busy(b) => {
16058 panic!("expected the direct acquire to win the lease first: {b:?}")
16059 }
16060 };
16061
16062 let mut state = RunState::new(
16063 repo.clone(),
16064 "main".to_owned(),
16065 head.clone(),
16066 "task".to_owned(),
16067 config,
16068 );
16069 state.candidates = vec![Candidate {
16070 index: 0,
16071 label: 'A',
16072 agent: "alpha".to_owned(),
16073 branch: "does-not-exist".to_owned(),
16074 worktree: repo.clone(),
16075 summary: String::new(),
16076 stat: String::new(),
16077 files: 0,
16078 commits: 0,
16079 empty: false,
16080 failed: None,
16081 verified_noop: None,
16082 duration_ms: 0,
16083 folded: false,
16084 }];
16085 state.tally = Some(Tally {
16086 first_choice: BTreeMap::from([('A', 1)]),
16087 borda: BTreeMap::new(),
16088 winner: 'A',
16089 rankings: 1,
16090 unanimous_initial: true,
16091 deliberated: false,
16092 changed_votes: 0,
16093 unanimous_final: true,
16094 tie_break: None,
16095 judges: 0,
16096 present: 0,
16097 quorum: 0,
16098 met_quorum: true,
16099 uncontested: Some("only candidate A produced a change".to_owned()),
16100 });
16101 state.reviews = vec![ReviewRound {
16105 round: 1,
16106 head: head.clone(),
16107 verified_head: None,
16108 verified_at: None,
16109 reviews: Vec::new(),
16110 e2e: Vec::new(),
16111 fix: None,
16112 blocking: 1,
16113 answered: 1,
16114 expected: 1,
16115 clean: false,
16116 verify_retried: false,
16117 e2e_deferred: true,
16118 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16119 progressed: false,
16120 vote_split: false,
16121 reconsideration: Vec::new(),
16122 verdict: None,
16123 }];
16124
16125 let mut runner = Runner {
16126 state,
16127 roles: ResolvedRoles {
16128 implementers: Vec::new(),
16129 judges: Vec::new(),
16130 reviewers: Vec::new(),
16131 fixer: None,
16132 conductor: conductor(),
16133 implementer_roster: Vec::new(),
16134 judge_roster: Vec::new(),
16135 reviewer_roster: Vec::new(),
16136 },
16137 sem: Arc::new(Semaphore::new(1)),
16138 pause: Pause::new(),
16139 interrupt: Pause::new(),
16140 };
16141
16142 let shell = runner.state.config.shell();
16143 runner
16144 .stop_reviewing("round budget spent", &shell, &repo)
16145 .await
16146 .expect("stop_reviewing");
16147
16148 let last = runner.state.reviews.last().expect("round record");
16149 assert_eq!(
16150 last.e2e_status(),
16151 E2eStatus::ResourceBlocked,
16152 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16153 failed: {last:?}"
16154 );
16155 assert_eq!(
16156 last.verified_head.as_deref(),
16157 Some(head.as_str()),
16158 "which commit this attempt targeted is known even though nothing finished checking \
16159 it"
16160 );
16161 let first_attempt_at = last
16162 .verified_at
16163 .expect("when this attempt ran is known too");
16164 assert_ne!(
16165 runner.state.status,
16166 RunStatus::Blocked,
16167 "contention is evidence about the machine, not the patch — it must not settle the \
16168 run as blocked: {:?}",
16169 runner.state.status
16170 );
16171 assert!(
16172 !runner
16173 .state
16174 .events
16175 .iter()
16176 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16177 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16178 runner.state.events
16179 );
16180
16181 runner
16186 .stop_reviewing("round budget spent", &shell, &repo)
16187 .await
16188 .expect("stop_reviewing retry");
16189 assert_eq!(
16190 runner.state.reviews.len(),
16191 1,
16192 "no new round was started: {:?}",
16193 runner.state.reviews
16194 );
16195 let last = runner.state.reviews.last().expect("round record");
16196 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16197 assert!(
16198 last.verified_at.expect("still known") > first_attempt_at,
16199 "a second reentry must be a fresh attempt, not a stale copy of the first"
16200 );
16201 assert_ne!(runner.state.status, RunStatus::Blocked);
16202
16203 held.release();
16204 }
16205
16206 #[tokio::test]
16218 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16219 crate::run::pin_test_home();
16220 let home = crate::run::home();
16221
16222 let tmp = tempfile::tempdir().expect("tempdir");
16223 let repo = tmp.path().join("repo");
16224 std::fs::create_dir_all(&repo).unwrap();
16225 init_repo(&repo);
16226 let head = crate::git::rev_parse(&repo, "HEAD")
16227 .await
16228 .expect("rev-parse");
16229 let cache_dir = tmp.path().join("target");
16230
16231 let mut config = Config::default();
16232 config.verify.e2e = vec![format!(
16233 "CARGO_TARGET_DIR='{}' test -f README.md",
16234 cache_dir.display()
16235 )];
16236 config.graph.review_rounds = 1;
16237 config.graph.timeout_verify = Some(2);
16238 config.graph.timeout_resource_wait = 2;
16239
16240 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16241 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16242 .expect("no io error acquiring directly")
16243 {
16244 crate::cache::AcquireOutcome::Acquired(g) => g,
16245 crate::cache::AcquireOutcome::Busy(b) => {
16246 panic!("expected the direct acquire to win the lease first: {b:?}")
16247 }
16248 };
16249
16250 let mut state = RunState::new(
16251 repo.clone(),
16252 "main".to_owned(),
16253 head.clone(),
16254 "task".to_owned(),
16255 config,
16256 );
16257 state.candidates = vec![Candidate {
16258 index: 0,
16259 label: 'A',
16260 agent: "alpha".to_owned(),
16261 branch: "does-not-exist".to_owned(),
16262 worktree: repo.clone(),
16263 summary: String::new(),
16264 stat: String::new(),
16265 files: 0,
16266 commits: 0,
16267 empty: false,
16268 failed: None,
16269 verified_noop: None,
16270 duration_ms: 0,
16271 folded: false,
16272 }];
16273 state.tally = Some(Tally {
16274 first_choice: BTreeMap::from([('A', 1)]),
16275 borda: BTreeMap::new(),
16276 winner: 'A',
16277 rankings: 1,
16278 unanimous_initial: true,
16279 deliberated: false,
16280 changed_votes: 0,
16281 unanimous_final: true,
16282 tie_break: None,
16283 judges: 0,
16284 present: 0,
16285 quorum: 0,
16286 met_quorum: true,
16287 uncontested: Some("only candidate A produced a change".to_owned()),
16288 });
16289 state.reviews = vec![ReviewRound {
16293 round: 1,
16294 head: head.clone(),
16295 verified_head: Some(head.clone()),
16296 verified_at: Some(jiff::Timestamp::now()),
16297 reviews: Vec::new(),
16298 e2e: vec![CommandOutcome {
16299 command: format!(
16300 "CARGO_TARGET_DIR='{}' test -f README.md",
16301 cache_dir.display()
16302 ),
16303 code: None,
16304 output_tail: "waiting for the shared build cache".to_owned(),
16305 duration_ms: 0,
16306 resource_blocked: true,
16307 }],
16308 fix: None,
16309 blocking: 1,
16310 answered: 1,
16311 expected: 1,
16312 clean: false,
16313 verify_retried: false,
16314 e2e_deferred: false,
16315 e2e_defer_reason: None,
16316 progressed: false,
16317 vote_split: false,
16318 reconsideration: Vec::new(),
16319 verdict: None,
16320 }];
16321
16322 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16323 let mut runner = Runner {
16324 state,
16325 roles: ResolvedRoles {
16326 implementers: Vec::new(),
16327 judges: Vec::new(),
16328 reviewers: Vec::new(),
16329 fixer: None,
16330 conductor: conductor(),
16331 implementer_roster: Vec::new(),
16332 judge_roster: Vec::new(),
16333 reviewer_roster: Vec::new(),
16334 },
16335 sem: Arc::new(Semaphore::new(1)),
16336 pause: Pause::new(),
16337 interrupt: Pause::new(),
16338 };
16339
16340 runner.review_loop().await.expect("review_loop");
16345
16346 assert_eq!(
16347 runner.state.reviews.len(),
16348 1,
16349 "no new round was started on top of the unresolved one: {:?}",
16350 runner.state.reviews
16351 );
16352 let last = &runner.state.reviews[0];
16353 assert_eq!(
16354 last.e2e_status(),
16355 E2eStatus::ResourceBlocked,
16356 "still contended: {last:?}"
16357 );
16358 assert!(
16359 last.verified_at.expect("still known") > first_attempt_at,
16360 "review_loop must have actually retried the check, not left it exactly as found"
16361 );
16362 assert_ne!(
16363 runner.state.status,
16364 RunStatus::Blocked,
16365 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16366 runner.state.status
16367 );
16368
16369 held.release();
16370 }
16371
16372 #[tokio::test]
16373 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16374 crate::run::pin_test_home();
16375 let tmp = tempfile::tempdir().expect("tempdir");
16376 let repo = tmp.path().join("repo");
16377 std::fs::create_dir_all(&repo).unwrap();
16378 init_repo(&repo);
16379
16380 let mut config = Config::default();
16381 config.merge.mode = MergeMode::Pr;
16382 config.graph.land = true;
16383 config.graph.land_approval = false;
16384
16385 let mut state = RunState::new(
16386 repo.clone(),
16387 "main".to_owned(),
16388 "deadbeef".to_owned(),
16389 "task".to_owned(),
16390 config,
16391 );
16392 state.candidates = vec![Candidate {
16393 index: 0,
16394 label: 'A',
16395 agent: "alpha".to_owned(),
16396 branch: "does-not-exist".to_owned(),
16397 worktree: repo.clone(),
16398 summary: String::new(),
16399 stat: String::new(),
16400 files: 0,
16401 commits: 0,
16402 empty: false,
16403 failed: None,
16404 verified_noop: None,
16405 duration_ms: 0,
16406 folded: false,
16407 }];
16408 state.tally = Some(Tally {
16409 first_choice: BTreeMap::from([('A', 1)]),
16410 borda: BTreeMap::new(),
16411 winner: 'A',
16412 rankings: 1,
16413 unanimous_initial: true,
16414 deliberated: false,
16415 changed_votes: 0,
16416 unanimous_final: true,
16417 tie_break: None,
16418 judges: 0,
16419 present: 0,
16420 quorum: 0,
16421 met_quorum: true,
16422 uncontested: Some("only candidate A produced a change".to_owned()),
16423 });
16424 state.reviews = vec![ReviewRound {
16425 round: 1,
16426 head: "deadbeef".to_owned(),
16427 verified_head: None,
16428 verified_at: None,
16429 reviews: Vec::new(),
16430 e2e: Vec::new(),
16431 fix: None,
16432 blocking: 0,
16433 answered: 0,
16434 expected: 0,
16435 clean: true,
16436 verify_retried: false,
16437 e2e_deferred: false,
16438 e2e_defer_reason: None,
16439 progressed: false,
16440 vote_split: false,
16441 reconsideration: Vec::new(),
16442 verdict: None,
16443 }];
16444 state.gate = vec![CommandOutcome {
16445 command: "test".to_owned(),
16446 code: Some(0),
16447 output_tail: String::new(),
16448 duration_ms: 0,
16449 resource_blocked: false,
16450 }];
16451 state.gate_ran = true;
16452 state.status = RunStatus::Landing;
16456 state.merge = Some(MergeOutcome {
16457 mode: MergeMode::Pr,
16458 ok: true,
16459 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16460 empty: false,
16461 });
16462
16463 ask_test_home();
16467 state.save().expect("save landing run");
16470 let store = ask::Questions::open();
16471 let q = ask_open_question(&store, &state.id);
16472
16473 let mut runner = Runner {
16474 state,
16475 roles: ResolvedRoles {
16476 implementers: Vec::new(),
16477 judges: Vec::new(),
16478 reviewers: Vec::new(),
16479 fixer: None,
16480 conductor: conductor(),
16481 implementer_roster: Vec::new(),
16482 judge_roster: Vec::new(),
16483 reviewer_roster: Vec::new(),
16484 },
16485 sem: Arc::new(Semaphore::new(1)),
16486 pause: Pause::new(),
16487 interrupt: Pause::new(),
16488 };
16489
16490 runner.execute().await.expect("execute");
16495
16496 assert_eq!(
16497 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16498 Some("https://example.invalid/x/y/pull/1"),
16499 "reentry must not push again or open a second pull request over the \
16500 one `land` is already watching"
16501 );
16502 assert_ne!(
16503 runner.state.status,
16504 RunStatus::Landing,
16505 "land could not actually reach the fake pull request, so it must \
16506 have given up rather than left the run silently parked forever"
16507 );
16508 assert_eq!(runner.state.status, RunStatus::Blocked);
16512 assert!(
16513 store.get(&q.id).unwrap().status.open(),
16514 "Blocked is still alive; settle_questions must have been a no-op here"
16515 );
16516 }
16517
16518 fn state_with_round(round: ReviewRound) -> RunState {
16519 let mut s = RunState::new(
16520 PathBuf::from("/repo"),
16521 "main".to_owned(),
16522 "abc1234".to_owned(),
16523 "add retries".to_owned(),
16524 Config::default(),
16525 );
16526 s.reviews = vec![round];
16527 s
16528 }
16529
16530 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16531 crate::verdict::Finding {
16532 id: id.to_owned(),
16533 severity,
16534 file: None,
16535 line: None,
16536 title: title.to_owned(),
16537 detail: String::new(),
16538 }
16539 }
16540
16541 #[test]
16542 fn pr_body_names_open_findings_and_declined_ones() {
16543 let round = ReviewRound {
16544 round: 2,
16545 head: "deadbee".to_owned(),
16546 verified_head: None,
16547 verified_at: None,
16548 reviews: vec![ReviewRecord {
16549 attempts: 0,
16550 reviewer: 1,
16551 agent: "alpha".to_owned(),
16552 summary: String::new(),
16553 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16554 vote: None,
16555 failed: None,
16556 duration_ms: 0,
16557 }],
16558 e2e: vec![CommandOutcome {
16559 command: "cargo test".to_owned(),
16560 code: Some(0),
16561 output_tail: String::new(),
16562 duration_ms: 0,
16563 resource_blocked: false,
16564 }],
16565 verify_retried: false,
16566 e2e_deferred: false,
16567 e2e_defer_reason: None,
16568 fix: Some(FixRecord {
16569 agent: "alpha".to_owned(),
16570 addressed: Vec::new(),
16571 rejected: vec![crate::verdict::Rejection {
16572 id: "R1-1-1".to_owned(),
16573 why: "not reachable from any caller".to_owned(),
16574 }],
16575 notes: String::new(),
16576 committed: true,
16577 failed: None,
16578 duration_ms: 0,
16579 continuation: None,
16580 }),
16581 blocking: 0,
16582 answered: 1,
16583 expected: 1,
16584 clean: false,
16585 progressed: true,
16586 vote_split: false,
16587 reconsideration: Vec::new(),
16588 verdict: None,
16589 };
16590 let state = state_with_round(round);
16591 let body = pr_message(&state, 'A').body;
16592
16593 assert!(body.contains("add retries"), "the task must still be there");
16594 assert!(body.contains("R2-1-1"), "{body}");
16595 assert!(!body.contains("unused import"), "{body}");
16596 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16597 assert!(
16598 !body.contains("not reachable from any caller"),
16599 "the reason it was declined: {body}"
16600 );
16601 }
16602
16603 #[test]
16604 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16605 let round = ReviewRound {
16606 round: 1,
16607 head: "deadbee".to_owned(),
16608 verified_head: None,
16609 verified_at: None,
16610 reviews: vec![ReviewRecord {
16611 attempts: 0,
16612 reviewer: 1,
16613 agent: "alpha".to_owned(),
16614 summary: String::new(),
16615 findings: Vec::new(),
16616 vote: None,
16617 failed: None,
16618 duration_ms: 0,
16619 }],
16620 e2e: Vec::new(),
16621 verify_retried: false,
16622 e2e_deferred: false,
16623 e2e_defer_reason: None,
16624 fix: None,
16625 blocking: 0,
16626 answered: 1,
16627 expected: 1,
16628 clean: true,
16629 progressed: false,
16630 vote_split: false,
16631 reconsideration: Vec::new(),
16632 verdict: None,
16633 };
16634 let state = state_with_round(round);
16635 let body = pr_message(&state, 'A').body;
16636 assert!(!body.contains("Open review findings"), "{body}");
16637 assert!(!body.contains("Declined"), "{body}");
16638 }
16639
16640 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16641 let mut state = RunState::new(
16642 PathBuf::from("/repo"),
16643 "main".to_owned(),
16644 "abc1234".to_owned(),
16645 instruction.to_owned(),
16646 Config::default(),
16647 );
16648 state.candidates.push(Candidate {
16649 index: 0,
16650 label: 'A',
16651 agent: "alpha".to_owned(),
16652 branch: "magi/x/A".to_owned(),
16653 worktree: PathBuf::from("/wt"),
16654 summary: summary.to_owned(),
16655 stat: String::new(),
16656 files: 1,
16657 commits: 1,
16658 empty: false,
16659 failed: None,
16660 verified_noop: None,
16661 folded: false,
16662 duration_ms: 0,
16663 });
16664 state
16665 }
16666
16667 fn review_state(subjects: &[&str]) -> RunState {
16668 let mut state = state_with_summary(
16669 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16670 "",
16671 );
16672 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16673 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16674 state
16675 }
16676
16677 #[tokio::test]
16680 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16681 ask_test_home();
16682 let tmp = tempfile::tempdir().unwrap();
16683 let repo = tmp.path().join("repo");
16684 std::fs::create_dir_all(&repo).unwrap();
16685 init_repo(&repo);
16686 let origin = tmp.path().join("origin.git");
16687 let g = |dir: &Path, args: &[&str]| {
16688 let out = std::process::Command::new("git")
16689 .args(args)
16690 .current_dir(dir)
16691 .quiet()
16692 .output()
16693 .expect("spawn git");
16694 assert!(
16695 out.status.success(),
16696 "git {args:?}: {}",
16697 String::from_utf8_lossy(&out.stderr)
16698 );
16699 };
16700 g(
16701 tmp.path(),
16702 &[
16703 "clone",
16704 "--bare",
16705 "-q",
16706 repo.to_str().unwrap(),
16707 origin.to_str().unwrap(),
16708 ],
16709 );
16710 g(
16711 &repo,
16712 &["remote", "add", "origin", origin.to_str().unwrap()],
16713 );
16714 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16715
16716 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16718 g(&repo, &["add", "-A"]);
16719 g(
16720 &repo,
16721 &["commit", "-q", "-m", "chore(deps): update a crate"],
16722 );
16723 g(&repo, &["push", "-q", "origin", "main"]);
16724 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16725 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16726 g(&repo, &["add", "-A"]);
16727 g(
16728 &repo,
16729 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16730 );
16731 g(&repo, &["checkout", "-q", "main"]);
16732
16733 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16734 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16735 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16737 assert_eq!(fallback, c1);
16738
16739 let mut state = review_state(&[
16740 "chore(deps): update a crate",
16741 "fix(daemon): apply a chosen action",
16742 ]);
16743 state.repo = repo.clone();
16744 state.base_branch = "main".to_owned();
16745 state.base_commit = c1;
16746 refresh_reviewed_commits(&mut state, "feat/own").await;
16747 assert_eq!(
16748 state.reviewed_commits,
16749 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16750 );
16751 assert_eq!(
16752 review_title(&state).as_deref(),
16753 Some("fix(daemon): apply a chosen action")
16754 );
16755 assert_eq!(
16756 leaked_subjects(&state, "feat/own").await,
16757 Some(vec!["chore(deps): update a crate".to_owned()])
16758 );
16759 state.config.merge.remote = "nowhere".to_owned();
16762 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16763 }
16764
16765 struct SummaryFixture {
16769 _tmp: tempfile::TempDir,
16770 repo: PathBuf,
16771 reply: PathBuf,
16772 runner: Runner,
16773 }
16774
16775 fn summary_fixture() -> SummaryFixture {
16776 ask_test_home();
16777 let tmp = tempfile::tempdir().unwrap();
16778 let repo = tmp.path().join("repo");
16779 std::fs::create_dir_all(&repo).unwrap();
16780 init_repo(&repo);
16781 let origin = tmp.path().join("origin.git");
16782 let g = |args: &[&str]| {
16783 let out = std::process::Command::new("git")
16784 .args(args)
16785 .current_dir(&repo)
16786 .quiet()
16787 .output()
16788 .expect("spawn git");
16789 assert!(
16790 out.status.success(),
16791 "git {args:?}: {}",
16792 String::from_utf8_lossy(&out.stderr)
16793 );
16794 };
16795 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16796 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16797 g(&["push", "-q", "origin", "main"]);
16798 g(&["checkout", "-q", "-b", "feat/own"]);
16799 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16800 g(&["add", "-A"]);
16801 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16802 let reply = tmp.path().join("reply.txt");
16803 std::fs::write(
16804 &reply,
16805 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16806 )
16807 .unwrap();
16808 let mut runner = runner_at(RunStatus::Gating);
16809 let mut state = review_state(&["feat(own): add the own file"]);
16810 state.repo = repo.clone();
16811 state.base_branch = "main".to_owned();
16812 state.candidates[0].branch = "feat/own".to_owned();
16813 state.candidates[0].worktree = repo.clone();
16814 let mut author = spec("alpha");
16815 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16816 author
16817 .env
16818 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16819 state.config.agents = vec![author];
16820 runner.state = state;
16821 SummaryFixture {
16822 _tmp: tmp,
16823 repo,
16824 reply,
16825 runner,
16826 }
16827 }
16828
16829 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16830 let winner = f.runner.state.candidates[0].clone();
16831 f.runner
16832 .guarded_pr_message(&winner, None, true)
16833 .await
16834 .unwrap()
16835 .expect("an English description never parks")
16836 }
16837
16838 fn clean_round_with_minor_findings() -> ReviewRound {
16839 let mut round = passing_round(2, None);
16840 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16841 round.reviews = vec![ReviewRecord {
16842 attempts: 0,
16843 reviewer: 1,
16844 agent: "alpha".into(),
16845 summary: "内部レビュー".into(),
16846 findings: vec![
16847 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16848 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16849 ],
16850 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16851 failed: None,
16852 duration_ms: 0,
16853 }];
16854 round
16855 }
16856
16857 #[tokio::test]
16858 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16859 for review_only in [false, true] {
16860 let mut f = summary_fixture();
16861 if !review_only {
16862 f.runner.state.candidates[0].agent = "alpha".into();
16863 }
16864 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16865 assert!(f.runner.state.reviews[0].clean);
16866 assert!(f.runner.state.reviews[0].fix.is_none());
16867 assert!(f.runner.state.open_findings().is_empty());
16868 assert_eq!(
16869 crate::pr_outcome::required_finding_ids(&f.runner.state),
16870 ["R2-1-1", "R2-1-2"]
16871 );
16872 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16873 let evidence = crate::pr_outcome::summary_evidence(
16874 &f.runner.state,
16875 crate::pr_outcome::FinalHead::At(&head),
16876 );
16877 assert!(
16878 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16879 );
16880 let omitted = posted(&mut f).await;
16882 assert!(omitted.body.contains("No description"));
16883 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16884 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16885 assert!(f.runner.state.change_summary.is_none());
16886 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16887 let described = posted(&mut f).await;
16888 assert!(described.body.contains("improve documentation for callers"));
16889 assert!(described.body.contains("remaining edge case"));
16890 assert!(described.body.contains("## Open review findings"));
16891 assert!(!described.body.contains("未修正"));
16892 }
16893 }
16894
16895 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16896 let mut f = summary_fixture();
16897 if !review_only {
16898 f.runner.state.candidates[0].agent = "alpha".into();
16899 }
16900 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16901 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16902 fixed
16903 .fix
16904 .as_mut()
16905 .unwrap()
16906 .rejected
16907 .push(crate::verdict::Rejection {
16908 id: "R1-1-1".into(),
16909 why: "呼び出されないコードのため辞退".into(),
16910 });
16911 let mut clean = clean_round_with_minor_findings();
16912 clean.verified_head = Some(head.clone());
16913 f.runner.state.reviews = vec![fixed, clean];
16914 ok_gate(&mut f.runner.state, Some(&head));
16915 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16916 f.runner.state.config.agents.push(scripted("beta", reply));
16917 f.runner.state.config.roles.language_judge =
16918 Some(crate::config::AgentChoice::One("beta".into()));
16919 f
16920 }
16921
16922 #[tokio::test]
16923 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16924 for review_only in [false, true] {
16925 let mut f = rewrite_fixture_with_findings(review_only,
16926 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16927 let m = posted(&mut f).await;
16928 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16929 let expected =
16930 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16931 assert!(m.body.contains(&expected));
16932 assert!(m.body.contains("## Review rounds"));
16933 assert!(
16934 m.body
16935 .contains("Gate (`verify.gate`): passed on the final commit")
16936 );
16937 assert!(
16938 m.body
16939 .contains("Tests (`verify.e2e`): passed on the final commit")
16940 );
16941 let c = f.runner.state.change_summary.as_ref().unwrap();
16942 assert!(
16943 m.body
16944 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16945 );
16946 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16947 assert!(crate::pr_outcome::covers_required_findings(
16948 &f.runner.state,
16949 &m.body
16950 ));
16951 assert!(m.body.contains("documentation remains incomplete"));
16952 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16953 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16954 assert!(f.runner.state.github_text.is_none());
16955 }
16956 }
16957
16958 #[tokio::test]
16959 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16960 for review_only in [false, true] {
16961 for reply in [
16962 "TITLE: fix: public description\nA short English description.",
16963 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16964 ] {
16965 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16966 let winner = f.runner.state.candidates[0].clone();
16967 let posted = f
16968 .runner
16969 .guarded_pr_message(&winner, None, true)
16970 .await
16971 .unwrap();
16972 assert!(posted.is_none());
16973 assert!(f.runner.state.parked);
16974 let withheld = f.runner.state.github_text.as_ref().unwrap();
16975 assert!(
16976 withheld.rewrite.is_none(),
16977 "an incomplete rewrite must not be selectable"
16978 );
16979 let q = ask::Questions::open()
16980 .get(withheld.question.as_ref().unwrap())
16981 .unwrap();
16982 assert!(
16983 !q.choices
16984 .contains(&crate::github_text::USE_REWRITE.to_owned())
16985 );
16986 assert!(
16987 q.detail.contains("R1-1-1")
16988 && q.detail.contains("R2-1-1")
16989 && q.detail.contains("R2-1-2")
16990 );
16991 }
16992 }
16993 }
16994
16995 #[tokio::test]
16996 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16997 for review_only in [false, true] {
16998 let mut f = summary_fixture();
16999 if !review_only {
17000 f.runner.state.candidates[0].agent = "alpha".into();
17001 }
17002 f.runner.state.instruction = "内部タスクの説明".into();
17003 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17004 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17005 f.runner.state.reviews = vec![
17006 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17007 round_with_notes(2, Some("二回目の内部修正報告です。")),
17008 passing_round(3, Some(&head)),
17009 ];
17010 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17011 ok_gate(&mut f.runner.state, Some(&head));
17012 let captured = f.repo.join("public-prompt.txt");
17013 let author = &mut f.runner.state.config.agents[0];
17014 author.command = vec![
17015 "sh".into(),
17016 "-c".into(),
17017 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17018 ];
17019 author.env.insert(
17020 "PROMPT_FILE".into(),
17021 captured.to_string_lossy().into_owned(),
17022 );
17023 let m = posted(&mut f).await;
17024 assert!(m.body.contains("Adds `own.txt`"));
17025 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17026 assert!(m.body.contains("R1-1-1"));
17027 assert!(
17028 m.body
17029 .contains("Gate (`verify.gate`): passed on the final commit")
17030 );
17031 assert!(
17032 m.body
17033 .contains("Tests (`verify.e2e`): passed on the final commit")
17034 );
17035 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17036 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17037 let prompt = std::fs::read_to_string(captured).unwrap();
17038 assert!(prompt.contains(&format!(
17039 "git diff {} {head}",
17040 f.runner.state.change_summary.as_ref().unwrap().base
17041 )));
17042 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17043 assert_eq!(
17044 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17045 "二回目の内部修正報告です。"
17046 );
17047 }
17048 }
17049
17050 #[tokio::test]
17051 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17052 for review_only in [false, true] {
17053 let mut f = summary_fixture();
17054 if !review_only {
17055 f.runner.state.candidates[0].agent = "alpha".into();
17056 }
17057 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17058 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17059 std::fs::write(&f.reply, "no usable description").unwrap();
17060 let failed = posted(&mut f).await;
17061 assert!(
17062 failed
17063 .body
17064 .contains("No description of what this change does could be generated")
17065 );
17066 assert!(failed.body.is_ascii());
17067 assert!(f.runner.state.github_text.is_none());
17068 assert!(f.runner.state.change_summary.is_none());
17069 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17070 let evidence = crate::pr_outcome::summary_evidence(
17071 &f.runner.state,
17072 crate::pr_outcome::FinalHead::At(&head),
17073 );
17074 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17075 assert!(
17076 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17077 .exists()
17078 );
17079 f.runner.state =
17080 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17081 std::fs::write(
17082 &f.reply,
17083 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17084 )
17085 .unwrap();
17086 let retried = posted(&mut f).await;
17087 assert!(retried.body.contains("Adds the committed own file."));
17088 assert_eq!(
17089 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17090 "日本語の修正報告"
17091 );
17092 }
17093 }
17094
17095 #[tokio::test]
17096 async fn same_head_verification_and_findings_invalidate_public_prose() {
17097 let mut f = summary_fixture();
17098 posted(&mut f).await;
17099 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17100 let original = f
17101 .runner
17102 .state
17103 .change_summary
17104 .as_ref()
17105 .unwrap()
17106 .evidence
17107 .clone();
17108 ok_gate(&mut f.runner.state, Some(&head));
17109 std::fs::write(
17110 &f.reply,
17111 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17112 )
17113 .unwrap();
17114 let verified = posted(&mut f).await;
17115 assert!(verified.body.contains("now verified"));
17116 assert_ne!(
17117 f.runner.state.change_summary.as_ref().unwrap().evidence,
17118 original
17119 );
17120 let mut round = round_with_notes(1, Some("内部報告"));
17121 round.reviews.push(ReviewRecord {
17122 attempts: 0,
17123 reviewer: 1,
17124 agent: "alpha".into(),
17125 summary: "内部レビュー".into(),
17126 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17127 vote: None,
17128 failed: None,
17129 duration_ms: 0,
17130 });
17131 round
17132 .fix
17133 .as_mut()
17134 .unwrap()
17135 .rejected
17136 .push(crate::verdict::Rejection {
17137 id: "R1-1-1".into(),
17138 why: "呼び出し元から到達できない".into(),
17139 });
17140 f.runner.state.reviews = vec![round];
17141 let omitted = posted(&mut f).await;
17143 assert!(omitted.body.contains("No description"));
17144 assert!(omitted.body.contains("R1-1-1"));
17145 assert!(!omitted.body.contains("到達"));
17146 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17147 let described = posted(&mut f).await;
17148 assert!(described.body.contains("unreachable from callers"));
17149 assert!(described.body.contains("R1-1-1"));
17150 assert!(!described.body.contains("到達"));
17151 let evidence = f
17152 .runner
17153 .state
17154 .change_summary
17155 .as_ref()
17156 .unwrap()
17157 .evidence
17158 .clone();
17159 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17160 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17161 assert!(posted(&mut f).await.body.contains("rationale updated"));
17162 assert_ne!(
17163 f.runner.state.change_summary.as_ref().unwrap().evidence,
17164 evidence
17165 );
17166 f.runner
17168 .state
17169 .change_summary
17170 .as_mut()
17171 .unwrap()
17172 .evidence
17173 .clear();
17174 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17175 assert!(posted(&mut f).await.body.contains("current specification"));
17176 }
17177
17178 #[tokio::test]
17179 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17180 let mut f = summary_fixture();
17181 let m = posted(&mut f).await;
17182 assert_eq!(m.title, "feat(own): add the own file");
17183 assert!(
17184 m.body
17185 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17186 "{}",
17187 m.body
17188 );
17189 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17190 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17191 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17192 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17193 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17194 let kept = f.runner.state.change_summary.as_ref().unwrap();
17195 assert_eq!(
17196 (kept.head.as_str(), kept.base.as_str()),
17197 (head.as_str(), base.as_str())
17198 );
17199 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17200 }
17201
17202 #[tokio::test]
17203 async fn review_only_summary_receives_the_configured_github_language() {
17204 let mut f = summary_fixture();
17205 f.runner.state.config.graph.github_language = "ja".into();
17206 let captured = f.repo.join("summary-prompt.txt");
17207 let author = &mut f.runner.state.config.agents[0];
17208 author.command = vec![
17209 "sh".into(),
17210 "-c".into(),
17211 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17212 ];
17213 author.env.insert(
17214 "PROMPT_FILE".into(),
17215 captured.to_string_lossy().into_owned(),
17216 );
17217 std::fs::write(
17218 &f.reply,
17219 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17220 )
17221 .unwrap();
17222 let m = posted(&mut f).await;
17223 let prompt = std::fs::read_to_string(captured).unwrap();
17224 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17225 assert!(!prompt.contains("in English"));
17226 assert_eq!(m.title, "feat(own): ファイルを追加");
17227 assert!(m.body.contains("own.txt を追加します。"));
17228 }
17229
17230 #[tokio::test]
17231 async fn review_fixes_come_after_the_whole_change_summary() {
17232 let mut f = summary_fixture();
17233 f.runner.state.reviews = vec![
17234 round_with_notes(1, Some("handled the PATH case")),
17235 round_with_notes(2, None),
17236 ];
17237 let m = posted(&mut f).await;
17238 let summary = m.body.find("## Summary").expect("summary");
17239 let fixes = m.body.find("## Review rounds").expect("review rounds");
17240 assert!(summary < fixes, "{}", m.body);
17241 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17242 assert!(m.body.contains("## Final verification"), "{}", m.body);
17243 }
17244
17245 #[tokio::test]
17246 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17247 let mut f = summary_fixture();
17248 let first = posted(&mut f).await;
17249 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17250 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17252 let again = posted(&mut f).await;
17253 assert_eq!((again.title, again.body), (first.title, first.body));
17254
17255 let g = |args: &[&str]| {
17256 let out = std::process::Command::new("git")
17257 .args(args)
17258 .current_dir(&f.repo)
17259 .quiet()
17260 .output()
17261 .expect("spawn git");
17262 assert!(out.status.success(), "git {args:?}");
17263 };
17264 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17265 g(&["add", "-A"]);
17266 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17267 let second = posted(&mut f).await;
17268 let kept = f.runner.state.change_summary.clone().unwrap();
17269 assert_ne!(kept.head, first_head);
17270 assert!(second.body.contains("Not read yet."), "{}", second.body);
17271 assert!(!second.body.contains("so the first summary is read"));
17272 assert_eq!(second.title, "feat(own): other");
17273 }
17274
17275 #[tokio::test]
17276 async fn an_unavailable_summary_is_stated_and_never_invented() {
17277 let mut f = summary_fixture();
17278 std::fs::write(&f.reply, "no title line here").unwrap();
17279 let m = posted(&mut f).await;
17280 assert!(
17281 m.body
17282 .contains("No description of what this change does could be generated"),
17283 "{}",
17284 m.body
17285 );
17286 assert!(m.body.contains("internal reports remain in local records"));
17287 assert!(f.runner.state.change_summary.is_none());
17288 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17289 let winner = f.runner.state.candidates[0].clone();
17291 let preview = f
17292 .runner
17293 .guarded_pr_message(&winner, None, false)
17294 .await
17295 .unwrap()
17296 .unwrap();
17297 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17298 }
17299
17300 #[test]
17301 fn pr_message_review_single_commit_uses_its_subject() {
17302 let state = review_state(&["feat(nats): per-role user"]);
17303 let m = pr_message(&state, 'A');
17304 assert_eq!(m.title, "feat(nats): per-role user");
17305 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17306 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17307 }
17308
17309 #[test]
17310 fn pr_message_review_multi_commit_takes_the_oldest() {
17311 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17312 let m = pr_message(&state, 'A');
17313 assert_eq!(m.title, "feat: the change");
17314 for s in ["feat: the change", "fix: typo", "fix: again"] {
17315 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17316 }
17317 }
17318
17319 #[test]
17320 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17321 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17322 let mut state = review_state(&[first, "fix: later fixup"]);
17323 state.candidates[0].branch = "機能/ログイン".to_owned();
17324 let m = pr_message(&state, 'A');
17325 assert!(
17326 m.title.starts_with("chore: land candidate A of run"),
17327 "{}",
17328 m.title
17329 );
17330 }
17331 }
17332
17333 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17334 BranchFacts {
17335 commits: commits
17336 .iter()
17337 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17338 .collect(),
17339 stat: stat.to_owned(),
17340 }
17341 }
17342
17343 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17344 let mut r = review_round(true, 0, 1, 1, true, true);
17345 r.round = round;
17346 r.fix = notes.map(|n| FixRecord {
17347 agent: "fixer".to_owned(),
17348 addressed: Vec::new(),
17349 rejected: Vec::new(),
17350 notes: n.to_owned(),
17351 committed: true,
17352 failed: None,
17353 duration_ms: 0,
17354 continuation: None,
17355 });
17356 r
17357 }
17358
17359 #[test]
17360 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17361 let state = review_state(&["ignored"]);
17362 let f = facts(
17363 &[
17364 (
17365 "fix(login): resolve PATH on macOS",
17366 "Login shells skip rc files.",
17367 ),
17368 ("fix: address review", ""),
17369 ],
17370 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17371 );
17372 let m = pr_message_with(&state, 'A', Some(&f));
17373 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17374 assert!(
17375 m.body
17376 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17377 "{}",
17378 m.body
17379 );
17380 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17381 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17382 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17383 for banned in [
17384 "Review the work already",
17385 "no task statement",
17386 "Original task",
17387 ] {
17388 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17389 }
17390 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17391 }
17392
17393 #[test]
17394 fn pr_message_review_truncates_a_huge_first_commit_body() {
17395 let state = review_state(&["ignored"]);
17396 let f = facts(
17397 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17398 "s",
17399 );
17400 let m = pr_message_with(&state, 'A', Some(&f));
17401 assert!(m.body.len() < 30_000, "{}", m.body.len());
17402 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17403 assert!(
17404 m.body.contains("- ... 1 more commit(s)"),
17405 "{}",
17406 m.body.len()
17407 );
17408 assert!(m.body.ends_with("magi:candidate-a\n"));
17409 }
17410
17411 #[test]
17412 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17413 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17414 assert_eq!(m.title, "feat: x");
17415 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17416 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17417 assert!(!m.body.contains("no task statement"), "{}", m.body);
17418 }
17419
17420 #[test]
17421 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17422 let mut state = review_state(&["日本語の件名"]);
17423 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17424 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17425 state.candidates[0].branch = "機能/ログイン".to_owned();
17426 assert!(
17427 pr_message(&state, 'A')
17428 .title
17429 .starts_with("chore: land candidate A")
17430 );
17431 }
17432
17433 #[test]
17434 fn pr_message_review_fixes_survive_a_clean_final_round() {
17435 let mut state = review_state(&["feat: x"]);
17436 state.reviews = vec![
17437 round_with_notes(1, Some("handled the PATH case")),
17438 round_with_notes(2, None),
17439 ];
17440 let body = pr_message(&state, 'A').body;
17441 assert!(
17442 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17443 "{body}"
17444 );
17445 assert!(body.contains("### Round 1"), "{body}");
17446
17447 state.reviews = vec![
17448 round_with_notes(1, Some("first fix")),
17449 round_with_notes(2, Some("")),
17450 round_with_notes(3, Some("second fix")),
17451 round_with_notes(4, None),
17452 ];
17453 let body = pr_message(&state, 'A').body;
17454 assert!(
17455 body.contains("### Round 1") && !body.contains("first fix"),
17456 "{body}"
17457 );
17458 assert!(
17459 body.contains("### Round 3") && !body.contains("second fix"),
17460 "{body}"
17461 );
17462 assert!(body.contains("### Round 2"), "{body}");
17463 }
17464
17465 const FINAL: &str = "abcdef1234567890";
17466
17467 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17468 let mut r = round_with_notes(round, None);
17469 r.verified_head = verified.map(str::to_owned);
17470 r
17471 }
17472
17473 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17474 state.gate = vec![CommandOutcome {
17475 command: "gate".to_owned(),
17476 code: Some(0),
17477 output_tail: String::new(),
17478 duration_ms: 0,
17479 resource_blocked: false,
17480 }];
17481 state.gate_ran = true;
17482 state.gate_head = head.map(str::to_owned);
17483 }
17484
17485 fn body_at(state: &RunState) -> String {
17486 pr_message_raw(
17487 state,
17488 'A',
17489 None,
17490 crate::pr_outcome::FinalHead::At(FINAL),
17491 ChangeNote::NotAsked,
17492 )
17493 .body
17494 }
17495
17496 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17497
17498 #[test]
17499 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17500 let mut state = review_state(&["feat: x"]);
17501 state.reviews = vec![
17502 round_with_notes(1, Some(SANDBOXED)),
17503 passing_round(2, Some(FINAL)),
17504 ];
17505 ok_gate(&mut state, Some(FINAL));
17506 let body = body_at(&state);
17507 assert!(!body.contains("Handled the PATH case."), "{body}");
17508 assert!(!body.contains("could not commit"), "{body}");
17509 assert!(!body.contains("could not run"), "{body}");
17510 assert!(
17511 body.contains("The fix is committed on the branch"),
17512 "{body}"
17513 );
17514 assert!(!body.contains("sandbox"), "{body}");
17515 assert!(
17516 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17517 "{body}"
17518 );
17519 assert!(
17520 body.contains("Gate (`verify.gate`): passed on the final commit"),
17521 "{body}"
17522 );
17523 assert!(
17525 state.reviews[0]
17526 .fix
17527 .as_ref()
17528 .unwrap()
17529 .notes
17530 .contains("could not commit")
17531 );
17532 }
17533
17534 #[test]
17535 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17536 let mut state = review_state(&["feat: x"]);
17537 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17538 r1.fix.as_mut().unwrap().committed = false;
17539 let mut r2 = passing_round(2, Some(FINAL));
17540 r2.e2e[0].code = Some(1);
17541 state.reviews = vec![r1, r2];
17542 let body = body_at(&state);
17543 assert!(!body.contains("could not commit"), "{body}");
17544 assert!(!body.contains("could not run"), "{body}");
17545 assert!(body.contains("This round produced no commit"), "{body}");
17546 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17547 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17548 assert!(!body.contains("passed"), "{body}");
17549 }
17550
17551 #[test]
17552 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17553 let mut state = review_state(&["feat: x"]);
17554 state.reviews = vec![
17555 round_with_notes(1, Some(SANDBOXED)),
17556 passing_round(2, Some("0123456789abcdef")),
17557 ];
17558 ok_gate(&mut state, Some("0123456789abcdef"));
17559 let body = body_at(&state);
17560 assert!(!body.contains("HTTP tests could not run"), "{body}");
17561 assert!(!body.contains("could not commit"), "{body}");
17563 assert!(
17564 body.contains("passed on an earlier commit (`0123456`)"),
17565 "{body}"
17566 );
17567 assert!(!body.contains("passed on the final commit"), "{body}");
17568
17569 ok_gate(&mut state, None);
17571 let body = body_at(&state);
17572 assert!(
17573 body.contains("passed on a commit that was not recorded"),
17574 "{body}"
17575 );
17576 }
17577
17578 #[test]
17579 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17580 let state = state_with_summary(
17581 "add retries to the client",
17582 "TITLE: feat: retries\n\nDid it.",
17583 );
17584 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17585 assert_eq!(m.title, "feat: retries");
17586 assert!(
17587 m.body.contains("<summary>Original task</summary>"),
17588 "{}",
17589 m.body
17590 );
17591 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17592 assert!(
17593 m.body
17594 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17595 "{}",
17596 m.body
17597 );
17598 }
17599
17600 #[test]
17601 fn pr_message_review_bounds_a_long_english_subject() {
17602 let long = format!("feat: {}", "word ".repeat(100));
17603 let m = pr_message(&review_state(&[&long]), 'A');
17604 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17605 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17606 }
17607
17608 #[test]
17609 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17610 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17611 let m = pr_message(&state, 'A');
17612 assert_eq!(m.title, "add retries");
17613 assert!(m.body.contains("<summary>Original task</summary>"));
17614 assert!(!m.body.contains("Commits under review"));
17615 assert_eq!(landing_subject_source(&state), state.instruction);
17616 }
17617
17618 #[test]
17619 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17620 let state = review_state(&["feat: the change", "fix: typo"]);
17621 let source = landing_subject_source(&state);
17622 assert_eq!(land::merge_subject("", &source), "feat: the change");
17623 assert_eq!(
17624 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17625 "feat: the change"
17626 );
17627 assert_eq!(
17629 land::merge_subject("feat: renamed by hand", &source),
17630 "feat: renamed by hand"
17631 );
17632 let blank = review_state(&["日本語"]);
17633 assert!(
17634 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17635 );
17636 }
17637
17638 #[test]
17639 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17640 let state = review_state(&["feat: the change"]);
17641 let source = landing_subject_source(&state);
17642 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17643 assert_eq!(
17644 land::merge_subject(landing_title(&state, old), &source),
17645 "feat: the change"
17646 );
17647 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17648 let task = state_with_summary("add retries", "");
17649 assert_eq!(landing_title(&task, old), old);
17650 }
17651
17652 #[test]
17653 fn pr_message_describes_the_change_not_the_task() {
17654 let state = state_with_summary(
17655 "今回やってほしいこと: results projector を直す",
17656 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17657 );
17658 let m = pr_message(&state, 'A');
17659 assert_eq!(m.title, "fix(web): batch the runs list reads");
17660 assert!(
17661 m.body.starts_with("## Summary\n\n- reads run.json once"),
17662 "{}",
17663 m.body
17664 );
17665 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17666 let task_at = m.body.find("今回やってほしいこと").unwrap();
17667 let details_at = m.body.find("<details>").unwrap();
17668 assert!(
17669 details_at < task_at,
17670 "the task lives inside <details>: {}",
17671 m.body
17672 );
17673 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17674 assert!(m.body.contains("magi:candidate-a"));
17675 }
17676
17677 #[test]
17678 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17679 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17680 let m = pr_message(&state, 'A');
17681 assert_eq!(m.title, "add retries");
17682 assert!(
17683 m.body.contains("## Summary\n\n- did some things"),
17684 "{}",
17685 m.body
17686 );
17687
17688 let none = RunState::new(
17689 PathBuf::from("/repo"),
17690 "main".to_owned(),
17691 "abc1234".to_owned(),
17692 "add retries".to_owned(),
17693 Config::default(),
17694 );
17695 let m = pr_message(&none, 'A');
17696 assert_eq!(m.title, "add retries");
17697 assert!(!m.body.contains("## Summary"), "{}", m.body);
17698 }
17699
17700 #[test]
17701 fn pr_message_refuses_the_candidate_commit_subject() {
17702 for bad in [
17703 "TITLE: magi: candidate A (uncommitted work)",
17704 "TITLE: chore: stuff (uncommitted work)",
17705 "TITLE: ",
17706 ] {
17707 let state = state_with_summary("add retries", bad);
17708 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17709 }
17710 }
17711
17712 #[test]
17713 fn pr_message_bounds_a_very_long_task_and_title() {
17714 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17715 let state = state_with_summary(&long, "- nothing");
17716 let m = pr_message(&state, 'A');
17717 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17718 assert!(!m.title.contains('\n'));
17719
17720 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17721 let m = pr_message(&state, 'A');
17722 assert!(m.title.starts_with("feat: "));
17723 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17724 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17725 }
17726
17727 fn long_title_of(instruction: &str) -> String {
17728 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17729 }
17730
17731 #[test]
17732 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17733 let first = "Make the landing path keep a readable title for long tasks";
17734 let line = format!(
17735 "{first}. {}",
17736 "And then keep going with more words ".repeat(20)
17737 );
17738 let t = long_title_of(&line);
17739 assert_eq!(t, first);
17740 assert!(!t.starts_with("chore: land"));
17741 }
17742
17743 #[test]
17744 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17745 let line = "word ".repeat(200);
17746 let t = long_title_of(&line);
17747 assert!(t.ends_with("word..."), "{t}");
17748 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17749 }
17750
17751 #[test]
17752 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17753 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17754 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17755 }
17756 }
17757
17758 #[test]
17759 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17760 let mut state = state_with_summary("再試行を追加する", "- nothing");
17761 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17762 state.config.graph.github_language = "ja".into();
17763 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17764 }
17765
17766 #[test]
17767 fn pr_message_title_limit_is_exact() {
17768 let at = "a".repeat(PR_TITLE_MAX);
17769 assert_eq!(long_title_of(&at), at);
17770 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17771 assert!(over.ends_with("..."), "{over}");
17772 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17773 }
17774
17775 #[test]
17776 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17777 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17778 let t = long_title_of(&line);
17779 assert!(t.ends_with("..."), "{t}");
17780 assert!(t.is_ascii(), "{t}");
17781 }
17782
17783 #[test]
17784 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17785 let line = format!(
17786 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17787 "plus more filler words ".repeat(20)
17788 );
17789 let t = long_title_of(&line);
17790 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17791 }
17792
17793 #[test]
17794 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17795 let line = format!("\n\n# {}", "title words ".repeat(40));
17796 let state = state_with_summary(&line, "- nothing");
17797 let m = pr_message(&state, 'A');
17798 assert!(m.title.starts_with("title words"), "{}", m.title);
17799 assert_eq!(
17800 land::merge_subject(&m.title, &landing_subject_source(&state)),
17801 m.title
17802 );
17803 assert_eq!(
17805 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17806 "feat: renamed by hand"
17807 );
17808 }
17809
17810 #[tokio::test]
17811 async fn github_text_rewrite_is_bounded_and_falls_back() {
17812 crate::run::pin_test_home();
17813 for (reply, accepted) in [
17814 (
17815 "TITLE: fix: retries\nAdd retries for failed requests.",
17816 true,
17817 ),
17818 (
17819 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17820 false,
17821 ),
17822 (
17823 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17824 true,
17825 ),
17826 (
17827 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17828 true,
17829 ),
17830 (
17831 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17832 true,
17833 ),
17834 (
17835 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17836 false,
17837 ),
17838 ("TITLE: fix: retries\n", false),
17839 ] {
17840 let dir = tempfile::tempdir().unwrap();
17841 let mut runner = runner_at(RunStatus::Gating);
17842 runner.state = state_with_summary(
17843 "add retries",
17844 "TITLE: fix: retries\n日本語の説明を書きます。",
17845 );
17846 runner.state.repo = dir.path().to_owned();
17847 runner.state.candidates[0].worktree = dir.path().to_owned();
17848 let mut author = spec("alpha");
17849 author.command = vec![
17850 "sh".into(),
17851 "-c".into(),
17852 "printf '%s' \"$REWRITE_REPLY\"".into(),
17853 ];
17854 author.env.insert("REWRITE_REPLY".into(), reply.into());
17855 runner.state.config.agents = vec![author];
17856 seed_public_summary(&mut runner);
17857 let winner = runner.state.candidates[0].clone();
17858 let first = runner
17859 .guarded_pr_message(&winner, None, true)
17860 .await
17861 .unwrap();
17862 let message = if accepted {
17863 first.expect("a passing rewrite never asks")
17864 } else {
17865 assert!(first.is_none());
17867 assert!(runner.state.parked);
17868 let store = ask::Questions::open();
17869 let asked: Vec<_> = store
17870 .list()
17871 .into_iter()
17872 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17873 .collect();
17874 assert_eq!(asked.len(), 1);
17875 assert!(!asked[0].detail.contains("secret"));
17876 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17877 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17878 assert_eq!(
17879 asked[0]
17880 .choices
17881 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17882 rewrite.is_some()
17883 );
17884 store
17887 .update(&asked[0].id, |q| {
17888 q.abandon("test");
17889 Ok(())
17890 })
17891 .unwrap();
17892 let again = runner
17893 .guarded_pr_message(&winner, None, true)
17894 .await
17895 .unwrap()
17896 .expect("an abandoned question falls back");
17897 let n = store
17898 .list()
17899 .into_iter()
17900 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17901 .count();
17902 assert_eq!(n, 1, "asked once per rejected text");
17903 again
17904 };
17905 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17906 assert!(!message.body.contains("secret"));
17907 assert!(!message.body.contains("prefix"));
17908 assert!(!message.title.contains("secret"));
17909 assert_eq!(
17910 message.body.contains("Add retries for failed requests."),
17911 accepted
17912 );
17913 assert_eq!(
17914 runner.state.seats["github-text-rewrite"].turns, 1,
17915 "only one rewrite invocation"
17916 );
17917 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17918 if accepted {
17919 assert!(runner.state.github_text.is_none());
17920 assert!(!runner.state.parked);
17921 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17922 assert!(message.body.contains("## Final verification"));
17923 assert!(message.body.contains("Evidence: final diff"));
17924 assert!(
17925 message
17926 .body
17927 .contains(&format!("magi:run/{}", runner.state.id))
17928 );
17929 }
17930 if !accepted {
17931 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17932 assert!(
17933 message
17934 .body
17935 .contains(&format!("magi:run/{}", runner.state.id))
17936 );
17937 }
17938 }
17939 }
17940
17941 fn scripted(id: &str, reply: &str) -> AgentSpec {
17942 let mut a = spec(id);
17943 a.command = vec![
17944 "sh".into(),
17945 "-c".into(),
17946 "printf '%s' \"$REWRITE_REPLY\"".into(),
17947 ];
17948 a.env.insert("REWRITE_REPLY".into(), reply.into());
17949 a
17950 }
17951
17952 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17953 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17954
17955 fn seed_public_summary(runner: &mut Runner) {
17957 let repo = &runner.state.repo;
17958 let g = |args: &[&str]| {
17959 let out = std::process::Command::new("git")
17960 .args(args)
17961 .current_dir(repo)
17962 .quiet()
17963 .output()
17964 .unwrap();
17965 assert!(out.status.success(), "git {args:?}");
17966 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17967 };
17968 g(&["init", "-q", "-b", "main"]);
17969 g(&[
17970 "-c",
17971 "user.name=Test",
17972 "-c",
17973 "user.email=test@example.invalid",
17974 "commit",
17975 "-q",
17976 "--allow-empty",
17977 "-m",
17978 "feat: fixture",
17979 ]);
17980 let head = g(&["rev-parse", "HEAD"]);
17981 g(&["update-ref", "refs/remotes/origin/main", &head]);
17982 let evidence = crate::pr_outcome::summary_evidence(
17983 &runner.state,
17984 crate::pr_outcome::FinalHead::At(&head),
17985 );
17986 let generated = &runner.state.candidates[0].summary;
17987 runner.state.change_summary = Some(crate::run::ChangeSummary {
17988 base: head.clone(),
17989 head,
17990 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17991 title: summary_title(generated).unwrap(),
17992 body: summary_without_title(generated),
17993 });
17994 }
17995
17996 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17997 let mut runner = runner_at(RunStatus::Gating);
17998 runner.state = state_with_summary("add retries", JA_SUMMARY);
17999 if review_only {
18000 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18001 }
18002 runner.state.repo = dir.to_owned();
18003 runner.state.candidates[0].worktree = dir.to_owned();
18004 runner.state.config.agents = agents;
18005 seed_public_summary(&mut runner);
18006 runner
18007 }
18008
18009 fn gate_questions(runner: &Runner) -> usize {
18010 ask::Questions::open()
18011 .list()
18012 .into_iter()
18013 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18014 .count()
18015 }
18016
18017 #[tokio::test]
18018 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18019 crate::run::pin_test_home();
18020 let dir = tempfile::tempdir().unwrap();
18021 let mut runner = rewrite_runner(
18022 dir.path(),
18023 true,
18024 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18025 );
18026 runner.state.config.roles.language_judge =
18027 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18028 let winner = runner.state.candidates[0].clone();
18029 let msg = runner
18030 .guarded_pr_message(&winner, None, true)
18031 .await
18032 .unwrap()
18033 .expect("an English rewrite posts without asking");
18034 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18035 assert!(msg.body.contains("<details>kept</details>"));
18036 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18037 assert_eq!(gate_questions(&runner), 0);
18038 assert!(runner.state.github_text.is_none());
18039 assert!(!runner.state.seats.contains_key("impl-A"));
18040 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18041 }
18042
18043 #[tokio::test]
18044 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18045 crate::run::pin_test_home();
18046 let dir = tempfile::tempdir().unwrap();
18047 let mut runner = rewrite_runner(
18048 dir.path(),
18049 false,
18050 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18051 );
18052 runner.state.config.roles.language_judge =
18053 Some(crate::config::AgentChoice::One("beta".into()));
18054 runner.state.seats.insert(
18055 "impl-A".into(),
18056 SeatState::new("impl-A", "alpha", runner.state.seed),
18057 );
18058 let winner = runner.state.candidates[0].clone();
18059 let msg = runner
18060 .guarded_pr_message(&winner, None, true)
18061 .await
18062 .unwrap()
18063 .expect("rewritten");
18064 assert!(msg.body.contains("<details>kept</details>"));
18065 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18066 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18067 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18068 }
18069
18070 #[tokio::test]
18071 async fn rewrite_reuses_a_resumable_implementer_session() {
18072 crate::run::pin_test_home();
18073 let dir = tempfile::tempdir().unwrap();
18074 let mut runner = rewrite_runner(
18075 dir.path(),
18076 false,
18077 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18078 );
18079 runner.state.config.roles.language_judge =
18080 Some(crate::config::AgentChoice::One("beta".into()));
18081 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18082 seat.turns = 1;
18083 runner.state.seats.insert("impl-A".into(), seat);
18084 let winner = runner.state.candidates[0].clone();
18085 runner
18086 .guarded_pr_message(&winner, None, true)
18087 .await
18088 .unwrap()
18089 .expect("rewritten");
18090 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18091 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18092 }
18093
18094 #[tokio::test]
18095 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18096 crate::run::pin_test_home();
18097 let dir = tempfile::tempdir().unwrap();
18098 let mut broken = spec("bad");
18099 broken.command = vec!["false".into()];
18100 let mut runner = rewrite_runner(
18101 dir.path(),
18102 true,
18103 vec![
18104 broken,
18105 scripted("noformat", "Add retries."),
18106 scripted("good", EN_REWRITE),
18107 ],
18108 );
18109 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18110 "bad".into(),
18111 "noformat".into(),
18112 "good".into(),
18113 ]));
18114 let winner = runner.state.candidates[0].clone();
18115 let msg = runner
18116 .guarded_pr_message(&winner, None, true)
18117 .await
18118 .unwrap()
18119 .expect("the third agent rewrites");
18120 assert!(msg.body.contains("`src/net.rs`"));
18121 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18122 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18123 }
18124
18125 #[tokio::test]
18126 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18127 crate::run::pin_test_home();
18128 for (agents, role, saved) in [
18130 (
18131 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18132 Some("beta"),
18133 true,
18134 ),
18135 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18136 (Vec::new(), None, false),
18137 ] {
18138 let dir = tempfile::tempdir().unwrap();
18139 let mut runner = rewrite_runner(dir.path(), true, agents);
18140 runner.state.config.roles.language_judge =
18141 role.map(|r| crate::config::AgentChoice::One(r.into()));
18142 let winner = runner.state.candidates[0].clone();
18143 let first = runner
18144 .guarded_pr_message(&winner, None, true)
18145 .await
18146 .unwrap();
18147 assert!(first.is_none(), "withheld text parks the run");
18148 assert!(runner.state.parked);
18149 assert_eq!(gate_questions(&runner), 1);
18150 let g = runner.state.github_text.as_ref().unwrap();
18151 assert_eq!(g.rewrite.is_some(), saved);
18152 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18153 }
18154 }
18155
18156 #[tokio::test]
18157 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18158 crate::run::pin_test_home();
18159 let dir = tempfile::tempdir().unwrap();
18160 let mut runner = runner_at(RunStatus::Gating);
18161 runner.state =
18162 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18163 runner.state.repo = dir.path().to_owned();
18164 runner.state.candidates[0].worktree = dir.path().to_owned();
18165 let mut author = spec("alpha");
18166 author.command = vec![
18167 "sh".into(),
18168 "-c".into(),
18169 "printf '%s' \"$REWRITE_REPLY\"".into(),
18170 ];
18171 author.env.insert(
18172 "REWRITE_REPLY".into(),
18173 format!(
18174 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18175 "English sentence. ".repeat(600)
18176 ),
18177 );
18178 runner.state.config.agents = vec![author];
18179 seed_public_summary(&mut runner);
18180 let winner = runner.state.candidates[0].clone();
18181 assert!(
18182 runner
18183 .guarded_pr_message(&winner, None, true)
18184 .await
18185 .unwrap()
18186 .is_none()
18187 );
18188 let record = runner.state.github_text.clone().unwrap();
18189 let rewrite = record.rewrite.clone().unwrap();
18190 assert_eq!(rewrite.failed, ["body-language"]);
18191 assert!(
18192 rewrite
18193 .body
18194 .ends_with(&format!("magi:run/{}", runner.state.id))
18195 );
18196 let store = ask::Questions::open();
18197 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18198 assert!(
18199 q.choices
18200 .contains(&crate::github_text::USE_REWRITE.to_owned())
18201 );
18202 for label in [
18203 "ORIGINAL",
18204 "FALLBACK",
18205 "REWRITE",
18206 "Rules still failed: body-language",
18207 "truncated",
18208 ] {
18209 assert!(q.detail.contains(label), "{label}");
18210 }
18211 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18212 &runner.state,
18213 &crate::github_text::artifact_name(&record.fingerprint),
18214 ))
18215 .unwrap();
18216 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18217 runner.state =
18219 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18220 store
18221 .update(&q.id, |q| {
18222 q.say("まだ日本語")?;
18223 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18224 })
18225 .unwrap();
18226 assert!(
18227 runner
18228 .guarded_pr_message(&winner, None, true)
18229 .await
18230 .unwrap()
18231 .is_none()
18232 );
18233 let retry = runner.state.github_text.as_ref().unwrap();
18234 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18235 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18236 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18237 assert!(
18238 q.posting_comparison
18239 .as_ref()
18240 .unwrap()
18241 .matches_detail(&q.detail)
18242 );
18243 store
18244 .update(&q.id, |q| {
18245 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18246 })
18247 .unwrap();
18248 let message = runner
18249 .guarded_pr_message(&winner, None, true)
18250 .await
18251 .unwrap()
18252 .unwrap();
18253 assert_eq!(message.title, rewrite.title);
18254 assert_eq!(message.body, rewrite.body);
18255 assert_eq!(
18256 runner.state.github_text.as_ref().unwrap().choice,
18257 Some(GithubTextChoice::Rewrite)
18258 );
18259 let again = runner
18260 .guarded_pr_message(&winner, None, true)
18261 .await
18262 .unwrap()
18263 .unwrap();
18264 assert_eq!(again.title, message.title);
18265 assert_eq!(again.body, message.body);
18266 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18267 assert!(
18268 !runner
18269 .state
18270 .events
18271 .iter()
18272 .any(|e| e.message.contains(&rewrite.title)
18273 || e.message.contains("English sentence."))
18274 );
18275 }
18276
18277 #[tokio::test]
18278 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18279 crate::run::pin_test_home();
18280 let dir = tempfile::tempdir().unwrap();
18281 let mut runner = runner_at(RunStatus::Gating);
18282 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18283 runner.state.repo = dir.path().to_owned();
18284 runner.state.candidates[0].worktree = dir.path().to_owned();
18285 seed_public_summary(&mut runner);
18286 let winner = runner.state.candidates[0].clone();
18287 assert!(
18288 runner
18289 .guarded_pr_message(&winner, None, true)
18290 .await
18291 .unwrap()
18292 .is_none()
18293 );
18294 let record = runner.state.github_text.as_ref().unwrap();
18295 assert!(record.rewrite.is_none());
18296 let mut old = serde_json::to_value(record).unwrap();
18298 old.as_object_mut().unwrap().remove("rewrite");
18299 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18300 assert!(old.rewrite.is_none());
18301 let id = record.question.clone().unwrap();
18302 ask::Questions::open()
18303 .update(&id, |q| {
18304 q.choices.push(crate::github_text::USE_REWRITE.into());
18305 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18306 })
18307 .unwrap();
18308 let message = runner
18309 .guarded_pr_message(&winner, None, true)
18310 .await
18311 .unwrap()
18312 .unwrap();
18313 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18314 assert_eq!(
18315 runner.state.github_text.as_ref().unwrap().choice,
18316 Some(GithubTextChoice::Fallback)
18317 );
18318 }
18319
18320 #[tokio::test]
18321 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18322 crate::run::pin_test_home();
18323 let dir = tempfile::tempdir().unwrap();
18324 let mut runner = runner_at(RunStatus::Gating);
18325 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18326 runner.state.repo = dir.path().to_owned();
18327 runner.state.candidates[0].worktree = dir.path().to_owned();
18328 seed_public_summary(&mut runner);
18329 let winner = runner.state.candidates[0].clone();
18330 assert!(
18331 runner
18332 .guarded_pr_message(&winner, None, true)
18333 .await
18334 .unwrap()
18335 .is_none()
18336 );
18337 let store = ask::Questions::open();
18338 let find = |run: &str| {
18339 store
18340 .list()
18341 .into_iter()
18342 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18343 .unwrap()
18344 };
18345 let run = runner.state.id.clone();
18346 let q = find(&run);
18347 let fp = runner
18348 .state
18349 .github_text
18350 .as_ref()
18351 .unwrap()
18352 .fingerprint
18353 .clone();
18354 let name = crate::github_text::artifact_name(&fp);
18355 let path = crate::run::artifact_path(&runner.state, &name);
18356 assert!(q.detail.contains(&path.display().to_string()));
18357 assert!(q.detail.contains("title-language: title"));
18358 let saved = std::fs::read_to_string(&path).expect("artifact written");
18359 assert!(saved.contains("日本語のタイトル"));
18360 assert!(q.cwd.is_none());
18361 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18362 let say = |id: &str, text: &str, choice: &str| {
18364 store
18365 .update(id, |q| {
18366 q.thread.push(ask::Turn {
18367 who: ask::Who::Operator,
18368 body: text.to_owned(),
18369 at: jiff::Timestamp::now(),
18370 note: None,
18371 });
18372 q.answer(ask::Answer::Choice(choice.to_owned()))
18373 })
18374 .unwrap();
18375 };
18376 say(&q.id, "まだ日本語", "use my text");
18377 assert!(
18378 runner
18379 .guarded_pr_message(&winner, None, true)
18380 .await
18381 .unwrap()
18382 .is_none()
18383 );
18384 let second = find(&run);
18385 assert_ne!(second.id, q.id);
18386 assert!(second.detail.contains("did not pass"));
18387 assert!(second.detail.contains("日本語のタイトル"));
18390 assert!(!second.detail.contains("まだ日本語"));
18391 assert!(second.detail.contains(&path.display().to_string()));
18392 say(&second.id, "fix: retry failed requests", "use my text");
18393 let message = runner
18394 .guarded_pr_message(&winner, None, true)
18395 .await
18396 .unwrap()
18397 .expect("a vetted replacement is posted");
18398 assert_eq!(message.title, "fix: retry failed requests");
18399 assert_eq!(
18400 runner
18401 .state
18402 .github_text
18403 .as_ref()
18404 .unwrap()
18405 .chosen_title
18406 .as_deref(),
18407 Some("fix: retry failed requests")
18408 );
18409 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18410 let again = runner
18412 .guarded_pr_message(&winner, None, true)
18413 .await
18414 .unwrap()
18415 .unwrap();
18416 assert_eq!(again.title, message.title);
18417 assert_eq!(
18418 store
18419 .list()
18420 .into_iter()
18421 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18422 .count(),
18423 2
18424 );
18425 }
18426
18427 #[test]
18428 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18429 let mut state = state_with_summary(
18433 "add retries",
18434 "TITLE: fix(web): batch reads\n- reads run.json once",
18435 );
18436 state.config.graph.language = "ja".to_owned();
18437 let m = pr_message(&state, 'A');
18438 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18439 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18440
18441 let task = "今回やってほしいこと: results projector を直す";
18444 let mut state = state_with_summary(task, "- no title line");
18445 state.config.graph.language = "ja".to_owned();
18446 let m = pr_message(&state, 'A');
18447 assert_eq!(
18448 m.title,
18449 format!("chore: land candidate A of run {}", state.id)
18450 );
18451 assert!(
18452 m.body.contains(&format!(
18453 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18454 )),
18455 "{}",
18456 m.body
18457 );
18458 }
18459
18460 #[test]
18461 fn pr_message_scrubs_home_paths_and_addresses() {
18462 let state = state_with_summary(
18463 "fix it in /Users/someone/src/x",
18464 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18465 );
18466 let m = pr_message(&state, 'A');
18467 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18468 assert!(!m.body.contains(leak), "{}", m.body);
18469 }
18470 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18471 }
18472
18473 #[test]
18474 fn pr_message_survives_a_task_that_closes_details() {
18475 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18476 let m = pr_message(&state, 'A');
18477 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18478 }
18479
18480 #[test]
18481 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18482 let cmd = manual_merge_command(
18483 MergeStyle::Squash,
18484 Path::new("/repo"),
18485 "b",
18486 "fix: \"quoted\" $(x) `y`\n\nbody",
18487 );
18488 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18489 }
18490
18491 #[test]
18492 fn manual_merge_command_matches_the_configured_style() {
18493 let repo = Path::new("/repo");
18494 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18495
18496 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18497 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18498
18499 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18500 assert_eq!(
18501 squash,
18502 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18503 \"Merge magi run 0832 (candidate A)\""
18504 );
18505
18506 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18507 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18508 }
18509
18510 #[test]
18511 fn a_nudge_gets_a_quarter_of_the_budget() {
18512 assert_eq!(retry_budget(secs(1200), true), secs(300));
18514 assert_eq!(retry_budget(secs(3600), true), secs(900));
18515 }
18516
18517 #[test]
18518 fn a_resent_prompt_keeps_the_whole_budget() {
18519 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18522 assert_eq!(retry_budget(secs(60), false), secs(60));
18523 }
18524
18525 #[test]
18526 fn the_floor_never_exceeds_the_original_budget() {
18527 assert_eq!(retry_budget(secs(60), true), secs(60));
18531 assert_eq!(retry_budget(secs(480), true), secs(120));
18532 assert_eq!(retry_budget(secs(0), true), secs(0));
18533 }
18534
18535 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18536 agent::CommandEvidence {
18537 id: "item1".to_owned(),
18538 description: "cargo test".to_owned(),
18539 exit_code,
18540 result_summary: String::new(),
18541 source: "codex".to_owned(),
18542 }
18543 }
18544
18545 #[test]
18546 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18547 assert!(!has_unconfirmed_command(&[]));
18551 }
18552
18553 #[test]
18554 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18555 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18559 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18560 assert!(!has_unconfirmed_command(&[
18561 evidence(Some(0)),
18562 evidence(Some(101))
18563 ]));
18564 }
18565
18566 #[test]
18567 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18568 assert!(has_unconfirmed_command(&[
18569 evidence(Some(0)),
18570 evidence(None)
18571 ]));
18572 }
18573
18574 #[test]
18575 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18576 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18577 assert_eq!(
18578 verified_noop_claim(true, &[], text).as_deref(),
18579 Some("already fixed by b32cfc4, on main.")
18580 );
18581 }
18582
18583 #[test]
18584 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18585 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18588 assert!(verified_noop_claim(false, &[], text).is_none());
18589 }
18590
18591 #[test]
18592 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18593 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18594 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18595 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18597 }
18598
18599 #[test]
18600 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18601 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18602 }
18603
18604 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18607 runner.state.candidates = shape
18608 .iter()
18609 .enumerate()
18610 .map(|(i, &(empty, verified))| Candidate {
18611 index: i,
18612 label: (b'A' + i as u8) as char,
18613 agent: "sonnet".to_owned(),
18614 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18615 worktree: PathBuf::from(format!("/wt/{i}")),
18616 summary: String::new(),
18617 stat: String::new(),
18618 files: 0,
18619 commits: 0,
18620 empty,
18621 failed: None,
18622 verified_noop: verified.map(str::to_owned),
18623 duration_ms: 0,
18624 folded: false,
18625 })
18626 .collect();
18627 }
18628
18629 #[test]
18630 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18631 ask_test_home();
18632 let mut runner = runner_at(RunStatus::Implementing);
18633 set_candidates(
18634 &mut runner,
18635 &[
18636 (true, Some("already on main at b32cfc4")),
18637 (true, Some("same fix, see the existing test")),
18638 ],
18639 );
18640
18641 runner
18642 .after_implement()
18643 .expect("a verified no-op is not an error");
18644
18645 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18646 }
18647
18648 #[test]
18649 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18650 ask_test_home();
18651 let mut runner = runner_at(RunStatus::Implementing);
18652 set_candidates(
18656 &mut runner,
18657 &[(true, Some("already on main at b32cfc4")), (true, None)],
18658 );
18659
18660 let err = runner
18661 .after_implement()
18662 .expect_err("an unverified empty candidate must still fail the run");
18663
18664 assert!(
18665 err.to_string().contains("no candidate produced a change"),
18666 "{err}"
18667 );
18668 assert_eq!(runner.state.status, RunStatus::Failed);
18669 }
18670
18671 #[test]
18672 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18673 ask_test_home();
18674 let mut runner = runner_at(RunStatus::Implementing);
18675 set_candidates(&mut runner, &[(true, None), (true, None)]);
18676
18677 let err = runner
18678 .after_implement()
18679 .expect_err("no candidate declared anything; this is an ordinary failure");
18680
18681 assert!(
18682 err.to_string().contains("no candidate produced a change"),
18683 "{err}"
18684 );
18685 assert_eq!(runner.state.status, RunStatus::Failed);
18686 }
18687}
18688
18689#[derive(Debug)]
18692struct GraftRestoreFailed(String);
18693
18694impl std::fmt::Display for GraftRestoreFailed {
18695 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18696 write!(
18697 f,
18698 "could not restore the winner to its pre-graft commit: {}",
18699 self.0
18700 )
18701 }
18702}
18703
18704impl std::error::Error for GraftRestoreFailed {}
18705
18706#[cfg(test)]
18710mod resource_wait_tests {
18711 use super::*;
18712 use std::collections::BTreeMap;
18713
18714 fn sh(id: &str, script: &str) -> AgentSpec {
18715 AgentSpec {
18716 id: id.to_owned(),
18717 kind: crate::config::AgentKind::Command,
18718 model: None,
18719 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18720 extra_args: Vec::new(),
18721 env: BTreeMap::new(),
18722 prompt_delivery: None,
18723 }
18724 }
18725
18726 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18727 SeatJob {
18728 seat: SeatState::new("impl-A", &spec.id, 1),
18729 spec,
18730 cwd: dir.to_path_buf(),
18731 prompt: "do it".to_owned(),
18732 timeout,
18733 allow_write: true,
18734 sessions: false,
18735 artifacts: dir.join("art"),
18736 stem: "impl-A".to_owned(),
18737 handover: None,
18738 }
18739 }
18740
18741 fn state(resource_wait_secs: u64) -> RunState {
18742 let mut config = Config::default();
18743 config.graph.timeout_resource_wait = resource_wait_secs;
18744 RunState::new(
18745 PathBuf::from("/repo"),
18746 "main".to_owned(),
18747 "abc1234".to_owned(),
18748 "task".to_owned(),
18749 config,
18750 )
18751 }
18752
18753 fn hold(cache: &Path) -> crate::cache::Guard {
18754 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18755 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18756 crate::cache::AcquireOutcome::Acquired(g) => g,
18757 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18758 }
18759 }
18760
18761 async fn run(
18762 jobs: Vec<SeatJob>,
18763 sem: Arc<Semaphore>,
18764 cache: Option<&Path>,
18765 st: &mut RunState,
18766 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18767 let prompts = Prompts::default();
18768 let ctx = WaveCtx {
18769 run: "r",
18770 node: "implement",
18771 prompts: &prompts,
18772 cache,
18773 round: None,
18774 carry_seats: false,
18775 pauses: None,
18776 };
18777 wave(jobs, sem, &ctx, st, 0).await
18778 }
18779
18780 #[tokio::test]
18781 async fn verification_cache_wait_preserves_the_command_budget() {
18782 crate::run::pin_test_home();
18783 let tmp = tempfile::tempdir().unwrap();
18784 let cache = tmp.path().join("target");
18785 let guard = hold(&cache);
18786 let release = tokio::spawn(async move {
18787 tokio::time::sleep(Duration::from_millis(100)).await;
18788 guard.release();
18789 });
18790 let mut st = state(10);
18791 let budget = Duration::from_millis(50);
18792 let wait = repository_wait(&st, None);
18793 let (outcomes, _) = with_cache_lease(
18794 &mut st,
18795 Some(&cache),
18796 "gate",
18797 "gate",
18798 tmp.path(),
18799 "abc1234",
18800 budget,
18801 "test gate",
18802 &wait,
18803 |_, available| async move {
18804 assert_eq!(
18805 available, budget,
18806 "contention must not shorten verification"
18807 );
18808 (Vec::new(), false, Vec::new())
18809 },
18810 )
18811 .await
18812 .unwrap();
18813 release.await.unwrap();
18814 assert!(outcomes.is_empty(), "{outcomes:?}");
18815 assert!(
18816 st.events
18817 .iter()
18818 .any(|event| event.message.contains("waiting for"))
18819 );
18820 assert!(st.resource_wait.is_none());
18821 hold(&cache).release();
18823 }
18824
18825 #[tokio::test]
18826 async fn verification_cache_wait_expires_without_starting_commands() {
18827 crate::run::pin_test_home();
18828 let tmp = tempfile::tempdir().unwrap();
18829 let cache = tmp.path().join("target");
18830 let _held = hold(&cache);
18831 let mut st = state(0);
18832 let wait = repository_wait(&st, None);
18833 let (outcomes, _) = with_cache_lease(
18834 &mut st,
18835 Some(&cache),
18836 "gate",
18837 "gate",
18838 tmp.path(),
18839 "abc1234",
18840 Duration::from_secs(60),
18841 "test gate",
18842 &wait,
18843 |_, _| async { panic!("commands must not start without the cache lease") },
18844 )
18845 .await
18846 .unwrap();
18847 assert_eq!(outcomes.len(), 1);
18848 assert!(outcomes[0].resource_blocked);
18849 assert!(st.resource_wait.is_none());
18850 }
18851
18852 #[tokio::test]
18853 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18854 crate::run::pin_test_home();
18855 let tmp = tempfile::tempdir().unwrap();
18856 let cache = tmp.path().join("target-a");
18857 let guard = hold(&cache);
18858 tokio::spawn(async move {
18860 tokio::time::sleep(Duration::from_millis(1500)).await;
18861 guard.release();
18862 });
18863 let mut st = state(60);
18864 let results = run(
18865 vec![job(
18866 sh("alpha", "echo ok"),
18867 tmp.path(),
18868 Duration::from_secs(1),
18869 )],
18870 Arc::new(Semaphore::new(1)),
18871 Some(&cache),
18872 &mut st,
18873 )
18874 .await;
18875 assert!(
18876 matches!(results[0].2, AgentOutcome::Ok(_)),
18877 "the wait must not be charged to the agent"
18878 );
18879 assert!(st.resource_block.is_none());
18880 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18881 assert!(st.resource_wait.is_none() && st.active.is_empty());
18882 }
18883
18884 #[tokio::test]
18885 #[cfg(unix)]
18886 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18887 use std::os::fd::AsRawFd as _;
18888 crate::run::pin_test_home();
18889 let tmp = tempfile::tempdir().unwrap();
18890 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18891 let common = tmp.path().join(".git");
18892 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18893 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18894 assert_eq!(
18895 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18896 0
18897 );
18898 let mut st = state(1);
18899 st.config.coordination = crate::config::test_coordination();
18900 let mut reader = job(
18901 sh("reader", "echo must-not-run"),
18902 tmp.path(),
18903 Duration::from_secs(30),
18904 );
18905 reader.allow_write = false;
18906 let results = run(
18907 vec![reader.clone()],
18908 Arc::new(Semaphore::new(1)),
18909 None,
18910 &mut st,
18911 )
18912 .await;
18913 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18914 assert_eq!(results[0].1.turns, 0);
18915 assert!(FailClass::of(&results[0].2).is_none());
18916 assert_eq!(
18917 st.resource_block.as_ref().unwrap().kind,
18918 "repository coordination"
18919 );
18920 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18921 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18922 drop(external);
18923 let editor = crate::repo_lock::SeatLease::for_checkout(
18924 tmp.path(),
18925 true,
18926 crate::config::test_coordination().as_ref(),
18927 )
18928 .await
18929 .unwrap();
18930 let pause = Pause::new();
18931 let interrupt = Pause::new();
18932 let prompts = Prompts::default();
18933 let ctx = WaveCtx {
18934 run: "r",
18935 node: "review",
18936 prompts: &prompts,
18937 cache: None,
18938 round: None,
18939 carry_seats: false,
18940 pauses: Some((&pause, &interrupt)),
18941 };
18942 let (results, ()) = tokio::join!(
18943 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18944 async {
18945 tokio::time::sleep(Duration::from_millis(50)).await;
18946 pause.park();
18947 }
18948 );
18949 assert!(
18950 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18951 );
18952 assert_eq!(results[0].1.turns, 0);
18953 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18954 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18955 drop(editor);
18956 }
18957
18958 #[tokio::test]
18959 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18960 crate::run::pin_test_home();
18961 let tmp = tempfile::tempdir().unwrap();
18962 let cache = tmp.path().join("target-b");
18963 let _held = hold(&cache);
18964 let mut st = state(1);
18965 let results = run(
18966 vec![job(
18967 sh("alpha", "echo ok"),
18968 tmp.path(),
18969 Duration::from_secs(1),
18970 )],
18971 Arc::new(Semaphore::new(1)),
18972 Some(&cache),
18973 &mut st,
18974 )
18975 .await;
18976 let (_, seat, out) = &results[0];
18977 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18978 assert!(wave_blocked(&results));
18979 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18980 assert_eq!(seat.turns, 0, "nothing was asked");
18981 assert!(
18982 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18983 "no agent was spawned"
18984 );
18985 assert!(FailClass::of(out).is_none(), "not a failure class");
18986 let block = st.resource_block.as_ref().expect("recorded for the resume");
18987 assert_eq!(block.kind, "shared build cache");
18988 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18989 assert!(st.active.is_empty());
18990 }
18991
18992 #[tokio::test]
18993 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18994 crate::run::pin_test_home();
18995 let tmp = tempfile::tempdir().unwrap();
18996 let mut st = state(1);
18997 let results = run(
18998 vec![job(
18999 sh("alpha", "echo ok"),
19000 tmp.path(),
19001 Duration::from_secs(1),
19002 )],
19003 Arc::new(Semaphore::new(0)),
19004 None,
19005 &mut st,
19006 )
19007 .await;
19008 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19009 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19010 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19011 }
19012
19013 #[tokio::test]
19014 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19015 crate::run::pin_test_home();
19016 let tmp = tempfile::tempdir().unwrap();
19017 let mut st = state(60);
19018 let results = run(
19019 vec![job(
19020 sh("alpha", "sleep 30"),
19021 tmp.path(),
19022 Duration::from_secs(1),
19023 )],
19024 Arc::new(Semaphore::new(1)),
19025 None,
19026 &mut st,
19027 )
19028 .await;
19029 let out = &results[0].2;
19030 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19031 let class = FailClass::of(out).expect("a failure");
19032 assert_eq!(class, FailClass::Timeout);
19033 assert!(should_hand_over(None, &class), "existing handover policy");
19034 assert!(st.resource_block.is_none());
19035 }
19036}