1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(
488 repo: &Path,
489 base_branch: &str,
490 remote: &str,
491 coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493 let _lease = crate::repo_lock::SeatLease::operation_with(
494 repo,
495 coordination,
496 &crate::repo_lock::WaitPolicy::current(),
497 )
498 .await?;
499 let tracking = format!("{remote}/{base_branch}");
500 let fetched = git::fetch(repo, remote, base_branch).await;
501 if let Ok(out) = &fetched
502 && out.ok()
503 && git::rev_exists(repo, &tracking).await
504 {
505 return Ok(crate::run::FetchedBase {
506 remote: remote.to_owned(),
507 branch: base_branch.to_owned(),
508 commit: git::rev_parse(repo, &tracking).await?,
509 fetched_at: jiff::Timestamp::now(),
510 });
511 }
512 let why = match &fetched {
513 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514 Ok(_) => format!("{remote} has no {base_branch}"),
515 Err(e) => e.to_string(),
516 };
517 bail!(
520 "cannot read {tracking} ({why}); refusing to branch off the local \
521 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522 base / remote in magi.toml. This is launcher preparation: check the \
523 launcher's Git administration access, network and repository coordination \
524 before retrying; an unattended seat cannot approve those permissions"
525 )
526}
527
528struct FixClaim {
545 path: PathBuf,
546}
547
548impl FixClaim {
549 fn acquire(dir: &Path) -> Result<Self> {
550 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551 let path = dir.join("fix.lock");
552 match Self::create(&path) {
553 Ok(claim) => Ok(claim),
554 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555 if Self::reclaim_if_dead(&path) {
556 Self::create(&path).with_context(|| format!("lock {}", path.display()))
557 } else {
558 bail!(
559 "another repair or `magi fix` is already running for this run ({} exists)",
560 path.display()
561 )
562 }
563 }
564 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565 }
566 }
567
568 fn create(path: &Path) -> std::io::Result<Self> {
569 let mut f = std::fs::OpenOptions::new()
570 .write(true)
571 .create_new(true)
572 .open(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
627 repo: &Path,
628 instruction: String,
629 also_scan: &str,
630 config: Config,
631 origin: Origin,
632 ) -> Result<Self> {
633 let repo = git::toplevel(repo).await?;
634 let missing = agent::missing_programs(&config.agents);
635 if !missing.is_empty() {
636 bail!(
637 "these agent programs are not on PATH: {}. Fix the roster in \
638 magi.toml or install them.",
639 missing.join(", ")
640 );
641 }
642 let base_branch =
643 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644 .await?;
645 let fetched_base = resolve_base(
646 &repo,
647 &base_branch,
648 &config.merge.remote,
649 config.coordination.as_ref(),
650 )
651 .await?;
652 let base_commit = fetched_base.commit.clone();
653 let roles = config.resolve_roles()?;
654 let max_parallel = config.graph.max_parallel.max(1);
655 let seeds = refs::resolve(
658 &repo,
659 &base_commit,
660 &config.merge.remote,
661 &format!("{also_scan}\n{instruction}"),
662 )
663 .await;
664 refs::plan(&repo, &seeds).await?;
665 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666 state.fetched_base = Some(fetched_base);
670 state.origin = Some(origin);
671 for seed in &seeds {
672 state.event(
673 "seed",
674 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675 );
676 }
677 state.seeds = seeds;
678 state.event("start", format!("run {} created", state.id));
679 state.save()?;
680 Ok(Self {
681 state,
682 roles,
683 sem: Arc::new(Semaphore::new(max_parallel)),
684 pause: Pause::new(),
685 interrupt: Pause::new(),
686 })
687 }
688
689 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703 Self::review_taking_over(repo, branch, config, None, origin).await
704 }
705
706 pub async fn review_taking_over(
713 repo: &Path,
714 branch: &str,
715 config: Config,
716 takeover: Option<crate::handover::Takeover>,
717 origin: Origin,
718 ) -> Result<Self> {
719 let repo = git::toplevel(repo).await?;
720 let missing = agent::missing_programs(&config.agents);
721 if !missing.is_empty() {
722 bail!(
723 "these agent programs are not on PATH: {}. Fix the roster in \
724 magi.toml or install them.",
725 missing.join(", ")
726 );
727 }
728 let base_branch =
729 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730 .await?;
731 if base_branch == branch {
732 bail!("`{branch}` is the base branch; there is nothing to review against");
733 }
734 let fetched_base = resolve_base(
735 &repo,
736 &base_branch,
737 &config.merge.remote,
738 config.coordination.as_ref(),
739 )
740 .await?;
741 let base_commit = fetched_base.commit.clone();
742
743 let roles = config.resolve_roles()?;
744 let max_parallel = config.graph.max_parallel.max(1);
745 let mut state = RunState::new(
746 repo.clone(),
747 base_branch,
748 base_commit.clone(),
749 String::new(),
750 config,
751 );
752 state.fetched_base = Some(fetched_base);
753 state.origin = Some(origin);
754
755 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762 earlier: Vec::new(),
763 home: crate::run::home(),
764 choice: None,
765 task_instruction: None,
766 });
767 let wait = repository_wait(&state, None);
768 let coordination_config = state.config.coordination.clone();
769 let coordination =
770 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771 .await?;
772 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773 if let Some(released) = &released {
774 state.event(
775 "release",
776 format!(
777 "took `{branch}` over from run {}: its worktree was released: {}",
778 crate::run::short_of(&released.old_id),
779 released.audit
780 ),
781 );
782 }
783 if let Some(choice) = takeover.choice.as_ref()
787 && let Err(e) =
788 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789 .await
790 {
791 if let Some(released) = &released {
792 released.restore(&repo, branch).await;
793 }
794 return Err(e.context("applying the owner's answer about the diverged branch"));
795 }
796 for id in takeover.earlier.iter().rev() {
802 if let Ok(old) = RunState::load_under(id, &takeover.home)
803 && let Some(head) = old.pr_head.clone()
804 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805 {
806 state.pr_head = Some(head);
807 break;
808 }
809 }
810 drop(coordination);
811 let opened = Self::open_review(
812 &repo,
813 branch,
814 state,
815 roles,
816 max_parallel,
817 base_commit,
818 &takeover,
819 )
820 .await;
821 if opened.is_err()
822 && let Some(released) = &released
823 {
824 let _coordination = crate::repo_lock::SeatLease::operation_with(
825 &repo,
826 coordination_config.as_ref(),
827 &wait,
828 )
829 .await?;
830 released.restore(&repo, branch).await;
831 }
832 opened
833 }
834
835 pub async fn recover_pr(
837 repo: &Path,
838 task: &str,
839 selection: crate::pr_recovery::Selection,
840 mut config: Config,
841 ) -> Result<Self> {
842 selection.configure(&mut config)?;
843 selection.check_task(task, &selection.instruction)?;
844 let base_branch =
845 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
846 .await?;
847 let fetched_base = resolve_base(
848 repo,
849 &base_branch,
850 &config.merge.remote,
851 config.coordination.as_ref(),
852 )
853 .await?;
854 let base_commit = fetched_base.commit.clone();
855 selection
856 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
857 .await?;
858 selection.verify_source(repo).await?;
859 let roles = config.resolve_roles()?;
860 let max_parallel = config.graph.max_parallel.max(1);
861 let mut state = RunState::new(
862 repo.to_path_buf(),
863 base_branch,
864 base_commit.clone(),
865 selection.instruction.clone(),
866 config,
867 );
868 state.fetched_base = Some(fetched_base);
869 state.origin = Some(crate::run::Origin::queue(task));
870 state.pr_head = Some(selection.branch.clone());
871 state.pr_recovery = Some(selection.clone());
872 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
873 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
874 }
875
876 async fn open_selected(
877 repo: &Path,
878 state: RunState,
879 roles: ResolvedRoles,
880 max_parallel: usize,
881 base_commit: String,
882 selection: crate::pr_recovery::Selection,
883 ) -> Result<Self> {
884 let mut state = state;
885 let branch = state.branch_for('A');
886 let wait = repository_wait(&state, None);
887 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
888 git::git(repo, &["branch", &branch, selection.source()]).await?;
889 drop(coordination);
890 state.event(
891 "recovery",
892 format!(
893 "selected implementation {} for {} (public lease {})",
894 selection.source(),
895 selection.url,
896 selection.head
897 ),
898 );
899 let takeover = crate::handover::Takeover {
900 earlier: Vec::new(),
901 home: crate::run::home(),
902 choice: None,
903 task_instruction: Some(selection.instruction.clone()),
904 };
905 state.save()?;
906 let id = state.id.clone();
907 let opened = Self::open_review(
908 repo,
909 &branch,
910 state,
911 roles,
912 max_parallel,
913 base_commit,
914 &takeover,
915 )
916 .await;
917 match opened {
918 Ok(runner) => Ok(runner),
919 Err(e) => {
920 if let Ok(mut partial) = RunState::load(&id) {
921 partial.status = RunStatus::Blocked;
922 partial.driver_exited = true;
923 partial.event(
924 "recovery",
925 format!("startup failed; retained source and candidate {branch}: {e:#}"),
926 );
927 partial.save()?;
928 }
929 Err(crate::pr_recovery::StartupFailed {
930 run: id,
931 cause: format!("{e:#}"),
932 }
933 .into())
934 }
935 }
936 }
937
938 async fn inherit(
943 repo: &Path,
944 branch: &str,
945 base_commit: &str,
946 takeover: &crate::handover::Takeover,
947 ) -> Option<crate::recovery::RecoveryRecord> {
948 use crate::recovery::{Relation, assess};
949 let newest = takeover.earlier.last()?;
950 let mut chosen = None;
951 for id in takeover.earlier.iter().rev() {
952 if let Ok(old) = RunState::load_under(id, &takeover.home)
953 && !old.reviews.is_empty()
954 && (old.winner().is_some_and(|w| w.branch == branch)
955 || old.pr_head.as_deref() == Some(branch))
956 {
957 chosen = Some(old);
958 break;
959 }
960 }
961 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
962 .await
963 .unwrap_or_default();
964 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
965 Some(last) if last.head == tip => Relation::Same,
966 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
967 _ => Relation::Diverged,
968 };
969 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
970 Some(assess(from, chosen.as_ref(), base_commit, relation))
971 }
972
973 async fn open_review(
976 repo: &Path,
977 branch: &str,
978 mut state: RunState,
979 roles: ResolvedRoles,
980 max_parallel: usize,
981 base_commit: String,
982 takeover: &crate::handover::Takeover,
983 ) -> Result<Self> {
984 let wait = repository_wait(&state, None);
985 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
986 if state.pr_recovery.is_none() {
987 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
988 }
989 let start = review_base(
992 repo,
993 &state.config.merge.remote,
994 &state.base_branch,
995 &base_commit,
996 branch,
997 )
998 .await;
999 let log = git::log_oneline(repo, &start, branch)
1000 .await
1001 .unwrap_or_default();
1002 let log_text = if log.trim().is_empty() {
1003 "(no commit messages)"
1004 } else {
1005 log.trim()
1006 };
1007 let mut instruction = format!(
1008 "Review the work already on branch `{branch}`. There is no task \
1009 statement: what the change claims to do is whatever its commits \
1010 say.\n\n{log_text}"
1011 );
1012 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1016 if let Some(task) = takeover
1017 .task_instruction
1018 .as_deref()
1019 .filter(|t| !t.trim().is_empty())
1020 && rec.reviewed_head.is_some()
1021 {
1022 instruction = format!(
1023 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1024 task.trim()
1025 );
1026 rec.criteria_inherited = true;
1027 }
1028 state.event("recovery", rec.summary());
1029 state.recovery = Some(rec);
1030 }
1031 if let Some(selected) = &state.pr_recovery {
1032 instruction = selected.instruction.clone();
1033 }
1034 state.task_instruction = takeover.task_instruction.clone();
1035 state.instruction = instruction;
1036 state.reviewed_commits = Some(
1037 git::subjects(repo, &start, branch)
1038 .await
1039 .unwrap_or_default(),
1040 );
1041
1042 let worktree = state.worktree_root().join("under-review");
1045 if let Some(parent) = worktree.parent() {
1046 tokio::fs::create_dir_all(parent).await.ok();
1047 }
1048 let path = worktree.to_string_lossy().to_string();
1049 git::git(repo, &["worktree", "add", &path, branch])
1050 .await
1051 .with_context(|| {
1052 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1053 })?;
1054
1055 let setup = match ensure_setup_config(&mut state, repo).await {
1056 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1057 Err(e) => Err(e),
1058 };
1059 if let Err(e) = setup {
1060 git::worktree_remove_clean(repo, &worktree).await.ok();
1061 return Err(e);
1062 }
1063 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1064 .await
1065 .unwrap_or(0);
1066 if commits == 0 {
1067 git::worktree_remove_clean(repo, &worktree).await.ok();
1068 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1069 }
1070 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1071 .await
1072 .map(|f| f.len())
1073 .unwrap_or(0);
1074 if files == 0
1075 && let (Ok(head_tree), Ok(base_tree)) = (
1076 git::tree_of(&worktree, "HEAD").await,
1077 git::tree_of(&worktree, &base_commit).await,
1078 )
1079 && head_tree == base_tree
1080 {
1081 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!(
1084 "`{branch}` at {} has a tree identical to base {}; this usually means \
1085 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1086 against `{}/{branch}`) rather than an empty change",
1087 short(&head),
1088 short(&base_commit),
1089 state.config.merge.remote
1090 );
1091 }
1092 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1093 .await
1094 .unwrap_or_default();
1095
1096 state.candidates.push(Candidate {
1097 index: 0,
1098 label: 'A',
1099 agent: EXISTING_BRANCH.to_owned(),
1102 branch: branch.to_owned(),
1103 worktree,
1104 summary: String::new(),
1105 stat,
1106 files,
1107 commits,
1108 empty: false,
1109 failed: None,
1110 verified_noop: None,
1111 duration_ms: 0,
1112 folded: false,
1113 });
1114 state.tally = Some(Tally {
1115 first_choice: BTreeMap::from([('A', 0)]),
1116 borda: BTreeMap::new(),
1117 winner: 'A',
1118 rankings: 0,
1119 unanimous_initial: false,
1120 deliberated: false,
1121 changed_votes: 0,
1122 unanimous_final: false,
1123 tie_break: None,
1124 judges: 0,
1128 present: 0,
1129 quorum: 0,
1130 met_quorum: true,
1131 uncontested: Some("review-only run: nothing competed".to_owned()),
1132 });
1133 state.status = RunStatus::Reviewing;
1134 state.prep_complete = Some(true);
1135 state.event(
1136 "start",
1137 format!(
1138 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1139 state.id
1140 ),
1141 );
1142 state.save()?;
1143 Ok(Self {
1144 state,
1145 roles,
1146 sem: Arc::new(Semaphore::new(max_parallel)),
1147 pause: Pause::new(),
1148 interrupt: Pause::new(),
1149 })
1150 }
1151
1152 pub fn resume(id: &str) -> Result<Self> {
1154 let state = RunState::load(id)?;
1155 crate::pr_head::ensure_owner(&state)?;
1156 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1157 bail!(
1158 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1159 );
1160 }
1161 if let Some(to) = &state.released_to {
1162 bail!(
1163 "run {} cannot be resumed: its worktree was released to run {}",
1164 state.short(),
1165 crate::run::short_of(to)
1166 );
1167 }
1168 let roles = state.config.resolve_roles()?;
1169 if let Some(id) = &state.config.graph.selected_implementer
1170 && state.candidates.iter().any(|c| c.agent != *id)
1171 {
1172 bail!(
1173 "replacement implementer pin does not match the saved candidates; task must stay held"
1174 );
1175 }
1176 let max_parallel = state.config.graph.max_parallel.max(1);
1177 Ok(Self {
1178 state,
1179 roles,
1180 sem: Arc::new(Semaphore::new(max_parallel)),
1181 pause: Pause::new(),
1182 interrupt: Pause::new(),
1183 })
1184 }
1185
1186 pub async fn execute(&mut self) -> Result<()> {
1193 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1196 || self.state.land_approval.is_some()
1197 || self.state.merge_decision.is_some()
1198 || !self.state.merge_repairs.is_empty()
1199 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1200 {
1201 let claim = FixClaim::acquire(&self.state.dir())?;
1202 let disk = RunState::load(&self.state.id)?;
1203 let checkpoint = |s: &RunState| {
1204 (
1205 s.merge_repairs.len(),
1206 s.merge_repairs
1207 .last()
1208 .map(|r| (r.question.clone(), r.phase)),
1209 s.land_approval
1210 .as_ref()
1211 .map(|a| (a.question.clone(), a.head.clone())),
1212 )
1213 };
1214 if checkpoint(&disk) != checkpoint(&self.state) {
1215 bail!(
1216 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1217 );
1218 }
1219 Some(claim)
1220 } else {
1221 None
1222 };
1223 let mut execution = self.claim_driver()?;
1224 let runner = &mut *execution.runner;
1225 let wait = runner.repository_wait();
1226 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1229 let result = match result {
1230 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1231 record_repository_block(&mut runner.state, &e);
1232 runner.park_here()?;
1233 Ok(())
1234 }
1235 other => other,
1236 };
1237 execution.driver.finish(&mut runner.state)?;
1238 let ended = if result.is_err() {
1239 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1240 } else {
1241 crate::notices::run_ended(&runner.state)
1242 };
1243 if let Some(notice) = ended {
1244 crate::notices::raise(notice);
1245 }
1246 result
1247 }
1248
1249 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1250 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1251 }
1252
1253 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1254 let wait = self.repository_wait();
1255 coordinate_operation(&mut self.state, cwd, &wait).await
1256 }
1257
1258 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1260 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1261 let disk = RunState::load(&self.state.id)?;
1262 crate::pr_head::ensure_owner(&disk)?;
1263 if disk.liveness(false) == crate::run::Liveness::Live {
1264 bail!("run already has a live driver; wait for its execution to end");
1265 }
1266 if let Some(to) = &disk.released_to {
1267 bail!(
1268 "run {} cannot continue: its worktree was released to run {}",
1269 self.state.short(),
1270 crate::run::short_of(to)
1271 );
1272 }
1273 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1274 let mut execution = DriverExecution {
1275 runner: self,
1276 driver,
1277 };
1278 let _owner = owner;
1281 let state = &mut execution.runner.state;
1282 state.parked = false;
1287 state.resource_wait = None;
1288 state.resource_block = None;
1289 state.clear_active();
1296 let pid = std::process::id();
1311 state.driver_pid = Some(pid);
1312 state.driver_started_at = crate::proc::process_started_at(pid);
1313 state.driver_exited = false;
1314 state.begin_pass();
1315 execution.driver.bind(state);
1316 state.save()?;
1317 execution.driver.record(state)?;
1318 Ok(execution)
1319 }
1320
1321 async fn execute_graph(&mut self) -> Result<()> {
1322 self.state.configure_recovery_resume()?;
1323 self.persist_candidates().await?;
1326 if self
1327 .state
1328 .merge_decision
1329 .as_ref()
1330 .is_some_and(|decision| !decision.closed)
1331 && self.state.status != RunStatus::Landing
1332 {
1333 if crate::merge_decision::resumable_hold(&self.state) {
1334 self.state.merge = Some(MergeOutcome {
1335 mode: MergeMode::Pr,
1336 ok: false,
1337 detail: self
1338 .state
1339 .merge_decision
1340 .as_ref()
1341 .unwrap()
1342 .target
1343 .pr
1344 .clone(),
1345 empty: false,
1346 });
1347 self.run_land().await?;
1348 self.settle_questions();
1349 return Ok(());
1350 }
1351 bail!(
1352 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1353 );
1354 }
1355 if self.state.config.graph.selected_implementer.is_some()
1356 && self.state.status == RunStatus::Blocked
1357 && self.state.tally.is_none()
1358 && self.state.candidate_preservation.is_empty()
1359 {
1360 bail!(
1361 "selected implementation is held; explicitly release the owning task to retry it"
1362 );
1363 }
1364
1365 if self.state.merge_repairs.last().is_some_and(|r| {
1366 matches!(
1367 r.phase,
1368 crate::run::MergeRepairPhase::Requested
1369 | crate::run::MergeRepairPhase::Fixing
1370 | crate::run::MergeRepairPhase::Reviewing
1371 )
1372 }) {
1373 if self.repair_merge_review().await? {
1374 self.run_land().await?;
1375 }
1376 self.settle_questions();
1377 return Ok(());
1378 }
1379 if self.state.pr_publication_pending {
1382 self.merge().await?;
1383 if self.resource_halt()? {
1384 return Ok(());
1385 }
1386 self.state.save()?;
1387 self.settle_questions();
1388 return Ok(());
1389 }
1390 if self.state.status == RunStatus::Stalled {
1403 if self.recover_stall().await? {
1404 self.finish_after_tally().await?;
1405 } else {
1406 self.state.save()?;
1408 }
1409 return Ok(());
1410 }
1411 if self.state.status == RunStatus::Landing {
1421 self.run_land().await?;
1422 self.settle_questions();
1427 return Ok(());
1428 }
1429 if self.state.github_text.is_some()
1434 && self.state.merge.is_none()
1435 && self.state.status == RunStatus::Gating
1436 {
1437 self.merge().await?;
1438 if self.resource_halt()? {
1439 return Ok(());
1440 }
1441 self.state.save()?;
1442 return Ok(());
1443 }
1444 if let Err(e) = self.prep().await {
1445 if e.is::<crate::repo_lock::ResourceBlocked>() {
1446 return Err(e);
1447 }
1448 self.state.status = RunStatus::Blocked;
1449 self.state
1450 .prep_failure
1451 .get_or_insert_with(|| format!("{e:#}"));
1452 self.state
1453 .event("prep", format!("preparation stopped: {e:#}"));
1454 self.state.save()?;
1455 return Err(e);
1456 }
1457 if self.park_here()? {
1458 return Ok(());
1459 }
1460 self.advise().await?;
1461 if self.park_here()? {
1462 return Ok(());
1463 }
1464 self.implement().await?;
1465 if self.park_here()? {
1466 return Ok(());
1467 }
1468 if self.state.status == RunStatus::VerifiedNoop {
1472 return Ok(());
1473 }
1474 self.judge().await?;
1475 if self.park_here()? {
1476 return Ok(());
1477 }
1478 self.deliberate().await?;
1479 if self.park_here()? {
1480 return Ok(());
1481 }
1482 self.vote().await?;
1483 if self.park_here()? {
1484 return Ok(());
1485 }
1486 self.tally()?;
1487 if self.state.status == RunStatus::Stalled {
1492 self.state.save()?;
1496 return Ok(());
1497 }
1498 self.finish_after_tally().await?;
1499 Ok(())
1500 }
1501
1502 fn stop_for_resource(&mut self) -> Result<()> {
1507 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1509 {
1510 self.state.status = RunStatus::Blocked;
1511 }
1512 self.state.save()
1513 }
1514
1515 fn resource_halt(&mut self) -> Result<bool> {
1517 if self.state.resource_block.is_none() {
1518 return Ok(false);
1519 }
1520 self.stop_for_resource()?;
1521 Ok(true)
1522 }
1523
1524 fn park_here(&mut self) -> Result<bool> {
1531 let parking = self.pause.parked() || self.interrupt.parked();
1532 if !parking && self.state.resource_block.is_some() {
1533 self.stop_for_resource()?;
1534 return Ok(true);
1535 }
1536 if !self.pause.parked() && !self.interrupt.parked() {
1541 return Ok(false);
1542 }
1543 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1544 let why = match cause.as_ref().map(ParkCause::label) {
1545 Some(reason) => format!(
1546 "parked after `{}` ({reason}) — resume to carry on from here",
1547 self.state.status.as_str()
1548 ),
1549 None => format!(
1550 "parked after `{}` — resume to carry on from here",
1551 self.state.status.as_str()
1552 ),
1553 };
1554 self.state.event("park", why);
1555 self.state
1556 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1557 self.state.parked = true;
1558 self.state.save()?;
1559 Ok(true)
1560 }
1561
1562 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1564 if self.state.prep_failure.is_none() {
1565 return Ok(());
1566 }
1567 let failed_at = self
1568 .state
1569 .events
1570 .iter()
1571 .rev()
1572 .find(|event| event.node == "prep")
1573 .map_or(self.state.updated_at, |event| event.at);
1574 anyhow::ensure!(
1575 approval_at > failed_at,
1576 "preparation retry requires a new explicit release after the failure"
1577 );
1578 self.state.prep_failure = None;
1579 self.state.prep_complete = Some(false);
1580 self.state
1581 .event("prep-retry", "operator approved retry of preparation");
1582 Ok(())
1583 }
1584
1585 pub fn on_pause(&mut self, pause: Pause) {
1587 self.pause = pause;
1588 }
1589
1590 pub fn watch_interrupt(&mut self, pause: Pause) {
1596 self.interrupt = pause;
1597 }
1598
1599 fn settle_questions(&mut self) {
1619 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1620 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1621 }
1622 }
1623
1624 async fn finish_after_tally(&mut self) -> Result<()> {
1627 self.graft().await?;
1630 if self.resource_halt()? {
1631 return Ok(());
1632 }
1633 self.fold_losers().await?;
1634 self.sync_to_base().await?;
1639 if self.state.status == RunStatus::AlreadyInBase {
1640 return Ok(());
1641 }
1642 self.review_loop().await?;
1643 if self.resource_halt()? {
1644 return Ok(());
1645 }
1646 self.sync_to_base().await?;
1647 if self.state.status == RunStatus::AlreadyInBase {
1648 return Ok(());
1649 }
1650 self.gate().await?;
1651 self.merge().await?;
1652 if self.resource_halt()? {
1653 return Ok(());
1654 }
1655 self.state.save()?;
1656 Ok(())
1657 }
1658
1659 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1666 let result = match ensure_setup_config(&mut self.state, repo).await {
1667 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1668 Err(e) => Err(e),
1669 };
1670 if let Err(e) = result {
1671 git::worktree_remove_clean(repo, wt).await.ok();
1672 if in_prep {
1673 self.rollback_prep(repo).await;
1674 }
1675 self.state.status = RunStatus::Blocked;
1676 self.state.event(
1677 "setup",
1678 format!("worktree setup failed in {}: {e:#}", wt.display()),
1679 );
1680 self.state.save()?;
1681 return Err(e);
1682 }
1683 Ok(())
1684 }
1685
1686 async fn rollback_prep(&mut self, repo: &Path) {
1688 let root = self.state.worktree_root();
1689 for c in std::mem::take(&mut self.state.candidates) {
1690 if matches!(
1691 git::worktree_remove_clean(repo, &c.worktree).await,
1692 Ok(true)
1693 ) {
1694 git::branch_delete(repo, &c.branch).await.ok();
1695 } else {
1696 self.state.candidates.push(c);
1697 }
1698 }
1699 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1700 for j in 1..=seats {
1701 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1702 .await
1703 .ok();
1704 }
1705 for k in 1..=self.state.config.graph.advisors {
1706 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1707 .await
1708 .ok();
1709 }
1710 }
1711
1712 async fn prep(&mut self) -> Result<()> {
1713 if let Some(reason) = &self.state.prep_failure {
1714 bail!(
1715 "candidate preparation previously failed: {reason}; correct the source before retrying"
1716 );
1717 }
1718 if self
1719 .state
1720 .prep_complete
1721 .unwrap_or(!self.state.candidates.is_empty())
1722 {
1723 return Ok(());
1724 }
1725 let repo = self.state.repo.clone();
1726 let _coordination = self.coordinate(&repo).await?;
1729 self.rollback_prep(&repo).await;
1730 if !self.state.candidates.is_empty() {
1731 bail!(
1732 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1733 );
1734 }
1735 self.state.prep_complete = Some(false);
1736 self.state.status = RunStatus::Prep;
1737 self.state.save()?;
1738 let repo = self.state.repo.clone();
1739 let base = self.state.base_commit.clone();
1740 let plan = refs::plan(&repo, &self.state.seeds).await?;
1741 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1742 let root = self.state.worktree_root();
1743 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1744
1745 let hooks_dir = self.state.dir().join("hooks");
1748 if self.state.config.blind.commit_msg_hook {
1749 std::fs::create_dir_all(&hooks_dir)
1750 .with_context(|| format!("create {}", hooks_dir.display()))?;
1751 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1752 let path = hooks_dir.join("commit-msg");
1753 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1754 make_executable(&path)?;
1755 git::acquire_worktree_config(&repo).await?;
1763 self.state.enabled_worktree_config = true;
1764 }
1765
1766 ensure_setup_config(&mut self.state, &repo).await?;
1769
1770 for (index, (spec, label)) in self
1771 .roles
1772 .implementers
1773 .clone()
1774 .into_iter()
1775 .zip(labels)
1776 .enumerate()
1777 {
1778 let branch = self.state.branch_for(label);
1779 let worktree = root.join(format!("cand-{label}"));
1780 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1781 if self.state.config.blind.commit_msg_hook {
1782 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1783 }
1784 git::local_exclude(&worktree, "/.magi/").await?;
1785 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1786 git::worktree_remove_clean(&repo, &worktree).await.ok();
1789 git::branch_delete(&repo, &branch).await.ok();
1790 self.rollback_prep(&repo).await;
1791 self.state.status = RunStatus::Blocked;
1792 self.state
1793 .event("prep", format!("worktree setup failed: {e:#}"));
1794 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1795 self.state.save()?;
1796 return Err(e);
1797 }
1798 self.state.candidates.push(Candidate {
1799 index,
1800 label,
1801 agent: spec.id.clone(),
1802 branch,
1803 worktree,
1804 summary: String::new(),
1805 stat: String::new(),
1806 files: 0,
1807 commits: 0,
1808 empty: false,
1809 failed: None,
1810 verified_noop: None,
1811 duration_ms: 0,
1812 folded: false,
1813 });
1814 self.state.save()?;
1815 for pick in &plan.picks {
1816 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1817 {
1818 self.state.status = RunStatus::Blocked;
1819 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1820 self.state
1821 .event("prep", format!("cannot apply referenced commit: {e}"));
1822 self.state.save()?;
1823 return Err(e);
1824 }
1825 }
1826 }
1827
1828 for j in 1..=self.roles.judges.len() {
1829 let wt = root.join(format!("judge-{j}"));
1830 if !wt.exists() {
1831 git::worktree_add_detached(&repo, &wt, &base).await?;
1832 self.setup_seat_worktree(&repo, &wt, true).await?;
1833 }
1834 }
1835
1836 if self.state.config.graph.advise {
1844 for k in 1..=self.state.config.graph.advisors {
1845 let wt = root.join(format!("advisor-{k}"));
1846 if !wt.exists() {
1847 git::worktree_add_detached(&repo, &wt, &base).await?;
1848 self.setup_seat_worktree(&repo, &wt, true).await?;
1849 }
1850 }
1851 }
1852
1853 let authors: Vec<&str> = self
1858 .roles
1859 .implementers
1860 .iter()
1861 .map(|a| a.id.as_str())
1862 .collect();
1863 let overlap: Vec<String> = self
1864 .roles
1865 .judges
1866 .iter()
1867 .enumerate()
1868 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1869 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1870 .collect();
1871 if !overlap.is_empty() {
1872 let note = format!(
1873 "{} also authored a candidate; blind, but the panel is less \
1874 independent than {} distinct agents would be",
1875 overlap.join(", "),
1876 self.roles.judges.len()
1877 );
1878 self.state.event("prep", note);
1879 }
1880
1881 self.state.event(
1882 "prep",
1883 format!(
1884 "{} candidates, {} judges, base {} ({})",
1885 self.state.candidates.len(),
1886 self.roles.judges.len(),
1887 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1888 self.state.base_branch
1889 ),
1890 );
1891 self.state.prep_complete = Some(true);
1892 self.state.status = RunStatus::Implementing;
1893 self.state.save()?;
1894 Ok(())
1895 }
1896
1897 async fn advise(&mut self) -> Result<()> {
1930 let implement_untouched = self
1931 .state
1932 .candidates
1933 .iter()
1934 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1935 if !self.state.config.graph.advise || self.state.advise_attempted {
1936 return Ok(());
1937 }
1938 if !implement_untouched {
1939 self.state.event(
1940 "advise",
1941 "skipping the design-deliberation stage: at least one \
1942 candidate already shows implementation progress, so this \
1943 run is past the point the stage exists to run before"
1944 .to_owned(),
1945 );
1946 self.state.advise_attempted = true;
1947 self.state.save()?;
1948 return Ok(());
1949 }
1950 let run_id = self.state.id.clone();
1951 let prompts = self.state.config.prompts.clone();
1952 let instruction = self.state.instruction.clone();
1953 let language = self.state.config.graph.language.clone();
1954 let root = self.state.worktree_root();
1955 let n = self.state.config.graph.advisors;
1956 let where_recorded = self.state.dir().join("run.json");
1957
1958 let seats = match self.state.config.advisors() {
1959 Ok(seats) if !seats.is_empty() => seats,
1960 Ok(_) => {
1961 self.state.event(
1962 "advise",
1963 format!(
1964 "[graph] advisors is 0; skipping the design-deliberation \
1965 stage and continuing without a synthesis brief (see {})",
1966 where_recorded.display()
1967 ),
1968 );
1969 self.state.advise_attempted = true;
1970 self.state.save()?;
1971 return Ok(());
1972 }
1973 Err(e) => {
1974 self.state.event(
1975 "advise",
1976 format!(
1977 "could not resolve advisor seats ({e:#}); continuing \
1978 without a design-deliberation brief (see {})",
1979 where_recorded.display()
1980 ),
1981 );
1982 self.state.advise_attempted = true;
1983 self.state.save()?;
1984 return Ok(());
1985 }
1986 };
1987
1988 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1989 let artifacts = agent::artifacts_dir(&self.state.dir());
1990 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1991
1992 let mut jobs = Vec::new();
1993 for (i, spec) in seats.iter().cloned().enumerate() {
1994 let seat_key = format!("advisor-{}", i + 1);
1995 let seat = self.seat(&seat_key, &spec.id);
1996 jobs.push(SeatJob {
1997 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1998 spec,
1999 seat,
2000 cwd: worktrees[i % worktrees.len()].clone(),
2001 timeout,
2002 allow_write: false,
2003 sessions: false,
2004 artifacts: artifacts.clone(),
2005 stem: seat_key,
2006 handover: None,
2007 });
2008 }
2009
2010 self.state.event(
2011 "advise",
2012 format!(
2013 "{} advisor seat(s) sketching a design in parallel",
2014 jobs.len()
2015 ),
2016 );
2017 let mut quota_losses = Vec::new();
2018 let cache = self.state.config.cache_dir();
2019 let ctx = WaveCtx {
2020 carry_seats: false,
2021 pauses: Some((&self.pause, &self.interrupt)),
2022 run: &run_id,
2023 node: "advise",
2024 prompts: &prompts,
2025 cache: cache.as_deref(),
2026 round: None,
2027 };
2028 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2029 let results = ask_json_wave::<Proposal>(
2030 jobs,
2031 Arc::clone(&self.sem),
2032 self.state.config.graph.retries,
2033 &advisor_roster,
2034 &ctx,
2035 &mut quota_losses,
2036 &mut self.state,
2037 &|p: &Proposal| p.validate(),
2038 )
2039 .await;
2040 if self.state.resource_block.is_some() {
2041 return self.stop_for_resource();
2042 }
2043 self.state.quota.extend(quota_losses);
2044
2045 let mut records = Vec::with_capacity(results.len());
2046 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2047 let agent_id = seat.agent.clone();
2048 self.state.seats.insert(seat.key.clone(), seat);
2049 match res {
2050 Ok((proposal, out)) => {
2051 self.state
2052 .event("advise", format!("advisor-{} proposed a design", i + 1));
2053 records.push(advise::AdvisorRecord::proposed(
2054 i + 1,
2055 agent_id,
2056 proposal,
2057 out.duration_ms,
2058 ));
2059 }
2060 Err(e) => {
2061 self.state.event(
2062 "advise",
2063 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2064 );
2065 records.push(advise::AdvisorRecord::failed(
2066 i + 1,
2067 agent_id,
2068 e.to_string(),
2069 ));
2070 }
2071 }
2072 }
2073
2074 let mut advice = advise::Advice {
2075 records,
2076 synthesis: None,
2077 };
2078 if advice.proposals().is_empty() {
2079 self.state.event(
2080 "advise",
2081 "no advisor produced a usable proposal; continuing without a \
2082 synthesis brief"
2083 .to_owned(),
2084 );
2085 } else {
2086 match self
2087 .synthesize_brief(
2088 &advice,
2089 &instruction,
2090 &language,
2091 &worktrees[0],
2092 &artifacts,
2093 &run_id,
2094 &prompts,
2095 cache.as_deref(),
2096 )
2097 .await
2098 {
2099 Ok(Some(text)) => {
2100 self.state.event(
2101 "advise",
2102 "synthesized a design brief for the implementer".to_owned(),
2103 );
2104 advice.synthesis = Some(text);
2105 }
2106 Ok(None) => {
2107 self.state.event(
2108 "advise",
2109 "the synthesis seat produced nothing usable; continuing \
2110 without a design brief"
2111 .to_owned(),
2112 );
2113 }
2114 Err(e) => {
2115 if e.is::<crate::repo_lock::ResourceBlocked>() {
2116 return Err(e);
2117 }
2118 self.state.event(
2119 "advise",
2120 format!("could not synthesize a design brief: {e:#}"),
2121 );
2122 }
2123 }
2124 }
2125 advise::apply_reflection(&mut advice);
2126
2127 self.state.advice = Some(advice);
2128 self.state.advise_attempted = true;
2129 self.state.save()?;
2130 Ok(())
2131 }
2132
2133 #[allow(clippy::too_many_arguments)]
2145 async fn synthesize_brief(
2146 &mut self,
2147 advice: &advise::Advice,
2148 instruction: &str,
2149 language: &str,
2150 cwd: &Path,
2151 artifacts: &Path,
2152 run_id: &str,
2153 prompts: &Prompts,
2154 cache: Option<&Path>,
2155 ) -> Result<Option<String>> {
2156 let chain = agent::pick_chain(
2157 &self.state.config.agents,
2158 self.state.config.roles.synthesizer.as_ref(),
2159 &agent::installed,
2160 "synthesizer",
2161 )?;
2162 let proposals = advice.proposals();
2163 let mut prompt = prompt::with_overlay(
2164 prompt::synthesize_brief(instruction, &proposals, language),
2165 prompts.overlay("advise"),
2166 );
2167 if cache.is_some() {
2168 prompt.push('\n');
2173 prompt.push_str(&prompt::build_cache_note("advise", false));
2174 }
2175 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2176 let mut last = None;
2181 for (n, spec) in chain.iter().enumerate() {
2182 if n > 0 {
2183 self.state
2184 .event("advise", format!("synthesis falling back to {}", spec.id));
2185 }
2186 let mut seat = self.seat("advise-synthesis", &spec.id);
2187 let wait = self.repository_wait();
2188 let outcome = wait
2189 .scope(agent::invoke(
2190 spec,
2191 &mut seat,
2192 &Invocation {
2193 scope: crate::agent::InvocationScope::Repository,
2194 coordination: self.state.config.coordination.as_ref(),
2195 fetched_base: None,
2196 cwd,
2197 prompt: &prompt,
2198 timeout,
2199 allow_write: false,
2200 unsandboxed: false,
2201 sessions: false,
2202 artifacts,
2203 stem: &if n == 0 {
2204 "advise-synthesis".to_owned()
2205 } else {
2206 format!("advise-synthesis-{}", spec.id)
2207 },
2208 run: run_id,
2209 node: "advise",
2210 cache_dir: None,
2211 attachments: &[],
2212 writable: &[],
2213 },
2214 ))
2215 .await;
2216 if outcome.is_ok() {
2217 self.state.seats.insert(seat.key.clone(), seat);
2218 }
2219 let advance = agent::chain_advances(&outcome);
2220 last = Some(outcome);
2221 if !advance {
2222 break;
2223 }
2224 }
2225 let out = last.expect("a chain holds at least one agent")?;
2228 if !out.usable() {
2229 return Ok(None);
2230 }
2231 let text =
2232 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2233 Ok((!text.trim().is_empty()).then_some(text))
2234 }
2235
2236 async fn implement(&mut self) -> Result<()> {
2239 self.persist_candidates().await?;
2240 let run_id = self.state.id.clone();
2245 let prompts = self.state.config.prompts.clone();
2246 let todo: Vec<usize> = self
2247 .state
2248 .candidates
2249 .iter()
2250 .enumerate()
2251 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2252 .map(|(i, _)| i)
2253 .collect();
2254 if todo.is_empty() {
2255 return self.after_implement();
2256 }
2257 self.state.status = RunStatus::Implementing;
2258
2259 let language = self.state.config.graph.language.clone();
2260 let github_language = self.state.config.graph.github_language.clone();
2261 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2262 let sessions = self.state.config.graph.sessions;
2263 let artifacts = agent::artifacts_dir(&self.state.dir());
2264 let brief = self
2268 .state
2269 .advice
2270 .as_ref()
2271 .and_then(|a| a.synthesis.as_deref())
2272 .map(str::to_owned);
2273 let attachments = self.state.attachments.clone();
2274
2275 let mut jobs = Vec::new();
2276 for &i in &todo {
2277 let (index, label, worktree) = {
2278 let c = &self.state.candidates[i];
2279 (c.index, c.label, c.worktree.clone())
2280 };
2281 let spec = self.roles.implementers[index].clone();
2282 let seat_key = format!("impl-{label}");
2283 let seat = self.seat(&seat_key, &spec.id);
2284 let instruction = seeded_instruction(&self.state);
2285 jobs.push(SeatJob {
2286 spec,
2287 seat,
2288 prompt: prompt::implement_in(
2289 &instruction,
2290 &worktree.to_string_lossy(),
2291 &language,
2292 &github_language,
2293 brief.as_deref(),
2294 &attachments,
2295 ),
2296 cwd: worktree,
2297 timeout,
2298 allow_write: true,
2299 sessions,
2300 artifacts: artifacts.clone(),
2301 stem: format!("impl-{label}"),
2302 handover: None,
2303 });
2304 }
2305
2306 self.state.event(
2307 "implement",
2308 format!("{} candidates in parallel", jobs.len()),
2309 );
2310 let mut sent = jobs.clone();
2316 let cache = self.state.config.cache_dir();
2317 let ctx = WaveCtx {
2318 carry_seats: false,
2319 pauses: Some((&self.pause, &self.interrupt)),
2320 run: &run_id,
2321 node: "implement",
2322 prompts: &prompts,
2323 cache: cache.as_deref(),
2324 round: None,
2325 };
2326 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2327 if !wave_blocked(&results) {
2328 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2329 .await;
2330 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2331 .await?;
2332 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2333 .await;
2334 }
2335 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2336
2337 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2338 && results
2339 .iter()
2340 .any(|(_, _, out)| FailClass::of(out).is_some());
2341 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2342 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2343 continue;
2344 }
2345 let seat_key = seat.key.clone();
2346 let agent = seat.agent.clone();
2355 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2356 self.state.seats.insert(seat.key.clone(), seat);
2357 let label = self.state.candidates[i].label;
2358
2359 let (summary, duration, failed, verified_claim) = match out {
2360 AgentOutcome::Ok(o) => {
2361 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2362 let failed = (!o.usable()).then(|| {
2363 if o.timed_out {
2364 "agent timed out".to_owned()
2365 } else {
2366 format!(
2367 "agent exited with {}",
2368 crate::notices::exit_status(o.exit_code)
2369 )
2370 }
2371 });
2372 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2373 (text, o.duration_ms, failed, verified_claim)
2374 }
2375 AgentOutcome::Dropped(o) => {
2381 let why = o
2382 .dropped
2383 .as_ref()
2384 .map(|d| d.why.as_str())
2385 .unwrap_or("the CLI ended the stream without delivering its answer");
2386 (
2387 String::new(),
2388 o.duration_ms,
2389 Some(format!("the CLI dropped the stream ({why})")),
2390 None,
2391 )
2392 }
2393 AgentOutcome::Quota(o) => {
2394 self.state.quota.push(QuotaLoss {
2395 seat: seat_key,
2396 node: "implement".to_owned(),
2397 at: Timestamp::now(),
2398 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2399 });
2400 (
2401 String::new(),
2402 o.duration_ms,
2403 Some("rate limited (quota); produced no change".to_owned()),
2404 None,
2405 )
2406 }
2407 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2408 (String::new(), 0, Some(e), None)
2409 }
2410 };
2411
2412 let c = &mut self.state.candidates[i];
2415 if !exhausted_the_fallback_chain {
2416 c.agent = agent;
2417 }
2418 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2419 c.duration_ms = duration;
2420 c.failed = failed;
2421 c.verified_noop = verified_claim;
2422 self.state
2423 .candidate_preservation
2424 .insert(label, "awaiting persistence".to_owned());
2425 }
2426 self.state.save()?;
2427 self.persist_candidates().await?;
2428
2429 if resource_blocked {
2430 return self.stop_for_resource();
2431 }
2432 if selected_failed {
2433 self.state.status = RunStatus::Blocked;
2434 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2435 self.state.save()?;
2436 bail!(
2437 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2438 );
2439 }
2440 self.after_implement()
2441 }
2442
2443 async fn persist_candidates(&mut self) -> Result<()> {
2445 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2446 for label in labels {
2447 let i = self
2448 .state
2449 .candidates
2450 .iter()
2451 .position(|c| c.label == label)
2452 .context("pending candidate is missing")?;
2453 let c = self.state.candidates[i].clone();
2454 let result = async {
2455 let wait = crate::repo_lock::WaitPolicy::new(
2456 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2457 || false,
2458 );
2459 let _coordination =
2460 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2461 if let Err(e) = git::snapshot_uncommitted(
2464 &c.worktree,
2465 &self
2466 .state
2467 .dir()
2468 .join("artifacts")
2469 .join(format!("candidate-{label}-recovery")),
2470 )
2471 .await
2472 {
2473 self.state.event(
2474 "preserve",
2475 format!("candidate {label} snapshot incomplete: {e:#}"),
2476 );
2477 }
2478 let rescued = git::rescue_commit(
2479 &c.worktree,
2480 &format!("magi: candidate {label} (uncommitted work)"),
2481 )
2482 .await?;
2483 self.state.note_withheld("implement", &rescued.withheld);
2484 let base = &self.state.base_commit;
2485 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2486 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2487 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2488 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2489 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2490 if c.empty && commits > 0 && !patch.trim().is_empty()
2491 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2492 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2493 || self.state.tally.is_some())
2494 {
2495 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2499 }
2500 let c = &mut self.state.candidates[i];
2501 c.commits = commits;
2502 c.stat = stat;
2503 c.files = files;
2504 c.empty = commits == 0 || patch.trim().is_empty();
2505 if !c.empty {
2506 c.failed = None;
2507 c.verified_noop = None;
2508 }
2509 self.state.event(
2510 "preserve",
2511 format!("candidate {label}: {files} files, {commits} commits"),
2512 );
2513 Ok::<_, anyhow::Error>(())
2514 }
2515 .await;
2516 match result {
2517 Ok(()) => {
2518 self.state.candidate_preservation.remove(&label);
2519 }
2520 Err(e) => {
2521 let reason = format!("candidate {label} preservation failed: {e:#}");
2522 self.state.event("preserve", &reason);
2523 self.state.candidate_preservation.insert(label, reason);
2524 }
2525 }
2526 if self.state.candidate_preservation.is_empty()
2527 && self.state.status == RunStatus::Blocked
2528 && self.state.candidates.iter().all(|c| c.failed.is_none())
2529 {
2530 self.state.status = RunStatus::Implementing;
2534 self.state.event(
2535 "preserve",
2536 "candidate persistence recovered; implementation can continue",
2537 );
2538 }
2539 self.state.save()?;
2540 }
2541 if !self.state.candidate_preservation.is_empty() {
2542 self.state.status = RunStatus::Blocked;
2543 self.state.save()?;
2544 bail!(
2545 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2546 );
2547 }
2548 Ok(())
2549 }
2550
2551 async fn resume_undelivered(
2579 &mut self,
2580 results: &mut [(usize, SeatState, AgentOutcome)],
2581 sent: &[SeatJob],
2582 prompts: &Prompts,
2583 run_id: &str,
2584 ) {
2585 for (wi, seat, out) in results.iter_mut() {
2586 let Some(dropped) = (match &*out {
2587 AgentOutcome::Dropped(o) => o.dropped.clone(),
2588 _ => None,
2589 }) else {
2590 continue;
2591 };
2592 let Some(job) = sent.get(*wi) else { continue };
2593 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2595 self.state.event(
2596 "implement",
2597 format!(
2598 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2599 work is in the tree",
2600 seat.key, dropped.output_tokens, dropped.why
2601 ),
2602 );
2603 continue;
2604 }
2605 if !has_context(&job.spec, seat, job.sessions) {
2613 self.state.event(
2614 "implement",
2615 format!(
2616 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2617 is no session left to resume",
2618 seat.key, dropped.output_tokens, dropped.why
2619 ),
2620 );
2621 continue;
2622 }
2623 self.state.event(
2624 "implement",
2625 format!(
2626 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2627 conversation",
2628 seat.key, dropped.output_tokens, dropped.why
2629 ),
2630 );
2631 let mut retry = job.clone();
2632 retry.seat = seat.clone();
2633 retry.prompt = prompt::resume_after_drop(&dropped.why);
2634 retry.timeout = retry_budget(job.timeout, true);
2635 retry.stem = format!("{}-resume", job.stem);
2636 let cache = self.state.config.cache_dir();
2637 let ctx = WaveCtx {
2638 carry_seats: false,
2639 pauses: Some((&self.pause, &self.interrupt)),
2640 run: run_id,
2641 node: "implement",
2642 prompts,
2643 cache: cache.as_deref(),
2644 round: None,
2645 };
2646 let (resumed_seat, resumed) =
2647 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2648 *seat = resumed_seat;
2649 *out = resumed;
2650 }
2651 }
2652
2653 async fn resume_seat_handovers(
2718 &mut self,
2719 results: &mut [(usize, SeatState, AgentOutcome)],
2720 sent: &mut [SeatJob],
2721 prompts: &Prompts,
2722 run_id: &str,
2723 ) -> Result<()> {
2724 let instruction = seeded_instruction(&self.state);
2725 let language = self.state.config.graph.language.clone();
2726 let github_language = self.state.config.graph.github_language.clone();
2727 let brief = self
2728 .state
2729 .advice
2730 .as_ref()
2731 .and_then(|a| a.synthesis.as_deref())
2732 .map(str::to_owned);
2733 let attachments = self.state.attachments.clone();
2734 for (wi, seat, out) in results.iter_mut() {
2735 let others: BTreeSet<String> = sent
2738 .iter()
2739 .enumerate()
2740 .filter(|(j, _)| j != wi)
2741 .map(|(_, j)| j.spec.id.clone())
2742 .collect();
2743 let Some(job) = sent.get_mut(*wi) else {
2744 continue;
2745 };
2746 let start = self
2751 .roles
2752 .implementer_roster
2753 .iter()
2754 .position(|s| s.id == job.spec.id)
2755 .unwrap_or(0);
2756 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2757 let mut fallback_attempt = 0usize;
2758 let mut prev: Option<FailClass> = None;
2759 while let Some(cur) = FailClass::of(&*out) {
2760 if !should_hand_over(prev.as_ref(), &cur) {
2761 break;
2762 }
2763 let Some(next) =
2764 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2765 .cloned()
2766 else {
2767 break;
2768 };
2769 tried.insert(next.id.clone());
2770 fallback_attempt += 1;
2771
2772 let coordination = self.coordinate(&job.cwd).await?;
2773 if let Ok(r) = git::rescue_commit(
2774 &job.cwd,
2775 &format!(
2776 "magi: candidate {} (uncommitted work before {} fallback)",
2777 seat.key,
2778 if cur == FailClass::Quota {
2779 "quota"
2780 } else {
2781 "handover"
2782 }
2783 ),
2784 )
2785 .await
2786 {
2787 self.state.note_withheld("implement", &r.withheld);
2788 }
2789 drop(coordination);
2790
2791 record_handover(
2792 &mut self.state,
2793 "implement",
2794 &seat.key,
2795 &seat.agent,
2796 &next.id,
2797 &cur,
2798 &fail_reason(&*out),
2799 );
2800 prev = Some(cur.clone());
2801
2802 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2803 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2804 job.spec = next.clone();
2812 let mut retry = job.clone();
2813 retry.seat = new_seat;
2814 retry.prompt = prompt::implement_in(
2815 &instruction,
2816 &job.cwd.to_string_lossy(),
2817 &language,
2818 &github_language,
2819 brief.as_deref(),
2820 &attachments,
2821 );
2822 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2823 let cache = self.state.config.cache_dir();
2824 let ctx = WaveCtx {
2825 carry_seats: false,
2826 pauses: Some((&self.pause, &self.interrupt)),
2827 run: run_id,
2828 node: "implement",
2829 prompts,
2830 cache: cache.as_deref(),
2831 round: None,
2832 };
2833 let (fallback_seat, fallback_out) = run_one(
2834 retry,
2835 Arc::clone(&self.sem),
2836 &ctx,
2837 &mut self.state,
2838 fallback_attempt,
2839 )
2840 .await;
2841 *seat = fallback_seat;
2842 *out = fallback_out;
2843 }
2844 }
2845 Ok(())
2846 }
2847
2848 async fn resume_unconfirmed_commands(
2872 &mut self,
2873 results: &mut [(usize, SeatState, AgentOutcome)],
2874 sent: &[SeatJob],
2875 prompts: &Prompts,
2876 run_id: &str,
2877 ) {
2878 for (wi, seat, out) in results.iter_mut() {
2879 let AgentOutcome::Ok(o) = &*out else {
2880 continue;
2881 };
2882 if !has_unconfirmed_command(&o.commands) {
2883 continue;
2884 }
2885 let Some(job) = sent.get(*wi) else { continue };
2886 if !has_context(&job.spec, seat, job.sessions) {
2887 self.state.event(
2888 "implement",
2889 format!(
2890 "{}: the reply named a command whose own CLI never confirmed the exit \
2891 status of, but there is no session left to resume",
2892 seat.key
2893 ),
2894 );
2895 continue;
2896 }
2897 self.state.event(
2898 "implement",
2899 format!(
2900 "{}: the reply named a command whose own CLI never confirmed the exit \
2901 status of; resuming the conversation",
2902 seat.key
2903 ),
2904 );
2905 let mut retry = job.clone();
2906 retry.seat = seat.clone();
2907 retry.prompt = prompt::resume_incomplete(
2908 "a command in your last reply had no confirmed exit status",
2909 );
2910 retry.timeout = retry_budget(job.timeout, true);
2911 retry.stem = format!("{}-confirm", job.stem);
2912 let cache = self.state.config.cache_dir();
2913 let ctx = WaveCtx {
2914 carry_seats: false,
2915 pauses: Some((&self.pause, &self.interrupt)),
2916 run: run_id,
2917 node: "implement",
2918 prompts,
2919 cache: cache.as_deref(),
2920 round: None,
2921 };
2922 let (resumed_seat, resumed) =
2923 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2924 *seat = resumed_seat;
2925 *out = resumed;
2926 }
2927 }
2928
2929 async fn continue_fix_report(
2950 &mut self,
2951 mut seat: SeatState,
2952 parse_err: String,
2953 job: &SeatJob,
2954 prompts: &Prompts,
2955 run_id: &str,
2956 round: usize,
2957 ) -> (
2958 SeatState,
2959 Option<FixReport>,
2960 Option<String>,
2961 ContinuationRecord,
2962 ) {
2963 let mut last_err = parse_err;
2964 let mut cumulative_wait_ms = 0u64;
2965 let mut attempts = 0usize;
2966 loop {
2967 if !has_context(&job.spec, &seat, job.sessions) {
2968 self.state.event(
2969 "fix",
2970 format!(
2971 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2972 session left to resume into"
2973 ),
2974 );
2975 let outcome = if attempts == 0 {
2976 ContinuationOutcome::NoSession
2977 } else {
2978 ContinuationOutcome::Exhausted
2979 };
2980 return (
2981 seat,
2982 None,
2983 Some(format!("unparsable fix report: {last_err}")),
2984 ContinuationRecord {
2985 attempts,
2986 cumulative_wait_ms,
2987 outcome,
2988 },
2989 );
2990 }
2991 if attempts >= MAX_FIX_CONTINUATIONS {
2992 self.state.event(
2993 "fix",
2994 format!(
2995 "round {round}: fixer's reply still had no adoption report after \
2996 {attempts} continuation(s) ({last_err}); giving up"
2997 ),
2998 );
2999 return (
3000 seat,
3001 None,
3002 Some(format!(
3003 "unparsable fix report after {attempts} continuation(s): {last_err}"
3004 )),
3005 ContinuationRecord {
3006 attempts,
3007 cumulative_wait_ms,
3008 outcome: ContinuationOutcome::Exhausted,
3009 },
3010 );
3011 }
3012 attempts += 1;
3013 self.state.event(
3014 "fix",
3015 format!(
3016 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3017 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3018 ),
3019 );
3020 let mut retry = job.clone();
3021 retry.seat = seat.clone();
3022 retry.prompt = prompt::resume_incomplete(&last_err);
3023 retry.timeout = retry_budget(job.timeout, true);
3024 retry.stem = format!("{}-continue{attempts}", job.stem);
3025 let cache = self.state.config.cache_dir();
3026 let ctx = WaveCtx {
3027 carry_seats: false,
3028 pauses: Some((&self.pause, &self.interrupt)),
3029 run: run_id,
3030 node: "fix",
3031 prompts,
3032 cache: cache.as_deref(),
3033 round: Some(round),
3034 };
3035 let (resumed_seat, resumed_out) = run_one(
3036 retry,
3037 Arc::clone(&self.sem),
3038 &ctx,
3039 &mut self.state,
3040 attempts,
3041 )
3042 .await;
3043 seat = resumed_seat;
3044 match resumed_out {
3045 AgentOutcome::Ok(o) => {
3046 cumulative_wait_ms += o.duration_ms;
3047 match verdict::extract_json::<FixReport>(&o.text) {
3048 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3049 self.state.event(
3050 "fix",
3051 format!(
3052 "round {round}: fixer's adoption report recovered after \
3053 {attempts} continuation(s)"
3054 ),
3055 );
3056 return (
3057 seat,
3058 Some(report),
3059 None,
3060 ContinuationRecord {
3061 attempts,
3062 cumulative_wait_ms,
3063 outcome: ContinuationOutcome::Resumed,
3064 },
3065 );
3066 }
3067 Ok(_) => {
3075 last_err = "the reply parsed, but it reported a command whose own CLI \
3076 never confirmed an exit status"
3077 .to_owned();
3078 }
3079 Err(e) => last_err = e.to_string(),
3080 }
3081 }
3082 AgentOutcome::Quota(o) => {
3083 cumulative_wait_ms += o.duration_ms;
3084 self.state.quota.push(QuotaLoss {
3085 seat: seat.key.clone(),
3086 node: "fix".to_owned(),
3087 at: Timestamp::now(),
3088 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3089 });
3090 self.state.event(
3091 "fix",
3092 format!(
3093 "round {round}: continuation rate limited (quota); not retrying now"
3094 ),
3095 );
3096 return (
3097 seat,
3098 None,
3099 Some("rate limited (quota) while recovering the fix report".to_owned()),
3100 ContinuationRecord {
3101 attempts,
3102 cumulative_wait_ms,
3103 outcome: ContinuationOutcome::QuotaLost,
3104 },
3105 );
3106 }
3107 AgentOutcome::Dropped(o) => {
3108 cumulative_wait_ms += o.duration_ms;
3109 let why = o
3110 .dropped
3111 .as_ref()
3112 .map(|d| d.why.as_str())
3113 .unwrap_or("the CLI ended the stream without delivering its answer");
3114 last_err = format!("the CLI dropped the stream ({why})");
3115 }
3116 AgentOutcome::Failed(e) => last_err = e,
3117 AgentOutcome::ResourceBlocked(e) => {
3120 return (
3121 seat,
3122 None,
3123 Some(format!("the continuation was not started: {e}")),
3124 ContinuationRecord {
3125 attempts: attempts.saturating_sub(1),
3126 cumulative_wait_ms,
3127 outcome: ContinuationOutcome::NoSession,
3128 },
3129 );
3130 }
3131 }
3132 }
3133 }
3134
3135 fn after_implement(&mut self) -> Result<()> {
3136 if !self.state.candidate_preservation.is_empty() {
3137 bail!("candidate preservation incomplete; judging is unsafe");
3138 }
3139 if self.state.leaks.is_empty() {
3141 let cfg = self.state.config.blind.clone();
3142 let mut leaks = Vec::new();
3143 for c in &self.state.candidates {
3144 let Some(patch) =
3145 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3146 else {
3147 continue;
3148 };
3149 leaks.extend(blind::scan(
3150 &format!("candidate {} patch", c.label),
3151 &patch,
3152 &cfg.vendor_tokens,
3153 ));
3154 }
3155 if !leaks.is_empty() {
3156 let summary = leaks
3157 .iter()
3158 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3159 .collect::<Vec<_>>()
3160 .join(", ");
3161 match cfg.on_leak {
3162 LeakPolicy::Fail => {
3163 self.state.status = RunStatus::Failed;
3164 self.state
3165 .event("blind", format!("vendor text in a patch: {summary}"));
3166 self.state.leaks = leaks;
3167 self.state.save()?;
3168 self.settle_questions();
3169 bail!(
3170 "blind.on_leak = \"fail\" and vendor text reached a \
3171 judged patch: {summary}"
3172 );
3173 }
3174 LeakPolicy::Redact => self.state.event(
3175 "blind",
3176 format!("redacting vendor text for judging: {summary}"),
3177 ),
3178 LeakPolicy::Warn => self.state.event(
3179 "blind",
3180 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3181 ),
3182 }
3183 self.state.leaks = leaks;
3184 }
3185 }
3186
3187 if self.state.viable().is_empty() {
3188 if self.state.all_candidates_verified_noop() {
3189 self.state.status = RunStatus::VerifiedNoop;
3200 self.state.save()?;
3201 self.settle_questions();
3202 return Ok(());
3203 }
3204 self.state.status = RunStatus::Failed;
3205 self.state.save()?;
3206 self.settle_questions();
3207 bail!("no candidate produced a change; nothing to judge");
3208 }
3209 self.state.status = RunStatus::Judging;
3210 self.state.save()?;
3211 Ok(())
3212 }
3213
3214 async fn judge(&mut self) -> Result<()> {
3217 let run_id = self.state.id.clone();
3222 let prompts = self.state.config.prompts.clone();
3223 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3224 return Ok(());
3225 }
3226 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3227 if viable.len() == 1 {
3228 self.state.judge_skipped = true;
3235 self.state.event(
3236 "judge",
3237 format!(
3238 "only candidate {} produced a change; judging skipped",
3239 viable[0].label
3240 ),
3241 );
3242 self.state.save()?;
3243 return Ok(());
3244 }
3245 self.state.status = RunStatus::Judging;
3246
3247 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3248 let language = self.state.config.graph.language.clone();
3249 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3250 let sessions = self.state.config.graph.sessions;
3251 let artifacts = agent::artifacts_dir(&self.state.dir());
3252 let root = self.state.worktree_root();
3253 let base_short = short(&self.state.base_commit);
3254
3255 let mut jobs = Vec::new();
3256 let mut orders = Vec::new();
3257 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3258 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3259 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3260 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3261 let seat_key = format!("judge-{}", j + 1);
3262 let seat = self.seat(&seat_key, &spec.id);
3263 jobs.push(SeatJob {
3264 prompt: prompt::judge(
3265 &self.state.instruction,
3266 &views,
3267 self.roles.judges.len(),
3268 &base_short,
3269 &language,
3270 ),
3271 spec,
3272 seat,
3273 cwd: root.join(format!("judge-{}", j + 1)),
3274 timeout,
3275 allow_write: false,
3276 sessions,
3277 artifacts: artifacts.clone(),
3278 stem: format!("judge-{}", j + 1),
3279 handover: None,
3280 });
3281 }
3282
3283 self.state.event(
3284 "judge",
3285 format!(
3286 "{} judges ranking {} candidates blind",
3287 jobs.len(),
3288 viable.len()
3289 ),
3290 );
3291 let labels_for_check = labels.clone();
3292 let mut quota_losses = Vec::new();
3293 let cache = self.state.config.cache_dir();
3294 let ctx = WaveCtx {
3295 carry_seats: false,
3296 pauses: Some((&self.pause, &self.interrupt)),
3297 run: &run_id,
3298 node: "judge",
3299 prompts: &prompts,
3300 cache: cache.as_deref(),
3301 round: None,
3302 };
3303 let results = ask_json_wave::<Ranking>(
3304 jobs,
3305 Arc::clone(&self.sem),
3306 self.state.config.graph.retries,
3307 &self.roles.judge_roster,
3308 &ctx,
3309 &mut quota_losses,
3310 &mut self.state,
3311 &move |r: &Ranking| r.validate(&labels_for_check),
3312 )
3313 .await;
3314 if self.state.resource_block.is_some() {
3315 return self.stop_for_resource();
3316 }
3317 self.state.quota.extend(quota_losses);
3318
3319 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3320 let agent_id = seat.agent.clone();
3321 self.state.seats.insert(seat.key.clone(), seat);
3322 let mut record = Judgement {
3323 judge: j + 1,
3324 seat: format!("judge-{}", j + 1),
3325 agent: agent_id,
3326 ranking: Vec::new(),
3327 reasons: BTreeMap::new(),
3328 confidence: None,
3329 strengths: BTreeMap::new(),
3330 order: orders[j].clone(),
3331 failed: None,
3332 duration_ms: 0,
3333 };
3334 match res {
3335 Ok((ranking, out)) => {
3336 record.ranking = ranking.normalized();
3337 record.reasons = ranking.reasons;
3338 record.strengths = ranking.strengths;
3339 record.confidence = ranking.confidence;
3340 record.duration_ms = out.duration_ms;
3341 self.state.event(
3342 "judge",
3343 format!(
3344 "judge {} ranked {}",
3345 j + 1,
3346 record.ranking.iter().collect::<String>()
3347 ),
3348 );
3349 }
3350 Err(e) => {
3351 record.failed = Some(e.to_string());
3352 self.state
3353 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3354 }
3355 }
3356 self.state.judgements.push(record);
3357 self.state.save()?;
3358 }
3359 Ok(())
3360 }
3361
3362 async fn deliberate(&mut self) -> Result<()> {
3365 let run_id = self.state.id.clone();
3370 let prompts = self.state.config.prompts.clone();
3371 if !self.state.deliberation.is_empty() {
3372 return Ok(());
3373 }
3374 let tops: Vec<char> = self
3375 .state
3376 .judgements
3377 .iter()
3378 .filter_map(|j| j.ranking.first().copied())
3379 .collect();
3380 let rounds = self.state.config.graph.deliberate_rounds;
3381 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3382 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3383 self.state.event(
3384 "deliberate",
3385 format!("judges agreed on {} outright; no deliberation", tops[0]),
3386 );
3387 }
3388 self.state.status = RunStatus::Voting;
3389 self.state.save()?;
3390 return Ok(());
3391 }
3392
3393 self.state.status = RunStatus::Deliberating;
3394 self.state.event(
3395 "deliberate",
3396 format!(
3397 "split: first choices were {} — opening {rounds} round(s)",
3398 tops.iter().collect::<String>()
3399 ),
3400 );
3401
3402 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3403 let language = self.state.config.graph.language.clone();
3404 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3405 let sessions = self.state.config.graph.sessions;
3406 let artifacts = agent::artifacts_dir(&self.state.dir());
3407 let root = self.state.worktree_root();
3408 let base_short = short(&self.state.base_commit);
3409
3410 for round in 1..=rounds {
3414 let mut turns: Vec<DeliberationTurn> = Vec::new();
3415 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3416 if self.state.judgements[j].failed.is_some() {
3417 continue;
3418 }
3419 let seat_key = format!("judge-{}", j + 1);
3420 let spec = self.occupant(&seat_key, spec);
3421 let mut seat = self.seat(&seat_key, &spec.id);
3422 let transcript = self.transcript(&turns, j);
3423 let build = |context: Option<&str>| {
3424 prompt::deliberate(
3425 &self.state.instruction,
3426 context,
3427 &transcript,
3428 round,
3429 rounds,
3430 &language,
3431 )
3432 };
3433 let block = self.candidate_block(&viable, &base_short);
3434 let full = build(Some(&block));
3435 let text = if has_context(&spec, &seat, sessions) {
3436 build(None)
3437 } else {
3438 full.clone()
3439 };
3440 let job = SeatJob {
3441 spec,
3442 seat: seat.clone(),
3443 prompt: text,
3444 cwd: root.join(format!("judge-{}", j + 1)),
3445 timeout,
3446 allow_write: false,
3447 sessions,
3448 artifacts: artifacts.clone(),
3449 stem: format!("delib-{round}-judge-{}", j + 1),
3450 handover: Some(full),
3451 };
3452 let cache = self.state.config.cache_dir();
3453 let ctx = WaveCtx {
3454 carry_seats: false,
3455 pauses: Some((&self.pause, &self.interrupt)),
3456 run: &run_id,
3457 node: "deliberate",
3458 prompts: &prompts,
3459 cache: cache.as_deref(),
3460 round: None,
3461 };
3462 let mut losses = Vec::new();
3466 let mut results = ask_wave_with::<String>(
3467 vec![job],
3468 Arc::clone(&self.sem),
3469 0,
3470 &self.roles.judge_roster,
3471 &ctx,
3472 &mut losses,
3473 &mut self.state,
3474 &|text: &str| {
3475 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3476 },
3477 )
3478 .await;
3479 self.state.quota.extend(losses);
3480 let (updated, res, _) = results.pop().expect("one job in, one result out");
3481 seat = updated;
3482 let agent_id = seat.agent.clone();
3483 self.state.seats.insert(seat.key.clone(), seat);
3484 let body = match res {
3485 Ok((body, _)) => body,
3486 Err(e) => {
3489 self.state
3490 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3491 continue;
3492 }
3493 };
3494 let tentative = verdict::extract_json::<Position>(&body)
3495 .ok()
3496 .and_then(|p| p.tentative)
3497 .and_then(|s| s.trim().chars().next())
3498 .map(|c| c.to_ascii_uppercase());
3499 self.state.event(
3500 "deliberate",
3501 format!(
3502 "round {round}: judge {} now favours {}",
3503 j + 1,
3504 tentative.map_or("—".to_owned(), |c| c.to_string())
3505 ),
3506 );
3507 turns.push(DeliberationTurn {
3508 judge: j + 1,
3509 agent: agent_id,
3510 body: blind::sanitize_prose(&body, &self.state.config.blind),
3511 tentative,
3512 });
3513 }
3514 self.state
3515 .deliberation
3516 .push(DeliberationRound { round, turns });
3517 self.state.save()?;
3518 }
3519
3520 self.state.status = RunStatus::Voting;
3521 self.state.save()?;
3522 Ok(())
3523 }
3524
3525 async fn vote(&mut self) -> Result<()> {
3528 let run_id = self.state.id.clone();
3533 let prompts = self.state.config.prompts.clone();
3534 if !self.state.votes.is_empty() {
3535 return Ok(());
3536 }
3537 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3538 if viable.len() == 1 {
3539 return Ok(());
3540 }
3541 self.state.status = RunStatus::Voting;
3542
3543 let language = self.state.config.graph.language.clone();
3544 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3545 let sessions = self.state.config.graph.sessions;
3546 let artifacts = agent::artifacts_dir(&self.state.dir());
3547 let root = self.state.worktree_root();
3548 let base_short = short(&self.state.base_commit);
3549 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3550
3551 let mut jobs = Vec::new();
3552 let mut seats_at = Vec::new();
3553 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3554 if self
3555 .state
3556 .judgements
3557 .get(j)
3558 .is_some_and(|r| r.failed.is_some())
3559 {
3560 continue;
3561 }
3562 let seat_key = format!("judge-{}", j + 1);
3563 let spec = self.occupant(&seat_key, spec);
3564 let seat = self.seat(&seat_key, &spec.id);
3565 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3566 let text = if has_context(&spec, &seat, sessions) {
3567 prompt::final_vote(&viable, &language)
3568 } else {
3569 full.clone()
3570 };
3571 jobs.push(SeatJob {
3572 spec,
3573 seat,
3574 prompt: text,
3575 cwd: root.join(format!("judge-{}", j + 1)),
3576 timeout,
3577 allow_write: false,
3578 sessions,
3579 artifacts: artifacts.clone(),
3580 stem: format!("vote-judge-{}", j + 1),
3581 handover: Some(full),
3582 });
3583 seats_at.push(j);
3584 }
3585
3586 self.state.event(
3587 "vote",
3588 format!(
3589 "collecting {} final votes one by one, privately",
3590 jobs.len()
3591 ),
3592 );
3593 let allowed = viable.clone();
3594 let mut quota_losses = Vec::new();
3595 let cache = self.state.config.cache_dir();
3596 let ctx = WaveCtx {
3597 carry_seats: false,
3598 pauses: Some((&self.pause, &self.interrupt)),
3599 run: &run_id,
3600 node: "vote",
3601 prompts: &prompts,
3602 cache: cache.as_deref(),
3603 round: None,
3604 };
3605 let results = ask_json_wave::<FinalVote>(
3606 jobs,
3607 Arc::clone(&self.sem),
3608 self.state.config.graph.retries,
3609 &self.roles.judge_roster,
3610 &ctx,
3611 &mut quota_losses,
3612 &mut self.state,
3613 &move |v: &FinalVote| match v.label() {
3614 Some(c) if allowed.contains(&c) => Ok(()),
3615 other => bail!("vote {other:?} is not one of {allowed:?}"),
3616 },
3617 )
3618 .await;
3619 if self.state.resource_block.is_some() {
3620 return self.stop_for_resource();
3621 }
3622 self.state.quota.extend(quota_losses);
3623
3624 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3625 let agent_id = seat.agent.clone();
3626 self.state.seats.insert(seat.key.clone(), seat);
3627 let initial = self
3628 .state
3629 .judgements
3630 .get(j)
3631 .and_then(|r| r.ranking.first().copied());
3632 let mut record = VoteRecord {
3633 judge: j + 1,
3634 agent: agent_id,
3635 vote: None,
3636 reason: String::new(),
3637 changed: false,
3638 };
3639 match res {
3640 Ok((v, _)) => {
3641 record.vote = v.label();
3642 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3643 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3644 self.state.event(
3645 "vote",
3646 format!(
3647 "judge {} voted {}{}",
3648 j + 1,
3649 record.vote.unwrap_or('?'),
3650 if record.changed { " (changed)" } else { "" }
3651 ),
3652 );
3653 }
3654 Err(e) => {
3655 self.state
3656 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3657 }
3658 }
3659 self.state.votes.push(record);
3660 self.state.save()?;
3661 }
3662 Ok(())
3663 }
3664
3665 fn tally(&mut self) -> Result<()> {
3668 if self.state.tally.is_some() {
3669 return Ok(());
3670 }
3671 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3672 let tops: Vec<char> = self
3673 .state
3674 .judgements
3675 .iter()
3676 .filter_map(|j| j.ranking.first().copied())
3677 .collect();
3678 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3679
3680 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3683 let mut cast: Vec<char> = Vec::new();
3684 for (i, j) in self.state.judgements.iter().enumerate() {
3685 let vote = self
3686 .state
3687 .votes
3688 .iter()
3689 .find(|v| v.judge == i + 1)
3690 .and_then(|v| v.vote)
3691 .or_else(|| j.ranking.first().copied());
3692 if let Some(v) = vote {
3693 *first_choice.entry(v).or_insert(0) += 1;
3694 cast.push(v);
3695 }
3696 }
3697
3698 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3699 for j in &self.state.judgements {
3700 let n = j.ranking.len();
3701 for (pos, label) in j.ranking.iter().enumerate() {
3702 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3703 }
3704 }
3705
3706 let best = first_choice.values().copied().max().unwrap_or(0);
3707 let mut leaders: Vec<char> = first_choice
3708 .iter()
3709 .filter(|(_, v)| **v == best)
3710 .map(|(k, _)| *k)
3711 .collect();
3712 let mut tie_break = None;
3713 if leaders.len() > 1 {
3714 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3715 let borda_leaders: Vec<char> = leaders
3716 .iter()
3717 .copied()
3718 .filter(|l| borda[l] == top_borda)
3719 .collect();
3720 tie_break = Some(if borda_leaders.len() == 1 {
3721 format!(
3722 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3723 leaders.len()
3724 )
3725 } else {
3726 format!(
3727 "{} way tie on both first-choice votes and Borda points, broken by label order",
3728 leaders.len()
3729 )
3730 });
3731 leaders = borda_leaders;
3732 leaders.sort_unstable();
3733 }
3734 let winner = *leaders
3735 .first()
3736 .or(viable.first())
3737 .context("no candidate to declare a winner from")?;
3738
3739 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3740 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3741 let deliberated = !self.state.deliberation.is_empty();
3742
3743 let quota_seats: std::collections::BTreeSet<&str> =
3747 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3748 let mut present = 0usize;
3749 for (i, j) in self.state.judgements.iter().enumerate() {
3750 if quota_seats.contains(j.seat.as_str()) {
3751 continue;
3752 }
3753 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3754 let voted = self
3755 .state
3756 .votes
3757 .iter()
3758 .any(|v| v.judge == i + 1 && v.vote.is_some());
3759 if ranked || voted {
3760 present += 1;
3761 }
3762 }
3763 let needs_quorum = viable.len() > 1;
3769 let judges_total = if needs_quorum {
3770 self.roles.judges.len()
3771 } else {
3772 0
3773 };
3774 let quorum = if needs_quorum {
3775 judges_total / 2 + 1
3776 } else {
3777 0
3778 };
3779 let met_quorum = !needs_quorum || present >= quorum;
3780 let uncontested = (!needs_quorum).then(|| {
3781 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3782 });
3783
3784 self.state.event(
3785 "tally",
3786 match &uncontested {
3787 Some(reason) => format!("winner {winner} — {reason}"),
3788 None => format!(
3789 "winner {winner} — votes {} | initial {} | {} changed | \
3790 {present}/{judges_total} judges{}",
3791 first_choice
3792 .iter()
3793 .map(|(k, v)| format!("{k}:{v}"))
3794 .collect::<Vec<_>>()
3795 .join(" "),
3796 if unanimous_initial {
3797 "unanimous"
3798 } else {
3799 "split"
3800 },
3801 changed_votes,
3802 if met_quorum {
3803 String::new()
3804 } else {
3805 format!(" — below quorum ({quorum} required)")
3806 },
3807 ),
3808 },
3809 );
3810 if !met_quorum {
3811 self.state.event(
3812 "stall",
3813 format!(
3814 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3815 the run stops here, resumable"
3816 ),
3817 );
3818 }
3819 self.state.tally = Some(Tally {
3820 first_choice,
3821 borda,
3822 winner,
3823 rankings: tops.len(),
3824 unanimous_initial,
3825 deliberated,
3826 changed_votes,
3827 unanimous_final,
3828 tie_break,
3829 judges: judges_total,
3830 present,
3831 quorum,
3832 met_quorum,
3833 uncontested,
3834 });
3835 self.state.status = if met_quorum {
3836 RunStatus::Reviewing
3837 } else {
3838 RunStatus::Stalled
3839 };
3840 self.state.save()?;
3841 Ok(())
3842 }
3843
3844 #[allow(clippy::too_many_lines)]
3865 async fn recover_stall(&mut self) -> Result<bool> {
3866 let run_id = self.state.id.clone();
3871 let prompts = self.state.config.prompts.clone();
3872 let quota_seats: BTreeSet<&str> =
3877 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3878 let absent: Vec<String> = self
3879 .state
3880 .judgements
3881 .iter()
3882 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3883 .map(|j| j.seat.clone())
3884 .collect();
3885 if absent.is_empty() {
3886 return Ok(false);
3887 }
3888 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3889 if viable.len() <= 1 {
3890 return Ok(false);
3891 }
3892 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3893 let language = self.state.config.graph.language.clone();
3894 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3895 let sessions = self.state.config.graph.sessions;
3896 let artifacts = agent::artifacts_dir(&self.state.dir());
3897 let root = self.state.worktree_root();
3898 let base_short = short(&self.state.base_commit);
3899 let candidates: Vec<Candidate> = viable.clone();
3900
3901 let mut positions: Vec<usize> = absent
3903 .iter()
3904 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3905 .collect();
3906 if positions.is_empty() {
3907 return Ok(false);
3908 }
3909 positions.sort_unstable();
3910 positions.dedup();
3911
3912 let mut judge_jobs = Vec::new();
3914 for &j in &positions {
3915 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3916 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3917 let seat_key = format!("judge-{}", j + 1);
3918 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3919 let seat = self.seat(&seat_key, &spec.id);
3920 judge_jobs.push(SeatJob {
3921 spec,
3922 seat,
3923 prompt: prompt::judge(
3924 &self.state.instruction,
3925 &views,
3926 self.roles.judges.len(),
3927 &base_short,
3928 &language,
3929 ),
3930 cwd: root.join(seat_key),
3931 timeout,
3932 allow_write: false,
3933 sessions,
3934 artifacts: artifacts.clone(),
3935 stem: format!("judge-{}-recover", j + 1),
3936 handover: None,
3937 });
3938 }
3939
3940 let labels_for_check = labels.clone();
3941 let mut judge_losses = Vec::new();
3942 let retries = self.state.config.graph.retries;
3943 let cache = self.state.config.cache_dir();
3944 let ctx = WaveCtx {
3945 carry_seats: false,
3946 pauses: Some((&self.pause, &self.interrupt)),
3947 run: &run_id,
3948 node: "judge",
3949 prompts: &prompts,
3950 cache: cache.as_deref(),
3951 round: None,
3952 };
3953 let results = ask_json_wave::<Ranking>(
3954 judge_jobs,
3955 Arc::clone(&self.sem),
3956 retries,
3957 &self.roles.judge_roster,
3958 &ctx,
3959 &mut judge_losses,
3960 &mut self.state,
3961 &move |r: &Ranking| r.validate(&labels_for_check),
3962 )
3963 .await;
3964 if self.state.resource_block.is_some() {
3965 self.stop_for_resource()?;
3966 return Ok(false);
3967 }
3968
3969 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3971 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3972 let agent_id = seat.agent.clone();
3973 self.state.seats.insert(seat.key.clone(), seat);
3974 let record = &mut self.state.judgements[j];
3975 match res {
3976 Ok((ranking, out)) => {
3977 record.agent = agent_id;
3978 record.ranking = ranking.normalized();
3979 record.reasons = ranking.reasons;
3980 record.strengths = ranking.strengths;
3981 record.confidence = ranking.confidence;
3982 record.failed = None;
3983 record.duration_ms = out.duration_ms;
3984 recovered.insert(j);
3985 self.state.event(
3986 "recover",
3987 format!("judge {} ranked again after the limit", j + 1),
3988 );
3989 }
3990 Err(e) => {
3991 self.state
3992 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3993 }
3994 }
3995 }
3996
3997 let mut vote_jobs = Vec::new();
3999 let mut vote_pos: Vec<usize> = Vec::new();
4000 for &j in &recovered {
4001 let seat_key = format!("judge-{}", j + 1);
4002 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003 let seat = self.seat(&seat_key, &spec.id);
4004 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4005 let text = if has_context(&spec, &seat, sessions) {
4006 prompt::final_vote(&labels, &language)
4007 } else {
4008 full.clone()
4009 };
4010 vote_jobs.push(SeatJob {
4011 spec,
4012 seat,
4013 prompt: text,
4014 cwd: root.join(seat_key),
4015 timeout,
4016 allow_write: false,
4017 sessions,
4018 artifacts: artifacts.clone(),
4019 stem: format!("vote-judge-{}-recover", j + 1),
4020 handover: Some(full),
4021 });
4022 vote_pos.push(j);
4023 }
4024 let allowed = labels.clone();
4025 let mut vote_losses = Vec::new();
4026 let vote_retries = self.state.config.graph.retries;
4027 let vote_cache = self.state.config.cache_dir();
4028 let ctx = WaveCtx {
4029 carry_seats: false,
4030 pauses: Some((&self.pause, &self.interrupt)),
4031 run: &run_id,
4032 node: "vote",
4033 prompts: &prompts,
4034 cache: vote_cache.as_deref(),
4035 round: None,
4036 };
4037 let votes = ask_json_wave::<FinalVote>(
4038 vote_jobs,
4039 Arc::clone(&self.sem),
4040 vote_retries,
4041 &self.roles.judge_roster,
4042 &ctx,
4043 &mut vote_losses,
4044 &mut self.state,
4045 &move |v: &FinalVote| match v.label() {
4046 Some(c) if allowed.contains(&c) => Ok(()),
4047 other => bail!("vote {other:?} is not one of {allowed:?}"),
4048 },
4049 )
4050 .await;
4051 if self.state.resource_block.is_some() {
4052 self.stop_for_resource()?;
4053 return Ok(false);
4054 }
4055 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4056 let agent_id = seat.agent.clone();
4057 self.state.seats.insert(seat.key.clone(), seat);
4058 match res {
4059 Ok((v, _)) => {
4060 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4061 rec.vote = v.label();
4062 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4063 } else {
4064 self.state.votes.push(VoteRecord {
4065 judge: j + 1,
4066 agent: agent_id,
4067 vote: v.label(),
4068 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4069 changed: false,
4070 });
4071 }
4072 self.state.event(
4073 "recover",
4074 format!("judge {} voted again after the limit", j + 1),
4075 );
4076 }
4077 Err(e) => {
4078 self.state
4079 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4080 }
4081 }
4082 }
4083
4084 let recovered_keys: BTreeSet<String> = recovered
4088 .iter()
4089 .map(|&j| format!("judge-{}", j + 1))
4090 .collect();
4091 self.state
4092 .quota
4093 .retain(|q| !recovered_keys.contains(&q.seat));
4094 for loss in judge_losses.into_iter().chain(vote_losses) {
4098 if recovered_keys.contains(&loss.seat) {
4099 continue;
4100 }
4101 self.state.quota.retain(|q| q.seat != loss.seat);
4102 self.state.quota.push(loss);
4103 }
4104
4105 self.state.tally = None;
4107 self.tally()?;
4108 Ok(self
4109 .state
4110 .tally
4111 .as_ref()
4112 .map(|t| t.met_quorum)
4113 .unwrap_or(false))
4114 }
4115
4116 async fn graft(&mut self) -> Result<()> {
4133 let Err(e) = self.graft_inner().await else {
4134 return Ok(());
4135 };
4136 if e.is::<crate::repo_lock::ResourceBlocked>() {
4137 return Err(e);
4138 }
4139 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4140 self.state.status = RunStatus::Blocked;
4141 self.state.event("graft", format!("blocked: {e:#}"));
4142 self.state.save()?;
4143 return Err(e);
4144 }
4145 if let Some(rec) = self.state.graft.clone()
4150 && !rec.outcome.settled()
4151 {
4152 if let Some(winner) = self.state.winner().cloned() {
4153 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4154 if re.is::<crate::repo_lock::ResourceBlocked>() {
4155 return Err(re);
4156 }
4157 self.state.status = RunStatus::Blocked;
4158 self.state.event("graft", format!("blocked: {re:#}"));
4159 self.state.save()?;
4160 return Err(re);
4161 }
4162 }
4163 if let Some(g) = self.state.graft.as_mut() {
4164 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4165 }
4166 }
4167 self.state.event("graft", format!("skipped: {e:#}"));
4168 if let Err(e) = self.state.save() {
4169 tracing::warn!("save after graft: {e:#}");
4170 }
4171 Ok(())
4172 }
4173
4174 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4177 let _coordination = self.coordinate(wt).await?;
4178 crate::recovery::restore_repair(wt, pre)
4179 .await
4180 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4181 }
4182
4183 fn graft_skipped(&mut self, why: &str) {
4185 let message = format!("skipped: {why}");
4186 if !self
4187 .state
4188 .events
4189 .iter()
4190 .any(|e| e.node == "graft" && e.message == message)
4191 {
4192 self.state.event("graft", message);
4193 }
4194 }
4195
4196 async fn graft_inner(&mut self) -> Result<()> {
4197 if let Some(rec) = self.state.graft.clone()
4200 && !rec.outcome.settled()
4201 {
4202 if self.state.reviews.is_empty()
4206 && let Some(winner) = self.state.winner().cloned()
4207 {
4208 self.graft_restore(&winner.worktree, &rec.pre).await?;
4209 }
4210 if rec.outcome == GraftOutcome::WaitingForRestore {
4211 self.state.graft = None;
4212 self.state.save()?;
4213 } else {
4214 if let Some(g) = self.state.graft.as_mut() {
4215 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4216 }
4217 self.state.event(
4218 "graft",
4219 "rolled back: the previous round was interrupted; the winner is untouched",
4220 );
4221 self.state.save()?;
4222 return Ok(());
4223 }
4224 }
4225 let plan = match graft::plan(&self.state) {
4226 Ok(plan) => plan,
4227 Err(skip) => {
4228 if skip != graft::Skip::AlreadyDone {
4229 self.graft_skipped(skip.reason());
4230 self.state.save()?;
4231 }
4232 return Ok(());
4233 }
4234 };
4235 let Some(winner) = self.state.winner().cloned() else {
4236 return Ok(());
4237 };
4238 let wt = winner.worktree.clone();
4239 if !git::is_clean(&wt).await? {
4240 self.graft_skipped("the winner's worktree has uncommitted changes");
4241 self.state.save()?;
4242 return Ok(());
4243 }
4244 let pre = git::rev_parse(&wt, "HEAD").await?;
4245 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4246 else {
4247 self.graft_skipped("the winner's diff could not be measured");
4248 self.state.save()?;
4249 return Ok(());
4250 };
4251 let limit = graft::limit(winner_lines);
4252 self.state.graft = Some(GraftRecord {
4255 pre: pre.clone(),
4256 post: None,
4257 lines: None,
4258 limit: Some(limit),
4259 agent: None,
4260 outcome: GraftOutcome::Running,
4261 requested: graft::requested(&plan),
4262 adoption: None,
4263 adoption_error: None,
4264 });
4265 self.state.event(
4266 "graft",
4267 format!(
4268 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4269 plan.strengths
4270 .iter()
4271 .map(|(l, _, _)| l.to_string())
4272 .collect::<Vec<_>>()
4273 .join(", "),
4274 plan.winner
4275 ),
4276 );
4277 self.state.save()?;
4278
4279 let fix_prompt = prompt::graft(
4280 &self.state.instruction,
4281 plan.winner,
4282 &plan.strengths,
4283 limit,
4284 &self.state.config.graph.language,
4285 );
4286 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4287 let sessions = self.state.config.graph.sessions;
4288 let artifacts = agent::artifacts_dir(&self.state.dir());
4289 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4290 let (_, seat, out) = self
4291 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4292 SeatJob {
4293 prompt: fix_prompt.clone(),
4294 spec,
4295 seat,
4296 cwd: wt.clone(),
4297 timeout,
4298 allow_write: true,
4299 sessions,
4300 artifacts: artifacts.clone(),
4301 stem: "graft".to_owned(),
4302 handover: None,
4303 }
4304 })
4305 .await;
4306 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4307 self.state.graft.as_mut().expect("graft record").outcome =
4310 GraftOutcome::WaitingForRestore;
4311 return self.stop_for_resource();
4312 }
4313 let mut why: Option<String> = None;
4314 match &out {
4315 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4316 why = Some("the fixer produced nothing usable".to_owned());
4317 }
4318 AgentOutcome::Ok(_) => {}
4319 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4320 AgentOutcome::Quota(o) => {
4321 self.state.quota.push(QuotaLoss {
4322 seat: seat.key.clone(),
4323 node: "graft".to_owned(),
4324 at: Timestamp::now(),
4325 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4326 });
4327 why = Some("rate limited (quota)".to_owned());
4328 }
4329 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4330 why = Some(format!(
4331 "the fixer failed: {}",
4332 e.lines()
4333 .next()
4334 .unwrap_or("")
4335 .chars()
4336 .take(160)
4337 .collect::<String>()
4338 ));
4339 }
4340 }
4341 let agent_id = seat.agent.clone();
4342 self.state.seats.insert(seat.key.clone(), seat);
4343
4344 let mut post = None;
4345 let mut lines = None;
4346 let _coordination = self.coordinate(&wt).await?;
4347 if why.is_none() {
4348 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4349 if let Ok(r) = &rescued {
4350 self.state.note_withheld("graft", &r.withheld);
4351 }
4352 let head = git::rev_parse(&wt, "HEAD").await?;
4353 let dirty = match &rescued {
4357 Ok(r) => git::status_porcelain(&wt)
4358 .await
4359 .map(|st| {
4360 st.lines().any(|l| {
4361 !r.withheld
4362 .iter()
4363 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4364 })
4365 })
4366 .unwrap_or(true),
4367 Err(_) => true,
4368 };
4369 if dirty {
4370 why = Some("the fixer left changes that could not be committed".to_owned());
4371 } else if head == pre {
4372 why = Some("the fixer changed nothing".to_owned());
4373 } else {
4374 lines = git::changed_lines(&wt, &pre, &head).await;
4375 if graft::within_budget(lines, limit) {
4376 post = Some(head);
4377 } else {
4378 why = Some(format!(
4379 "{} changed lines against a limit of {limit}",
4380 lines.map_or_else(
4381 || "an unmeasurable number of".to_owned(),
4382 |l| l.to_string()
4383 )
4384 ));
4385 }
4386 }
4387 }
4388 let (adoption, adoption_error) = match &out {
4389 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4390 _ => (None, None),
4391 };
4392 let requested = graft::requested(&plan);
4393 let outcome = match why {
4394 None => {
4395 self.state.event(
4396 "graft",
4397 format!(
4398 "applied: {} changed lines (limit {limit})",
4399 lines.unwrap_or(0)
4400 ),
4401 );
4402 GraftOutcome::Applied
4403 }
4404 Some(why) => {
4405 crate::recovery::restore_repair(&wt, &pre)
4408 .await
4409 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4410 self.state.event(
4411 "graft",
4412 format!("rolled back: {why}; the winner is untouched"),
4413 );
4414 GraftOutcome::RolledBack(why)
4415 }
4416 };
4417 self.state.graft = Some(GraftRecord {
4418 pre,
4419 post,
4420 lines,
4421 limit: Some(limit),
4422 agent: Some(agent_id),
4423 outcome,
4424 requested,
4425 adoption,
4426 adoption_error,
4427 });
4428 self.state.save()?;
4429 Ok(())
4430 }
4431
4432 async fn fold_losers(&mut self) -> Result<()> {
4435 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4436 return Ok(());
4437 };
4438 if !self.state.candidate_preservation.is_empty() {
4439 bail!("candidate preservation incomplete; refusing cleanup");
4440 }
4441 let repo = self.state.repo.clone();
4442 let _coordination = self.coordinate(&repo).await?;
4443 let mut folded = Vec::new();
4444 for i in 0..self.state.candidates.len() {
4445 let c = &self.state.candidates[i];
4446 if c.label == winner || c.folded {
4447 continue;
4448 }
4449 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4450 remove_preserved_worktree(&repo, &wt).await?;
4451 if git::branch_exists(&repo, &branch).await?
4452 && !git::branch_delete(&repo, &branch).await?
4453 {
4454 bail!("candidate branch removal refused");
4455 }
4456 self.state.candidates[i].folded = true;
4457 self.state.save()?;
4458 folded.push(label.to_string());
4459 }
4460 let root = self.state.worktree_root();
4462 for j in 1..=self.roles.judges.len() {
4463 let wt = root.join(format!("judge-{j}"));
4464 if wt.exists() {
4465 remove_preserved_worktree(&repo, &wt).await?;
4466 }
4467 }
4468 if self.state.config.graph.advise {
4471 for k in 1..=self.state.config.graph.advisors {
4472 let wt = root.join(format!("advisor-{k}"));
4473 if wt.exists() {
4474 remove_preserved_worktree(&repo, &wt).await?;
4475 }
4476 }
4477 }
4478 if !folded.is_empty() {
4479 self.state
4480 .event("fold", format!("folded candidates {}", folded.join(", ")));
4481 self.state.save()?;
4482 }
4483 Ok(())
4484 }
4485
4486 async fn sync_to_base(&mut self) -> Result<()> {
4526 if self.state.status == RunStatus::AlreadyInBase {
4527 return Ok(());
4528 }
4529 let conflicted = self
4530 .state
4531 .base_sync
4532 .as_ref()
4533 .is_some_and(|s| s.conflict.is_some());
4534 let Some(winner) = self.state.winner().cloned() else {
4535 return Ok(());
4536 };
4537
4538 let repo = self.state.repo.clone();
4539 let remote = self.state.config.merge.remote.clone();
4540 let base_branch = self.state.base_branch.clone();
4541 let tracking = format!("{remote}/{base_branch}");
4542
4543 git::fetch(&repo, &remote, &base_branch).await.ok();
4544 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4549 return Ok(());
4550 };
4551
4552 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4553 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4554 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4555
4556 if (behind > 0 || conflicted || head == tip)
4565 && self
4566 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4567 .await?
4568 {
4569 return Ok(());
4570 }
4571 if conflicted {
4572 return Ok(());
4573 }
4574
4575 if behind == 0 {
4576 if let Some(from) = self
4585 .state
4586 .rebase_fixes
4587 .iter()
4588 .rev()
4589 .find_map(|r| r.from.clone())
4590 && from != head
4591 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4592 .await
4593 .is_ok_and(|o| o.ok())
4594 {
4595 git::sync_to_head(&winner.worktree).await?;
4596 }
4597 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4603 if let Some(why) = &conflict {
4604 self.state.status = RunStatus::Blocked;
4605 self.state.event("land", why.clone());
4606 }
4607 self.state.base_sync = Some(BaseSync {
4608 tip,
4609 behind: 0,
4610 attempts,
4611 conflict,
4612 already_in: None,
4613 });
4614 self.state.save()?;
4615 return Ok(());
4616 }
4617
4618 if attempts >= BASE_SYNC_ROUNDS {
4619 let why = format!(
4620 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4621 rebase(s); rebasing again would only race it",
4622 winner.branch
4623 );
4624 self.state.status = RunStatus::Blocked;
4625 self.state.base_sync = Some(BaseSync {
4626 tip,
4627 behind,
4628 attempts,
4629 conflict: Some(why.clone()),
4630 already_in: None,
4631 });
4632 self.state.event("land", why);
4633 self.state.save()?;
4634 return Ok(());
4635 }
4636
4637 self.state.event(
4638 "land",
4639 format!(
4640 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4641 winner.branch
4642 ),
4643 );
4644 self.state.save()?;
4645
4646 let branch_tracking = format!("{remote}/{}", winner.branch);
4652 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4653 let remote_tip = if self.state.task_pr_branch().is_none()
4654 && matches!(&fetched_branch, Ok(o) if o.ok())
4655 {
4656 git::rev_parse(&repo, &branch_tracking).await.ok()
4657 } else {
4658 None
4659 };
4660 if let Some(theirs) = &remote_tip
4664 && !git::is_ancestor(&repo, theirs, &head).await
4665 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4666 .await
4667 .is_ok_and(|missing| missing.is_empty())
4668 {
4669 let why = format!(
4670 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4671 them",
4672 short(theirs),
4673 winner.branch
4674 );
4675 self.state.status = RunStatus::Blocked;
4676 self.state.base_sync = Some(BaseSync {
4677 tip,
4678 behind,
4679 attempts,
4680 conflict: Some(why.clone()),
4681 already_in: None,
4682 });
4683 self.state.event("land", why);
4684 self.state.save()?;
4685 return Ok(());
4686 }
4687
4688 let scratch = self.state.dir().join("base-sync");
4689 let rebased = match crate::rebase::rebase_with_fixer(
4690 &mut self.state,
4691 &scratch,
4692 &winner.branch,
4693 &tracking,
4694 )
4695 .await
4696 {
4697 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4698 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4699 Err(e) => Err(e),
4700 };
4701 let attempts = attempts + 1;
4702 match rebased {
4703 Ok(None) => {
4704 git::sync_to_head(&winner.worktree).await?;
4708 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4709 let mut conflict = None;
4710 if let Some(pinned) = &remote_tip {
4711 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4712 match pushed {
4713 Ok(o) if o.ok() => self.state.event(
4714 "land",
4715 format!("pushed rebased {} to {remote}", winner.branch),
4716 ),
4717 Ok(o) => {
4718 conflict = Some(format!(
4719 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4720 winner.branch,
4721 short(pinned),
4722 o.stderr.chars().take(600).collect::<String>()
4723 ));
4724 }
4725 Err(e) => {
4726 conflict = Some(format!(
4727 "rebased {} locally but could not push it: {e:#}",
4728 winner.branch
4729 ));
4730 }
4731 }
4732 }
4733 if let Some(why) = &conflict {
4734 self.state.status = RunStatus::Blocked;
4735 self.state.event("land", why.clone());
4736 }
4737 self.state.base_sync = Some(BaseSync {
4738 tip: tip.clone(),
4739 behind: 0,
4740 attempts,
4741 conflict,
4742 already_in: None,
4743 });
4744 self.state
4745 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4746 }
4747 Ok(Some(conflict)) => {
4748 let why = format!(
4749 "{} conflicts with {tracking} and did not rebase: {}",
4750 winner.branch,
4751 conflict.chars().take(600).collect::<String>()
4752 );
4753 self.state.status = RunStatus::Blocked;
4754 self.state.base_sync = Some(BaseSync {
4755 tip,
4756 behind,
4757 attempts,
4758 conflict: Some(why.clone()),
4759 already_in: None,
4760 });
4761 self.state.event("land", why);
4762 }
4763 Err(e) => {
4764 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4765 self.state.status = RunStatus::Blocked;
4766 self.state.base_sync = Some(BaseSync {
4767 tip,
4768 behind,
4769 attempts,
4770 conflict: Some(why.clone()),
4771 already_in: None,
4772 });
4773 self.state.event("land", why);
4774 }
4775 }
4776 self.state.save()?;
4777 Ok(())
4778 }
4779
4780 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4786 if self.state.task_pr_branch().is_some() {
4789 return None;
4790 }
4791 let repo = self.state.repo.clone();
4792 let remote = self.state.config.merge.remote.clone();
4793 let fetched = git::fetch(&repo, &remote, branch).await;
4794 if !matches!(&fetched, Ok(o) if o.ok()) {
4795 return None;
4796 }
4797 let branch_tracking = format!("{remote}/{branch}");
4798 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4799 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4800 return None;
4801 }
4802 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4803 .await
4804 .is_ok_and(|missing| missing.is_empty())
4805 {
4806 return Some(format!(
4807 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4808 them",
4809 short(&theirs)
4810 ));
4811 }
4812 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4813 Ok(o) if o.ok() => {
4814 self.state
4815 .event("land", format!("pushed rebased {branch} to {remote}"));
4816 None
4817 }
4818 Ok(o) => Some(format!(
4819 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4820 someone may have pushed): {}",
4821 short(&theirs),
4822 o.stderr.chars().take(600).collect::<String>()
4823 )),
4824 Err(e) => Some(format!(
4825 "{branch} is rebased locally but could not be pushed: {e:#}"
4826 )),
4827 }
4828 }
4829
4830 async fn settle_already_in(
4847 &mut self,
4848 branch: &str,
4849 tip: &str,
4850 head: &str,
4851 attempts: usize,
4852 behind: usize,
4853 ) -> Result<bool> {
4854 let repo = self.state.repo.clone();
4855 let remote = self.state.config.merge.remote.clone();
4856 let start = self.state.base_commit.clone();
4857 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4858 Ok(Some(e)) => e,
4859 Ok(None) => return Ok(false),
4860 Err(e) => {
4861 tracing::warn!("already-in-base check for {branch}: {e:#}");
4862 return Ok(false);
4863 }
4864 };
4865 let mut verified = vec![head.to_owned()];
4866 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4867 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4868 if matches!(&fetched, Ok(o) if o.ok())
4869 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4870 && theirs != head
4871 {
4872 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4873 Ok(Some(_)) => verified.push(theirs),
4874 _ => return Ok(false),
4875 }
4876 }
4877 let base_branch = self.state.base_branch.clone();
4878 let message = format!(
4879 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4880 land",
4881 evidence.names(),
4882 evidence.proof.as_str()
4883 );
4884 let closed =
4885 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4886 .await;
4887 match closed {
4888 Ok(Ok(url)) => self
4889 .state
4890 .event("land", format!("closed {url}: superseded on {base_branch}")),
4891 Ok(Err(why)) => self
4892 .state
4893 .event("land", format!("did not close a pull request: {why}")),
4894 Err(e) => {
4895 let why = format!(
4896 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4897 not be closed ({e:#}); resume to retry"
4898 );
4899 self.state.status = RunStatus::Blocked;
4900 self.state.base_sync = Some(BaseSync {
4901 tip: tip.to_owned(),
4902 behind,
4903 attempts,
4904 conflict: Some(why.clone()),
4905 already_in: None,
4906 });
4907 self.state.event("land", why);
4908 self.state.save()?;
4909 return Ok(true);
4910 }
4911 }
4912 self.state.status = RunStatus::AlreadyInBase;
4913 self.state.base_sync = Some(BaseSync {
4914 tip: tip.to_owned(),
4915 behind,
4916 attempts,
4917 conflict: None,
4918 already_in: Some(evidence),
4919 });
4920 self.state.event("land", message);
4921 self.state.save()?;
4922 self.settle_questions();
4923 Ok(true)
4924 }
4925
4926 fn landing_base(&self) -> String {
4936 self.state
4937 .base_sync
4938 .as_ref()
4939 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4940 }
4941
4942 pub async fn fix_selected(
4975 &mut self,
4976 ids: &[String],
4977 reason: &str,
4978 allow_stale: bool,
4979 ) -> Result<()> {
4980 let reason = reason.trim();
4981 if reason.is_empty() {
4982 bail!("a fix request needs a reason — that is the operator's own record of why");
4983 }
4984 if ids.is_empty() {
4985 bail!("no finding id given");
4986 }
4987 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4988 bail!(
4989 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4990 has already concluded — can be given a targeted fix. A run still \
4991 in progress should simply be resumed; a `merged` run's branch has \
4992 already landed, so its answer is a fresh `magi review <branch>`, \
4993 not reopening this run's own record",
4994 self.state.id,
4995 self.state.status.as_str()
4996 );
4997 }
4998 let Some(winner) = self.state.winner().cloned() else {
4999 bail!("run {} has no winning candidate to fix", self.state.id);
5000 };
5001 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5002 bail!(
5003 "branch `{}` no longer exists; this run cannot be extended",
5004 winner.branch
5005 );
5006 }
5007 let home = crate::run::home();
5008 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5009 bail!(
5010 "run {} is currently being worked on by another magi process",
5011 self.state.id
5012 );
5013 }
5014 let _claim = FixClaim::acquire(&self.state.dir())?;
5020
5021 let mut seen = BTreeSet::new();
5025 let mut findings = Vec::new();
5026 let mut missing = Vec::new();
5027 for id in ids {
5028 if !seen.insert(id.clone()) {
5029 continue;
5030 }
5031 match self.state.finding(id) {
5032 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5033 id: f.id.clone(),
5034 severity: f.severity,
5035 reviewer_vote: rec.vote,
5036 round: round.round,
5037 round_head: round.head.clone(),
5038 reviewer: rec.reviewer,
5039 agent: rec.agent.clone(),
5040 file: f.file.clone(),
5041 line: f.line,
5042 title: f.title.clone(),
5043 detail: f.detail.clone(),
5044 outcome: OperatorFixOutcome::Pending,
5045 }),
5046 None => missing.push(id.clone()),
5047 }
5048 }
5049 if !missing.is_empty() {
5050 bail!(
5051 "unknown finding id(s): {}; nothing was changed",
5052 missing.join(", ")
5053 );
5054 }
5055
5056 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5057 let stale_details: Vec<(String, String)> = findings
5058 .iter()
5059 .filter(|f| f.round_head != head_at_request)
5060 .map(|f| (f.id.clone(), f.round_head.clone()))
5061 .collect();
5062 let stale = !stale_details.is_empty();
5063 if stale && !allow_stale {
5064 bail!(
5065 "the branch has moved since some finding(s) were raised — {} — now \
5066 at {}; pass --allow-stale to fix anyway, or re-run review first",
5067 stale_details
5068 .iter()
5069 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5070 .collect::<Vec<_>>()
5071 .join(", "),
5072 short(&head_at_request)
5073 );
5074 }
5075
5076 let request = OperatorFixRequest {
5077 requested_at: Timestamp::now(),
5078 reason: reason.to_owned(),
5079 findings,
5080 head_at_request: head_at_request.clone(),
5081 allow_stale,
5082 stale,
5083 fix: None,
5084 result_head: None,
5085 follow_up_review_run: None,
5086 };
5087 self.state.event(
5088 "fix",
5089 format!(
5090 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5091 request.findings.len(),
5092 request
5093 .findings
5094 .iter()
5095 .map(|f| f.id.as_str())
5096 .collect::<Vec<_>>()
5097 .join(", "),
5098 ),
5099 );
5100 self.state.operator_fixes.push(request);
5107 self.state.save()?;
5108 let request_index = self.state.operator_fixes.len() - 1;
5109
5110 let repo = self.state.repo.clone();
5119 let coordination = self.coordinate(&repo).await?;
5120 if winner.worktree.exists() {
5121 let dirty = git::git(
5124 &winner.worktree,
5125 &["status", "--porcelain", "--untracked-files=all"],
5126 )
5127 .await?;
5128 let only_withheld = dirty.lines().all(|l| {
5129 l.strip_prefix("?? ")
5130 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5131 });
5132 if !only_withheld {
5133 bail!(
5134 "`{}` has uncommitted changes; refusing to touch it — commit or \
5135 discard them first",
5136 winner.worktree.display()
5137 );
5138 }
5139 git::worktree_remove(&self.state.repo, &winner.worktree)
5140 .await
5141 .ok();
5142 }
5143 let fix_worktree = self.state.worktree_root().join("operator-fix");
5144 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5145 git::git(
5146 &self.state.repo,
5147 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5148 )
5149 .await
5150 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5151 if !git::is_clean(&fix_worktree).await? {
5152 git::worktree_remove(&self.state.repo, &fix_worktree)
5153 .await
5154 .ok();
5155 bail!(
5156 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5157 winner.branch
5158 );
5159 }
5160 let repo_for_setup = self.state.repo.clone();
5161 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5162 Ok(()) => {
5163 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5164 }
5165 Err(e) => Err(e),
5166 };
5167 if let Err(e) = setup {
5168 git::worktree_remove(&self.state.repo, &fix_worktree)
5169 .await
5170 .ok();
5171 return Err(e);
5172 }
5173
5174 let run_id = self.state.id.clone();
5175 let prompts = self.state.config.prompts.clone();
5176 let language = self.state.config.graph.language.clone();
5177 let sessions = self.state.config.graph.sessions;
5178 let artifacts = agent::artifacts_dir(&self.state.dir());
5179 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5180 .findings
5181 .iter()
5182 .map(|f| Finding {
5183 id: f.id.clone(),
5184 severity: f.severity,
5185 file: f.file.clone(),
5186 line: f.line,
5187 title: f.title.clone(),
5188 detail: f.detail.clone(),
5189 })
5190 .collect();
5191 let fix_prompt = prompt::operator_fix_in(
5192 &self.state.instruction,
5193 &finding_list,
5194 reason,
5195 &stale_details,
5196 &head_at_request,
5197 &language,
5198 &self.state.config.graph.github_language,
5199 );
5200 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5201 drop(coordination);
5202 let (job, seat, out) = self
5203 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5204 prompt: fix_prompt.clone(),
5205 spec,
5206 seat,
5207 cwd: fix_worktree.clone(),
5208 timeout,
5209 allow_write: true,
5210 sessions,
5211 artifacts: artifacts.clone(),
5212 stem: "operator-fix".to_owned(),
5213 handover: None,
5214 })
5215 .await;
5216 let agent_id = seat.agent.clone();
5217
5218 let mut fix = FixRecord {
5219 agent: agent_id,
5220 addressed: Vec::new(),
5221 rejected: Vec::new(),
5222 notes: String::new(),
5223 committed: false,
5224 failed: None,
5225 duration_ms: 0,
5226 continuation: None,
5227 };
5228 let mut final_seat = seat.clone();
5229 match out {
5230 AgentOutcome::Ok(o) => {
5231 fix.duration_ms = o.duration_ms;
5232 let parsed = verdict::extract_json::<FixReport>(&o.text);
5233 let incomplete_reason = match &parsed {
5234 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5235 "the reply parsed, but it reported a command whose own CLI \
5236 never confirmed an exit status"
5237 .to_owned(),
5238 ),
5239 Ok(_) => None,
5240 Err(e) => Some(e.to_string()),
5241 };
5242 match incomplete_reason {
5243 None => {
5244 let report = parsed.expect("checked Ok above");
5245 fix.addressed = report.addressed;
5246 fix.rejected = report.rejected;
5247 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5248 }
5249 Some(reason) => {
5250 let (resumed_seat, resolved, failure, cont) = self
5251 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5252 .await;
5253 fix.duration_ms += cont.cumulative_wait_ms;
5254 fix.continuation = Some(cont);
5255 final_seat = resumed_seat;
5256 match resolved {
5257 Some(report) => {
5258 fix.addressed = report.addressed;
5259 fix.rejected = report.rejected;
5260 fix.notes =
5261 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5262 }
5263 None => fix.failed = failure,
5264 }
5265 }
5266 }
5267 }
5268 AgentOutcome::Dropped(o) => {
5269 fix.duration_ms = o.duration_ms;
5270 let why = o
5271 .dropped
5272 .as_ref()
5273 .map(|d| d.why.as_str())
5274 .unwrap_or("the CLI ended the stream without delivering its answer");
5275 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5276 }
5277 AgentOutcome::Quota(o) => {
5278 self.state.quota.push(QuotaLoss {
5279 seat: final_seat.key.clone(),
5280 node: "fix".to_owned(),
5281 at: Timestamp::now(),
5282 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5283 });
5284 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5285 }
5286 AgentOutcome::Failed(e) => fix.failed = Some(e),
5287 AgentOutcome::ResourceBlocked(e) => {
5288 fix.failed = Some(format!("the fixer was not started: {e}"));
5289 }
5290 }
5291 if fix.continuation.is_none() {
5292 fix.continuation = Some(ContinuationRecord::not_needed());
5293 }
5294 self.state.seats.insert(final_seat.key.clone(), final_seat);
5295
5296 let rescue_message = format!(
5297 "magi: operator-selected fix ({}) (uncommitted work)",
5298 self.state.operator_fixes[request_index]
5299 .findings
5300 .iter()
5301 .map(|f| f.id.as_str())
5302 .collect::<Vec<_>>()
5303 .join(", ")
5304 );
5305 let repo = self.state.repo.clone();
5306 let coordination = self.coordinate(&repo).await?;
5307 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5308 self.state.note_withheld("fix", &r.withheld);
5309 }
5310 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5311 fix.committed = after != head_at_request;
5312 git::worktree_remove(&self.state.repo, &fix_worktree)
5313 .await
5314 .ok();
5315 drop(coordination);
5316
5317 self.state.event(
5318 "fix",
5319 match &fix.failed {
5320 Some(reason) => format!(
5321 "operator fix: adoption report was lost ({reason}); {}",
5322 if fix.committed {
5323 "committed"
5324 } else {
5325 "NO new commit"
5326 }
5327 ),
5328 None => format!(
5329 "operator fix: {} addressed, {} rejected, {}",
5330 fix.addressed.len(),
5331 fix.rejected.len(),
5332 if fix.committed {
5333 "committed"
5334 } else {
5335 "NO new commit"
5336 }
5337 ),
5338 },
5339 );
5340
5341 for f in &mut self.state.operator_fixes[request_index].findings {
5348 f.outcome = if fix.failed.is_some() {
5349 OperatorFixOutcome::Unreported
5350 } else if fix.addressed.contains(&f.id) {
5351 OperatorFixOutcome::Addressed
5352 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5353 OperatorFixOutcome::Rejected { why: r.why.clone() }
5354 } else {
5355 OperatorFixOutcome::Unreported
5356 };
5357 }
5358
5359 let committed = fix.committed;
5360 if committed {
5361 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5362 }
5363 self.state.operator_fixes[request_index].fix = Some(fix);
5364 self.state.save()?;
5367
5368 if committed {
5369 self.state.event(
5370 "fix",
5371 format!(
5372 "operator fix committed {}; opening a follow-up review-only run",
5373 short(&after)
5374 ),
5375 );
5376 let origin =
5379 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5380 match Self::review(
5381 &self.state.repo,
5382 &winner.branch,
5383 self.state.config.clone(),
5384 origin,
5385 )
5386 .await
5387 {
5388 Ok(mut follow_up) => {
5389 follow_up.state.event(
5390 "start",
5391 format!(
5392 "requested by an operator fix on run {} for finding(s) {}",
5393 self.state.id,
5394 self.state.operator_fixes[request_index]
5395 .findings
5396 .iter()
5397 .map(|f| f.id.as_str())
5398 .collect::<Vec<_>>()
5399 .join(", "),
5400 ),
5401 );
5402 follow_up.state.save()?;
5403 let follow_up_id = follow_up.state.id.clone();
5404 let adopted = match crate::direct::adopt(
5407 &follow_up.state,
5408 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5409 ) {
5410 Ok(a) => a,
5411 Err(e) => {
5412 self.state.event(
5415 "fix",
5416 format!(
5417 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5418 ),
5419 );
5420 self.state.save()?;
5421 return Ok(());
5422 }
5423 };
5424 let executed = follow_up.execute().await;
5425 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5426 if let Some(a) = adopted {
5427 a.finish(&follow_up.state, executed);
5428 }
5429 if let Some(e) = failure {
5430 self.state.event(
5431 "fix",
5432 format!(
5433 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5434 ),
5435 );
5436 }
5437 self.state.operator_fixes[request_index].follow_up_review_run =
5438 Some(follow_up_id);
5439 }
5440 Err(e) => {
5441 self.state.event(
5442 "fix",
5443 format!("committed the fix but could not open a follow-up review: {e:#}"),
5444 );
5445 }
5446 }
5447 self.state.save()?;
5448 }
5449
5450 Ok(())
5451 }
5452
5453 async fn ask_fixer(
5473 &mut self,
5474 winner: &Candidate,
5475 node: &'static str,
5476 round: Option<usize>,
5477 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5478 ) -> (SeatJob, SeatState, AgentOutcome) {
5479 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5480 self.ask_fixer_among(attempts, node, round, None, build)
5481 .await
5482 }
5483
5484 async fn ask_fixer_among(
5489 &mut self,
5490 attempts: Vec<(AgentSpec, String)>,
5491 node: &'static str,
5492 round: Option<usize>,
5493 restore: Option<(&Path, &str)>,
5494 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5495 ) -> (SeatJob, SeatState, AgentOutcome) {
5496 let run_id = self.state.id.clone();
5497 let prompts = self.state.config.prompts.clone();
5498 let cache = self.state.config.cache_dir();
5499 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5500 let mut last = None;
5501 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5502 let seat = self.seat(&key, &spec.id);
5503 let mut job = build(spec.clone(), seat);
5504 if i > 0 {
5505 job.stem = format!("{}-{}", job.stem, spec.id);
5506 }
5507 let ctx = WaveCtx {
5508 carry_seats: false,
5509 pauses: Some((&self.pause, &self.interrupt)),
5510 run: &run_id,
5511 node,
5512 prompts: &prompts,
5513 cache: cache.as_deref(),
5514 round,
5515 };
5516 let (seat, out) =
5517 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5518 let advances = match &out {
5521 AgentOutcome::Ok(o) => agent::output_advances(o),
5522 AgentOutcome::ResourceBlocked(_) => false,
5523 _ => true,
5524 };
5525 if let Some(next) = ids.get(i + 1)
5526 && advances
5527 {
5528 self.state.seats.insert(seat.key.clone(), seat.clone());
5529 let class =
5530 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5531 let reason = match &out {
5532 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5533 other => fail_reason(other),
5534 };
5535 if let Some((wt, rev)) = restore
5539 && let Err(e) = self.graft_restore(wt, rev).await
5540 {
5541 let why =
5542 format!("could not restore the worktree before the next fixer: {e:#}");
5543 tracing::warn!("{why}");
5544 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5545 AgentOutcome::ResourceBlocked(why)
5546 } else {
5547 AgentOutcome::Failed(why)
5548 };
5549 last = Some((job, seat, out));
5550 break;
5551 }
5552 record_handover(
5553 &mut self.state,
5554 node,
5555 &seat.key,
5556 &spec.id,
5557 next,
5558 &class,
5559 &reason,
5560 );
5561 continue;
5562 }
5563 last = Some((job, seat, out));
5564 break;
5565 }
5566 last.expect("the fixer chain always has an entry")
5567 }
5568
5569 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5580 use crate::recovery::{Mode, Repair};
5581 if !self.state.reviews.is_empty() {
5582 return Ok(());
5583 }
5584 let Some(mut rec) = self.state.recovery.clone() else {
5585 return Ok(());
5586 };
5587 if rec.mode != Mode::VerifyRepair {
5588 return Ok(());
5589 }
5590 let wt = winner.worktree.clone();
5591 let mut head = git::rev_parse(&wt, "HEAD").await?;
5592 match rec.repair.clone() {
5593 Repair::Done { .. } => return Ok(()),
5594 Repair::WaitingForRestore { before } => {
5595 self.graft_restore(&wt, &before).await?;
5596 head = before;
5597 rec.repair = Repair::NotStarted;
5598 self.state.recovery = Some(rec.clone());
5599 self.state.save()?;
5600 }
5601 Repair::Started { before } => {
5602 if let Err(e) = self.graft_restore(&wt, &before).await {
5606 if e.is::<crate::repo_lock::ResourceBlocked>() {
5607 return Err(e);
5608 }
5609 return self
5610 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5611 .await;
5612 }
5613 rec.repair = Repair::Done {
5614 before: before.clone(),
5615 after: before,
5616 committed: false,
5617 failed: Some("interrupted; rolled back".to_owned()),
5618 };
5619 self.state
5620 .event("recovery", "the repair was interrupted and rolled back");
5621 self.state.recovery = Some(rec);
5622 self.state.save()?;
5623 return Ok(());
5624 }
5625 Repair::NotStarted => {}
5626 }
5627 if crate::recovery::recheck(&mut rec, &head) {
5628 self.state.event("recovery", rec.summary());
5629 self.state.recovery = Some(rec);
5630 self.state.save()?;
5631 return Ok(());
5632 }
5633 rec.repair = Repair::Started {
5634 before: head.clone(),
5635 };
5636 self.state.recovery = Some(rec.clone());
5637 self.state.save()?;
5638
5639 let language = self.state.config.graph.language.clone();
5640 let sessions = self.state.config.graph.sessions;
5641 let artifacts = agent::artifacts_dir(&self.state.dir());
5642 let fix_prompt = prompt::recovery_fix(
5643 &self.state.instruction,
5644 &rec,
5645 self.state.config.graph.review_rounds,
5646 &language,
5647 );
5648 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5649 self.state.event(
5650 "recovery",
5651 format!(
5652 "repair: the fixer starts from the failed verification on {}",
5653 short(&head)
5654 ),
5655 );
5656 let (_job, seat, out) = self
5657 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5658 prompt: fix_prompt.clone(),
5659 spec,
5660 seat,
5661 cwd: wt.clone(),
5662 timeout,
5663 allow_write: true,
5664 sessions,
5665 artifacts: artifacts.clone(),
5666 stem: "fix-recovery".to_owned(),
5667 handover: None,
5668 })
5669 .await;
5670 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5671 rec.repair = Repair::WaitingForRestore { before: head };
5672 self.state.recovery = Some(rec);
5673 return self.stop_for_resource();
5674 }
5675 let failed = match &out {
5676 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5677 AgentOutcome::Quota(o) => {
5678 self.state.quota.push(QuotaLoss {
5679 seat: seat.key.clone(),
5680 node: "fix".to_owned(),
5681 at: Timestamp::now(),
5682 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5683 });
5684 Some("rate limited (quota); the fixer could not run".to_owned())
5685 }
5686 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5687 AgentOutcome::Failed(e) => Some(e.clone()),
5688 };
5689 self.state.seats.insert(seat.key.clone(), seat);
5690 let coordination = self.coordinate(&wt).await?;
5691 let failed = match failed {
5692 Some(why) => {
5694 if let Err(e) = git::restore_to(&wt, &head).await {
5695 return self
5696 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5697 .await;
5698 }
5699 Some(why)
5700 }
5701 None => {
5702 if let Ok(r) = git::rescue_commit(
5703 &wt,
5704 "magi: recovery repair of the failed verification (uncommitted work)",
5705 )
5706 .await
5707 {
5708 self.state.note_withheld("fix", &r.withheld);
5709 }
5710 None
5711 }
5712 };
5713 let after = git::rev_parse(&wt, "HEAD").await?;
5714 drop(coordination);
5715 self.state.event(
5716 "recovery",
5717 match &failed {
5718 Some(why) => format!("repair: nothing kept ({why})"),
5719 None if after != head => {
5720 format!("repair: committed {}", short(&after))
5721 }
5722 None => "repair: the fixer made no commit".to_owned(),
5723 },
5724 );
5725 rec.repair = Repair::Done {
5726 before: head.clone(),
5727 committed: after != head,
5728 after,
5729 failed,
5730 };
5731 self.state.recovery = Some(rec);
5732 self.state.save()?;
5733 Ok(())
5734 }
5735
5736 async fn stop_recovery(
5739 &mut self,
5740 mut rec: crate::recovery::RecoveryRecord,
5741 head: &str,
5742 why: String,
5743 ) -> Result<()> {
5744 rec.repair = crate::recovery::Repair::Started {
5746 before: head.to_owned(),
5747 };
5748 self.state.recovery = Some(rec);
5749 self.state
5750 .event("recovery", format!("repair could not be undone: {why}"));
5751 self.state.status = RunStatus::Blocked;
5752 self.state.save()?;
5753 bail!("recovery repair could not be rolled back: {why}")
5754 }
5755
5756 async fn review_loop(&mut self) -> Result<()> {
5757 if self
5762 .state
5763 .base_sync
5764 .as_ref()
5765 .is_some_and(|s| s.conflict.is_some())
5766 {
5767 return Ok(());
5768 }
5769 let run_id = self.state.id.clone();
5774 let prompts = self.state.config.prompts.clone();
5775 let Some(winner) = self.state.winner().cloned() else {
5776 return Ok(());
5777 };
5778 let max_rounds = self.state.config.graph.review_rounds;
5779 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5789 if status == RunStatus::Gating {
5792 self.record_contested_handoff();
5793 }
5794 self.state.status = status;
5795 self.state.save()?;
5796 return Ok(());
5797 }
5798 self.state.status = RunStatus::Reviewing;
5799 if self
5809 .state
5810 .reviews
5811 .last()
5812 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5813 {
5814 let shell = self.state.config.shell();
5815 return self
5816 .stop_reviewing(
5817 "the last round's own verification never resolved",
5818 &shell,
5819 &winner.worktree,
5820 )
5821 .await;
5822 }
5823
5824 self.recovery_repair(&winner).await?;
5825 if self.resource_halt()? {
5826 return Ok(());
5827 }
5828
5829 let repo = self.state.repo.clone();
5830 let root = self.state.worktree_root();
5831 let language = self.state.config.graph.language.clone();
5832 let sessions = self.state.config.graph.sessions;
5833 let artifacts = self.merge_repair_artifacts();
5834 let base = self.landing_base();
5835 let base_short = short(&base);
5836 let reviewers = self.roles.reviewers.clone();
5837 let shell = self.state.config.shell();
5838
5839 for round in (self.state.reviews.len() + 1)..=max_rounds {
5840 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5841 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5842 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5843 let prev_verification = self
5852 .state
5853 .reviews
5854 .last()
5855 .and_then(|r| r.verification_summary(&head));
5856 let prior_section = match (&self.state.recovery, round) {
5859 (Some(rec), 1) if self.state.reviews.is_empty() => {
5860 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5861 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5862 Some(crate::recovery::head_of(
5863 &git::diff(&winner.worktree, reviewed, "HEAD")
5864 .await
5865 .unwrap_or_default(),
5866 crate::recovery::SINCE_DIFF_MAX,
5867 ))
5868 }
5869 _ => None,
5870 };
5871 Some(prompt::recovery_review(rec, since.as_deref()))
5872 }
5873 _ => None,
5874 };
5875
5876 let mut jobs = Vec::new();
5880 let coordination = self.coordinate(&repo).await?;
5881 for (r, spec) in reviewers.iter().cloned().enumerate() {
5882 let wt = root.join(format!("review-{}", r + 1));
5883 if wt.exists() {
5886 git::reset_detached(&wt, &head).await?;
5887 } else {
5888 git::worktree_add_detached(&repo, &wt, &head).await?;
5889 }
5890 self.setup_seat_worktree(&repo, &wt, false).await?;
5891 let seat_key = format!("review-{}", r + 1);
5892 let spec = pick_start_spec(
5895 &self.roles.reviewer_roster,
5896 spec,
5897 self.state.seat_history.get(&seat_key),
5898 );
5899 let seat = self.seat(&seat_key, &spec.id);
5900 jobs.push(SeatJob {
5901 prompt: prompt::review(&prompt::ReviewCtx {
5902 instruction: &self.state.instruction,
5903 branch: &winner.branch,
5904 base_short: &base_short,
5905 stat: &stat,
5906 patch: &patch,
5907 verification: prev_verification.as_ref(),
5908 prior: prior_section.as_deref(),
5909 reviewers: reviewers.len(),
5910 round,
5911 rounds: max_rounds,
5912 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5915 lens: Lens::for_seat(r),
5916 language: &language,
5917 github_language: &self.state.config.graph.github_language,
5918 }),
5919 spec,
5920 seat,
5921 cwd: wt,
5922 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5923 allow_write: false,
5924 sessions,
5925 artifacts: artifacts.clone(),
5926 stem: format!("review-{round}-{}", r + 1),
5927 handover: None,
5928 });
5929 }
5930
5931 drop(coordination);
5932 self.state.event(
5933 "review",
5934 format!(
5935 "round {round}: {} reviewers on {}",
5936 jobs.len(),
5937 short(&head)
5938 ),
5939 );
5940 let mut quota_losses = Vec::new();
5941 let review_retries = self.state.config.graph.retries;
5942 let review_cache = self.state.config.cache_dir();
5943 let ctx = WaveCtx {
5944 carry_seats: true,
5945 pauses: Some((&self.pause, &self.interrupt)),
5946 run: &run_id,
5947 node: "review",
5948 prompts: &prompts,
5949 cache: review_cache.as_deref(),
5950 round: Some(round),
5951 };
5952 let results = ask_json_wave::<Review>(
5953 jobs,
5954 Arc::clone(&self.sem),
5955 review_retries,
5956 &self.roles.reviewer_roster,
5957 &ctx,
5958 &mut quota_losses,
5959 &mut self.state,
5960 &|_: &Review| Ok(()),
5961 )
5962 .await;
5963 if self.state.resource_block.is_some() {
5964 return self.stop_for_resource();
5965 }
5966 let round_quota_missing = quota_losses.len();
5970 self.state.quota.extend(quota_losses);
5971
5972 let mut records = Vec::new();
5973 let mut all_findings = Vec::new();
5974 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5975 let agent_id = seat.agent.clone();
5976 self.state.seats.insert(seat.key.clone(), seat);
5977 let mut record = ReviewRecord {
5978 reviewer: r + 1,
5979 agent: agent_id,
5980 summary: String::new(),
5981 findings: Vec::new(),
5982 vote: None,
5983 failed: None,
5984 duration_ms: 0,
5985 attempts,
5991 };
5992 match res {
5993 Ok((review, out)) => {
5994 record.summary =
6002 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6003 record.vote = Some(review.vote);
6004 record.duration_ms = out.duration_ms;
6005 for (n, mut f) in review.findings.into_iter().enumerate() {
6006 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6009 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6010 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6011 f.file = f
6017 .file
6018 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6019 all_findings.push(f.clone());
6020 record.findings.push(f);
6021 }
6022 self.state.event(
6023 "review",
6024 format!(
6025 "round {round}: reviewer {} voted {} with {} finding(s)",
6026 r + 1,
6027 review.vote.label(),
6028 record.findings.len()
6029 ),
6030 );
6031 }
6032 Err(e) => {
6033 record.failed = Some(e.to_string());
6034 self.state.event(
6035 "review",
6036 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6037 );
6038 }
6039 }
6040 records.push(record);
6041 }
6042
6043 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6050 let vote_split =
6051 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6052 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6053 if vote_split {
6054 self.state.event(
6055 "review",
6056 format!(
6057 "round {round}: votes split ({}) — one round of reconsideration",
6058 initial_votes
6059 .iter()
6060 .map(|v| v.label())
6061 .collect::<Vec<_>>()
6062 .join(", ")
6063 ),
6064 );
6065 let panel: Vec<ReviewSeatReport<'_>> = records
6068 .iter()
6069 .filter_map(|r| {
6070 r.vote.map(|vote| ReviewSeatReport {
6071 reviewer: r.reviewer,
6072 vote,
6073 summary: &r.summary,
6074 findings: &r.findings,
6075 })
6076 })
6077 .collect();
6078
6079 let mut jobs = Vec::new();
6080 let mut seats_at = Vec::new();
6081 for (r, spec) in reviewers.iter().cloned().enumerate() {
6082 if records[r].vote.is_none() {
6086 continue;
6087 }
6088 let wt = root.join(format!("review-{}", r + 1));
6089 let seat_key = format!("review-{}", r + 1);
6090 let spec = self.occupant(&seat_key, spec);
6091 let seat = self.seat(&seat_key, &spec.id);
6092 let build = |with_patch: bool| {
6100 prompt::review_reconsider(&ReviewReconsiderCtx {
6101 instruction: &self.state.instruction,
6102 reviewer: r + 1,
6103 lens: Lens::for_seat(r),
6104 panel: &panel,
6105 patch: with_patch.then_some(ReviewPatch {
6106 branch: &winner.branch,
6107 base_short: &base_short,
6108 stat: &stat,
6109 patch: &patch,
6110 }),
6111 round,
6112 rounds: max_rounds,
6113 language: &language,
6114 })
6115 };
6116 let full = build(true);
6117 let prompt = if has_context(&spec, &seat, sessions) {
6118 build(false)
6119 } else {
6120 full.clone()
6121 };
6122 jobs.push(SeatJob {
6123 prompt,
6124 spec,
6125 seat,
6126 cwd: wt,
6127 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6128 allow_write: false,
6129 sessions,
6130 artifacts: artifacts.clone(),
6131 stem: format!("review-{round}-reconsider-{}", r + 1),
6132 handover: Some(full),
6133 });
6134 seats_at.push(r);
6135 }
6136
6137 let mut recon_quota_losses = Vec::new();
6138 let recon_cache = self.state.config.cache_dir();
6139 let recon_ctx = WaveCtx {
6140 carry_seats: true,
6141 pauses: Some((&self.pause, &self.interrupt)),
6142 run: &run_id,
6143 node: "review",
6144 prompts: &prompts,
6145 cache: recon_cache.as_deref(),
6146 round: Some(round),
6147 };
6148 let recon_results = ask_json_wave::<ReviewRevote>(
6149 jobs,
6150 Arc::clone(&self.sem),
6151 review_retries,
6152 &self.roles.reviewer_roster,
6153 &recon_ctx,
6154 &mut recon_quota_losses,
6155 &mut self.state,
6156 &|_: &ReviewRevote| Ok(()),
6157 )
6158 .await;
6159 if self.state.resource_block.is_some() {
6160 return self.stop_for_resource();
6161 }
6162 self.state.quota.extend(recon_quota_losses);
6163
6164 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6165 let agent_id = seat.agent.clone();
6166 self.state.seats.insert(seat.key.clone(), seat);
6167 let mut rec = ReviewRevoteRecord {
6168 reviewer: r + 1,
6169 agent: agent_id,
6170 vote: None,
6171 reason: String::new(),
6172 failed: None,
6173 };
6174 match res {
6175 Ok((rv, _)) => {
6176 rec.vote = Some(rv.vote);
6177 rec.reason =
6178 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6179 self.state.event(
6180 "review",
6181 format!(
6182 "round {round}: reviewer {} revoted {}",
6183 r + 1,
6184 rv.vote.label()
6185 ),
6186 );
6187 }
6188 Err(e) => {
6189 rec.failed = Some(e.to_string());
6190 self.state.event(
6191 "review",
6192 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6193 );
6194 }
6195 }
6196 reconsideration.push(rec);
6197 }
6198 } else if initial_votes.len() > 1 {
6199 self.state.event(
6200 "review",
6201 format!(
6202 "round {round}: votes agreed ({}) — no reconsideration",
6203 initial_votes[0].label()
6204 ),
6205 );
6206 }
6207
6208 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6209 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6210 let defer_e2e =
6221 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6222 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6223 let reason =
6224 format!("{blocking} blocking finding(s) already required a fix this round");
6225 self.state.event(
6226 "verify",
6227 format!(
6228 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6229 {}); it will run once a round has none left",
6230 short(&head)
6231 ),
6232 );
6233 (Vec::new(), false, true, Some(reason))
6234 } else {
6235 let e2e_commands = self.state.config.verify.e2e.clone();
6236 let cache_dir = self.state.config.cache_dir();
6237 let context = format!("round {round}");
6238 let wait = self.repository_wait();
6239 let (e2e, verify_retried) = with_cache_lease(
6240 &mut self.state,
6241 cache_dir.as_deref(),
6242 "e2e",
6243 "e2e",
6244 &winner.worktree,
6245 &head,
6246 verify_timeout,
6247 &context,
6248 &wait,
6249 |state, budget| {
6250 let shell = shell.clone();
6251 let e2e_commands = e2e_commands.clone();
6252 let worktree = winner.worktree.clone();
6253 let context = context.clone();
6254 async move {
6255 run_e2e_with_retry(
6256 state,
6257 &shell,
6258 &e2e_commands,
6259 &worktree,
6260 budget,
6261 &context,
6262 )
6263 .await
6264 }
6265 },
6266 )
6267 .await?;
6268 (e2e, verify_retried, false, None)
6269 };
6270
6271 let expected = records.len();
6272 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6273 let incomplete = answered < expected;
6274 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6275 let policy = self.state.config.graph.incomplete_review;
6276 let clean = round_is_clean(
6277 blocking,
6278 e2e_ok,
6279 answered,
6280 expected,
6281 round_quota_missing,
6282 policy,
6283 );
6284
6285 let mut round_record = ReviewRound {
6286 round,
6287 head: head.clone(),
6288 verified_head: None,
6289 verified_at: None,
6290 reviews: records,
6291 e2e,
6292 verify_retried,
6293 e2e_deferred,
6294 e2e_defer_reason,
6295 fix: None,
6296 blocking,
6297 answered,
6298 expected,
6299 clean,
6300 progressed: false,
6301 vote_split,
6302 reconsideration,
6303 verdict: None,
6304 };
6305 round_record.verdict = ReviewVote::worst(
6309 round_record
6310 .final_votes()
6311 .into_iter()
6312 .map(|(_, _, vote)| vote),
6313 );
6314 if !matches!(
6325 round_record.e2e_status(),
6326 E2eStatus::Deferred | E2eStatus::NotConfigured
6327 ) {
6328 round_record.verified_head = Some(head.clone());
6329 round_record.verified_at = Some(Timestamp::now());
6330 }
6331 let this_round_verification = round_record.verification_summary(&head);
6332
6333 if incomplete {
6334 let missing: Vec<String> = round_record
6335 .reviews
6336 .iter()
6337 .filter(|r| r.failed.is_some())
6338 .map(|r| format!("review-{}", r.reviewer))
6339 .collect();
6340 self.state.event(
6341 "review",
6342 format!(
6343 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6344 missing.join(", ")
6345 ),
6346 );
6347 }
6348
6349 if clean {
6350 self.state.event(
6351 "review",
6352 if incomplete && policy == IncompleteReviewPolicy::Warn {
6353 format!(
6354 "round {round}: clean (warn policy, incomplete panel) — no \
6355 blocking findings from the seats that answered, verification green"
6356 )
6357 } else if incomplete {
6358 format!(
6359 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6360 quorum) — no blocking findings from the seats that answered, \
6361 verification green",
6362 expected - answered
6363 )
6364 } else {
6365 format!("round {round}: clean — no blocking findings, verification green")
6366 },
6367 );
6368 self.state.reviews.push(round_record);
6369 self.state.status = RunStatus::Gating;
6370 self.state.save()?;
6371 return Ok(());
6372 }
6373
6374 if incomplete && blocking == 0 && e2e_ok {
6382 self.state.reviews.push(round_record);
6383 self.state.save()?;
6384 if round == max_rounds {
6385 self.state.status = RunStatus::Blocked;
6386 self.state.event(
6387 "review",
6388 format!(
6389 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6390 refusing to call it clean",
6391 expected - answered
6392 ),
6393 );
6394 return Ok(());
6395 }
6396 continue;
6397 }
6398
6399 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6411 self.state.reviews.push(round_record);
6412 return self
6413 .stop_reviewing(
6414 "the round's own verification could not run",
6415 &shell,
6416 &winner.worktree,
6417 )
6418 .await;
6419 }
6420
6421 if round == max_rounds {
6422 self.state.reviews.push(round_record);
6423 return self
6424 .stop_reviewing(
6425 &format!(
6426 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6427 ),
6428 &shell,
6429 &winner.worktree,
6430 )
6431 .await;
6432 }
6433
6434 let blocking_findings: Vec<_> = all_findings
6437 .iter()
6438 .filter(|f| f.severity.blocks())
6439 .cloned()
6440 .collect();
6441 let fix_prompt = prompt::fix_in(
6442 &self.state.instruction,
6443 &blocking_findings,
6444 this_round_verification.as_ref(),
6445 round,
6446 max_rounds,
6447 &language,
6448 &self.state.config.graph.github_language,
6449 );
6450 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6451 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6452 let (job, seat, out) = self
6453 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6454 prompt: fix_prompt.clone(),
6455 spec,
6456 seat,
6457 cwd: winner.worktree.clone(),
6458 timeout,
6459 allow_write: true,
6460 sessions,
6461 artifacts: artifacts.clone(),
6462 stem: format!("fix-{round}"),
6463 handover: None,
6464 })
6465 .await;
6466 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6469 return self.stop_for_resource();
6470 }
6471 let agent_id = seat.agent.clone();
6472
6473 let mut fix = FixRecord {
6474 agent: agent_id,
6475 addressed: Vec::new(),
6476 rejected: Vec::new(),
6477 notes: String::new(),
6478 committed: false,
6479 failed: None,
6480 duration_ms: 0,
6481 continuation: None,
6482 };
6483 let mut continuation = ContinuationRecord::not_needed();
6484 let mut final_seat = seat.clone();
6485 match out {
6486 AgentOutcome::Ok(o) => {
6487 fix.duration_ms = o.duration_ms;
6488 let parsed = verdict::extract_json::<FixReport>(&o.text);
6489 let incomplete_reason = match &parsed {
6496 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6497 "the reply parsed, but it reported a command whose own CLI never \
6498 confirmed an exit status"
6499 .to_owned(),
6500 ),
6501 Ok(_) => None,
6502 Err(e) => Some(e.to_string()),
6503 };
6504 match incomplete_reason {
6505 None => {
6506 let report = parsed.expect("checked Ok above");
6507 fix.addressed = report.addressed;
6508 fix.rejected = report.rejected;
6509 fix.notes =
6510 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6511 }
6512 Some(reason) => {
6513 let (resumed_seat, resolved, failure, cont) = self
6514 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6515 .await;
6516 fix.duration_ms += cont.cumulative_wait_ms;
6517 continuation = cont;
6518 final_seat = resumed_seat;
6519 match resolved {
6520 Some(report) => {
6521 fix.addressed = report.addressed;
6522 fix.rejected = report.rejected;
6523 fix.notes = blind::sanitize_prose(
6524 &report.notes,
6525 &self.state.config.blind,
6526 );
6527 }
6528 None => fix.failed = failure,
6529 }
6530 }
6531 }
6532 }
6533 AgentOutcome::Dropped(o) => {
6535 fix.duration_ms = o.duration_ms;
6536 let why = o
6537 .dropped
6538 .as_ref()
6539 .map(|d| d.why.as_str())
6540 .unwrap_or("the CLI ended the stream without delivering its answer");
6541 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6542 }
6543 AgentOutcome::Quota(o) => {
6544 self.state.quota.push(QuotaLoss {
6545 seat: final_seat.key.clone(),
6546 node: "fix".to_owned(),
6547 at: Timestamp::now(),
6548 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6549 });
6550 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6551 }
6552 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6553 }
6554 fix.continuation = Some(continuation);
6555 self.state.seats.insert(final_seat.key.clone(), final_seat);
6556 let coordination = self.coordinate(&winner.worktree).await?;
6557 if let Ok(r) = git::rescue_commit(
6558 &winner.worktree,
6559 &format!("magi: review round {round} fixes (uncommitted work)"),
6560 )
6561 .await
6562 {
6563 self.state.note_withheld("fix", &r.withheld);
6564 }
6565 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6566 fix.committed = after != before;
6567 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6575 drop(coordination);
6576 let progressed = diff_after != patch;
6577 let commit_note = if fix.committed {
6578 "committed"
6579 } else {
6580 "NO new commit"
6581 };
6582 let tree_note = if progressed {
6583 "changed vs base"
6584 } else {
6585 "unchanged vs base"
6586 };
6587 self.state.event(
6588 "fix",
6589 match &fix.failed {
6590 Some(reason) => {
6596 format!(
6597 "round {round}: fixer's adoption report was lost ({reason}); \
6598 {commit_note}, tree {tree_note}"
6599 )
6600 }
6601 None => format!(
6602 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6603 {tree_note}{}",
6604 fix.addressed.len(),
6605 fix.rejected.len(),
6606 if continuation.outcome == ContinuationOutcome::Resumed {
6607 format!(
6608 " (adoption report recovered after {} continuation(s))",
6609 continuation.attempts
6610 )
6611 } else {
6612 String::new()
6613 },
6614 ),
6615 },
6616 );
6617 round_record.fix = Some(fix);
6618 round_record.progressed = progressed;
6619 self.state.reviews.push(round_record);
6620 self.state.save()?;
6621
6622 if matches!(
6635 continuation.outcome,
6636 ContinuationOutcome::Exhausted
6637 | ContinuationOutcome::QuotaLost
6638 | ContinuationOutcome::NoSession
6639 ) {
6640 return self
6641 .stop_reviewing(
6642 "the fixer's adoption report never came back, even after resuming its \
6643 own seat; refusing to start another round against the same worktree \
6644 while that is unresolved",
6645 &shell,
6646 &winner.worktree,
6647 )
6648 .await;
6649 }
6650
6651 let streak = self
6652 .state
6653 .reviews
6654 .iter()
6655 .rev()
6656 .take_while(|r| !r.progressed)
6657 .count();
6658 if streak >= STAGNANT_LIMIT {
6659 return self
6660 .stop_reviewing(
6661 &format!(
6662 "the tree has not moved against base for {streak} round(s) in a row"
6663 ),
6664 &shell,
6665 &winner.worktree,
6666 )
6667 .await;
6668 }
6669 }
6670 Ok(())
6671 }
6672
6673 fn record_contested_handoff(&mut self) {
6707 if self.state.contested_handoff.is_some() {
6708 return;
6709 }
6710 let Some(contested) = self
6711 .state
6712 .reviews
6713 .last()
6714 .and_then(ReviewRound::contested_handoff)
6715 else {
6716 return;
6717 };
6718 self.state.event(
6719 "review",
6720 format!(
6721 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6722 wait for the owner's approval",
6723 contested.findings.len(),
6724 contested.rejecters.len()
6725 ),
6726 );
6727 self.state.contested_handoff = Some(contested);
6728 }
6729
6730 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6731 let round_idx = self.state.reviews.len() - 1;
6732 let needs_catchup_run = matches!(
6740 self.state.reviews[round_idx].e2e_status(),
6741 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6742 );
6743 if needs_catchup_run {
6744 let round = self.state.reviews[round_idx].round;
6745 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6746 let commands = self.state.config.verify.e2e.clone();
6747 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6748 let cache_dir = self.state.config.cache_dir();
6749 let context = format!(
6750 "round {round}: verification unresolved, catching up before the final decision"
6751 );
6752 let wait = self.repository_wait();
6753 let (outcomes, verify_retried) = with_cache_lease(
6754 &mut self.state,
6755 cache_dir.as_deref(),
6756 "e2e",
6757 "e2e",
6758 worktree,
6759 &attempted_head,
6760 timeout,
6761 &context,
6762 &wait,
6763 |state, budget| {
6764 let shell = shell.to_vec();
6765 let commands = commands.clone();
6766 let context = context.clone();
6767 async move {
6768 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6769 .await
6770 }
6771 },
6772 )
6773 .await?;
6774 let last = &mut self.state.reviews[round_idx];
6775 last.e2e = outcomes;
6776 last.verify_retried = verify_retried;
6777 last.verified_head = Some(attempted_head);
6784 last.verified_at = Some(Timestamp::now());
6785 if verify_inconclusive(&last.e2e) {
6786 self.state.save()?;
6793 return Ok(());
6794 }
6795 last.e2e_deferred = false;
6796 }
6797 let last = &self.state.reviews[round_idx];
6798 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6799
6800 match last.e2e_status() {
6801 E2eStatus::Failed => {
6802 let red: Vec<String> = last
6803 .e2e
6804 .iter()
6805 .filter(|o| !o.ok())
6806 .map(|o| {
6807 format!(
6808 "`{}` -> {:?}\n{}",
6809 o.command,
6810 o.code,
6811 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6812 )
6813 })
6814 .collect();
6815 self.state
6816 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6817 self.state.status = RunStatus::Blocked;
6818 }
6819 E2eStatus::ResourceBlocked => {
6824 self.state.event(
6825 "review",
6826 format!(
6827 "{why}; e2e could not run (shared build cache unavailable); not \
6828 deciding yet"
6829 ),
6830 );
6831 }
6832 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6833 self.state.event(
6834 "review",
6835 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6836 );
6837 self.record_contested_handoff();
6838 self.state.status = RunStatus::Gating;
6839 }
6840 }
6841 self.state.save()?;
6842 Ok(())
6843 }
6844
6845 async fn gate(&mut self) -> Result<()> {
6848 if self.state.status == RunStatus::Failed
6860 || self
6861 .state
6862 .base_sync
6863 .as_ref()
6864 .is_some_and(|s| s.conflict.is_some())
6865 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6866 != Some(RunStatus::Gating)
6867 {
6868 return Ok(());
6869 }
6870 if self.state.gate_ran {
6871 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6882 self.state.status = RunStatus::Blocked;
6883 self.state.save()?;
6884 }
6885 return Ok(());
6886 }
6887 let Some(winner) = self.state.winner().cloned() else {
6888 return Ok(());
6889 };
6890 self.state.status = RunStatus::Gating;
6891 let mut outcomes = self.run_gate(&winner).await?;
6892 loop {
6893 if verify_inconclusive(&outcomes) {
6904 self.state.save()?;
6905 return Ok(());
6906 }
6907 if outcomes.iter().all(CommandOutcome::ok) {
6908 break;
6909 }
6910 match self.gate_fix_round(&winner, &outcomes).await? {
6911 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6912 GateFix::Stop => break,
6913 GateFix::Defer => {
6914 self.state.save()?;
6915 return Ok(());
6916 }
6917 }
6918 }
6919 let passed = outcomes.iter().all(CommandOutcome::ok);
6920 self.state.gate = outcomes;
6921 self.state.gate_ran = true;
6922 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6923 if !passed {
6924 self.state.status = RunStatus::Blocked;
6925 let spent = self.state.gate_fixes.len();
6926 self.state.event(
6927 "gate",
6928 if spent == 0 {
6929 "gate failed; not merging".to_owned()
6930 } else {
6931 format!("gate failed after {spent} gate-fix round(s); not merging")
6932 },
6933 );
6934 }
6935 self.state.save()?;
6936 Ok(())
6937 }
6938
6939 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6949 let commands = state.config.verify.pre_gate.clone();
6950 if commands.is_empty() {
6951 return;
6952 }
6953 let shell = state.config.shell();
6954 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6955 let (outcomes, _) = run_commands(
6956 state,
6957 "pre_gate",
6958 "pre_gate",
6959 0,
6960 &shell,
6961 &commands,
6962 &winner.worktree,
6963 timeout,
6964 )
6965 .await;
6966 for o in &outcomes {
6967 if !o.ok() {
6968 tracing::warn!(
6969 "pre_gate `{}` failed ({:?}); the gate decides",
6970 o.command,
6971 o.code
6972 );
6973 }
6974 state.event(
6975 "pre_gate",
6976 format!(
6977 "`{}` -> {}",
6978 o.command,
6979 if o.ok() {
6980 "pass".to_owned()
6981 } else {
6982 format!(
6983 "FAIL ({:?})\n{}",
6984 o.code,
6985 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6986 )
6987 }
6988 ),
6989 );
6990 }
6991 state.pre_gate = outcomes;
6992 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6993 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6994 Ok(head) => {
6995 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6996 state.pre_gate_commit = Some(head);
6997 }
6998 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6999 },
7000 Ok(false) => {}
7001 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7002 }
7003 if let Err(e) = state.save() {
7004 tracing::warn!("could not persist the pre_gate record: {e:#}");
7005 }
7006 }
7007
7008 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7011 let shell = self.state.config.shell();
7012 let gate_commands = self.state.config.verify.gate.clone();
7013 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7014 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7015 Vec::new()
7016 } else {
7017 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7018 let cache_dir = self.state.config.cache_dir();
7019 let wait = self.repository_wait();
7020 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7021 let (outcomes, _) = with_cache_lease(
7022 &mut self.state,
7023 cache_dir.as_deref(),
7024 "gate",
7025 "gate",
7026 &winner.worktree,
7027 &head,
7028 timeout,
7029 "final gate",
7030 &wait,
7031 |state, budget| {
7032 let shell = shell.clone();
7033 let gate_commands = gate_commands.clone();
7034 let winner = winner.clone();
7035 async move {
7036 Self::run_pre_gate(state, &winner).await;
7037 let (outcomes, timed_out_pids) = run_commands(
7038 state,
7039 "gate",
7040 "gate",
7041 0,
7042 &shell,
7043 &gate_commands,
7044 &winner.worktree,
7045 budget,
7046 )
7047 .await;
7048 (outcomes, false, timed_out_pids)
7049 }
7050 },
7051 )
7052 .await?;
7053 outcomes
7054 };
7055 if outcomes.is_empty() {
7056 self.state
7061 .event("gate", "no gate commands configured; not tested");
7062 }
7063 for o in &outcomes {
7064 self.state.event(
7065 "gate",
7066 format!(
7067 "`{}` -> {}",
7068 o.command,
7069 if o.ok() {
7070 "pass".to_owned()
7071 } else {
7072 format!(
7073 "FAIL ({:?})\n{}",
7074 o.code,
7075 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7076 )
7077 }
7078 ),
7079 );
7080 }
7081 Ok(outcomes)
7082 }
7083
7084 async fn gate_fix_round(
7097 &mut self,
7098 winner: &Candidate,
7099 outcomes: &[CommandOutcome],
7100 ) -> Result<GateFix> {
7101 let cap = self.state.config.graph.gate_fix_rounds;
7102 let spent = self.state.gate_fixes.len();
7103 if spent >= cap {
7104 if cap > 0 {
7105 self.state.event(
7106 "gate",
7107 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7108 );
7109 }
7110 return Ok(GateFix::Stop);
7111 }
7112 if !gate_fixable(outcomes) {
7113 self.state.event(
7114 "gate",
7115 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7116 command or similar); not spending a fix round on it",
7117 );
7118 return Ok(GateFix::Stop);
7119 }
7120 let min_free = self.state.config.disk.min_free_bytes;
7121 if min_free > 0 {
7122 match crate::disk::free_bytes(&winner.worktree) {
7123 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7124 Ok(free) => {
7125 self.state.event(
7126 "gate",
7127 format!(
7128 "only {free} bytes free ({min_free} required by `[disk] \
7129 min_free_bytes`); not spending a fix round on a failure the disk \
7130 may explain"
7131 ),
7132 );
7133 return Ok(GateFix::Stop);
7134 }
7135 Err(e) => {
7136 self.state.event(
7137 "gate",
7138 format!("free disk space could not be measured ({e:#}); no fix round"),
7139 );
7140 return Ok(GateFix::Stop);
7141 }
7142 }
7143 }
7144
7145 let attempt = spent + 1;
7146 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7147 let base = self.landing_base();
7148 let fix_prompt = prompt::gate_fix_in(
7149 &self.state.instruction,
7150 &failed,
7151 attempt,
7152 cap,
7153 &self.state.config.graph.language,
7154 &self.state.config.graph.github_language,
7155 );
7156 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7157 let sessions = self.state.config.graph.sessions;
7158 let artifacts = self.merge_repair_artifacts();
7159 self.state.event(
7160 "gate",
7161 format!("gate failed; gate-fix round {attempt} of {cap}"),
7162 );
7163 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7164 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7165 let (_, seat, out) = self
7166 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7167 prompt: fix_prompt.clone(),
7168 spec,
7169 seat,
7170 cwd: winner.worktree.clone(),
7171 timeout,
7172 allow_write: true,
7173 sessions,
7174 artifacts: artifacts.clone(),
7175 stem: format!("gate-fix-{attempt}"),
7176 handover: None,
7177 })
7178 .await;
7179 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7182 return Ok(GateFix::Defer);
7183 }
7184 let mut record = GateFixRecord {
7185 agent: seat.agent.clone(),
7186 failed,
7187 notes: String::new(),
7188 committed: false,
7189 error: None,
7190 };
7191 match out {
7192 AgentOutcome::Ok(o) => {
7193 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7196 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7197 }
7198 }
7199 AgentOutcome::Dropped(_) => {
7200 record.error = Some("the CLI dropped the stream".to_owned());
7201 }
7202 AgentOutcome::Quota(o) => {
7203 self.state.quota.push(QuotaLoss {
7204 seat: seat.key.clone(),
7205 node: "gate-fix".to_owned(),
7206 at: Timestamp::now(),
7207 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7208 });
7209 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7210 }
7211 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7212 }
7213 self.state.seats.insert(seat.key.clone(), seat);
7214 let coordination = self.coordinate(&winner.worktree).await?;
7215 if let Ok(r) = git::rescue_commit(
7216 &winner.worktree,
7217 &format!("magi: gate fix {attempt} (uncommitted work)"),
7218 )
7219 .await
7220 {
7221 self.state.note_withheld("gate-fix", &r.withheld);
7222 }
7223 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7224 record.committed = after != before;
7225 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7226 let note = record.error.clone();
7227 self.state.gate_fixes.push(record);
7228 self.state.save()?;
7229 if !changed {
7230 self.state.event(
7231 "gate",
7232 match note {
7233 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7234 None => format!("gate-fix round {attempt}: the tree did not change"),
7235 },
7236 );
7237 return Ok(GateFix::Stop);
7238 }
7239 self.state.event(
7240 "gate",
7241 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7242 );
7243
7244 drop(coordination);
7245 let commands = self.state.config.verify.e2e.clone();
7246 if !commands.is_empty() {
7247 let shell = self.state.config.shell();
7248 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7249 let cache_dir = self.state.config.cache_dir();
7250 let context = format!("gate-fix round {attempt}");
7251 let wait = self.repository_wait();
7252 let (e2e, _) = with_cache_lease(
7253 &mut self.state,
7254 cache_dir.as_deref(),
7255 "e2e",
7256 "e2e",
7257 &winner.worktree,
7258 &after,
7259 timeout,
7260 &context,
7261 &wait,
7262 |state, budget| {
7263 let shell = shell.clone();
7264 let commands = commands.clone();
7265 let context = context.clone();
7266 let worktree = winner.worktree.clone();
7267 async move {
7268 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7269 .await
7270 }
7271 },
7272 )
7273 .await?;
7274 if verify_inconclusive(&e2e) {
7275 return Ok(GateFix::Defer);
7276 }
7277 if e2e.iter().any(|o| !o.ok()) {
7278 self.state.event(
7279 "gate",
7280 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7281 );
7282 return Ok(GateFix::Stop);
7283 }
7284 }
7285 Ok(GateFix::Retry)
7286 }
7287
7288 async fn guarded_pr_message(
7295 &mut self,
7296 winner: &Candidate,
7297 facts: Option<&BranchFacts>,
7298 posting: bool,
7299 ) -> Result<Option<PrMessage>> {
7300 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7301 let head = match final_head.as_deref() {
7302 Some(h) => crate::pr_outcome::FinalHead::At(h),
7303 None => crate::pr_outcome::FinalHead::Unknown,
7304 };
7305 let summarized = if posting {
7307 Some(self.summarize_change(winner, final_head.as_deref()).await)
7308 } else {
7309 None
7310 };
7311 if self.state.resource_block.is_some() {
7316 self.state.event(
7317 "pr-summary",
7318 "a shared resource was busy; nothing was published, resume to summarize again",
7319 );
7320 return Ok(None);
7321 }
7322 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7323 anyhow::bail!(
7324 "the final commit moved during public description generation; retry before publishing"
7325 );
7326 }
7327 let note = match (&summarized, &self.state.change_summary) {
7328 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7329 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7330 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7331 (None, _) => ChangeNote::NotAsked,
7332 };
7333 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7334 if !posting {
7335 let identity = crate::scrub::Identity::current();
7336 return Ok(Some(PrMessage {
7337 title: crate::scrub::scrub(&pr.title, &identity),
7338 body: crate::scrub::scrub(&pr.body, &identity),
7339 }));
7340 }
7341 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7344 if let Some(known) = self
7345 .state
7346 .github_text
7347 .clone()
7348 .filter(|g| g.fingerprint == raw_fp)
7349 {
7350 return self.settle_withheld(winner, &pr, known).await;
7351 }
7352 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7353 let mut violations =
7354 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7355 let identity = crate::scrub::Identity::current();
7356 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7357 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7358 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7359 {
7360 violations.push(crate::github_text::Violation::SensitiveData);
7361 }
7362 let has_language = violations
7366 .iter()
7367 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7368 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7369 self.state.event(
7370 "github-text",
7371 if has_language {
7372 format!("description rejected: {violations:?}; requesting one rewrite")
7373 } else {
7374 format!("description rejected: {violations:?}; asking the owner")
7375 },
7376 );
7377 let mut rewritten = false;
7378 let mut rejected_rewrite = None;
7379 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7380 {
7381 let output = text;
7382 if let Some(title) = summary_title(&output) {
7383 let raw_body = summary_without_title(&output);
7386 let clean_title = crate::scrub::scrub(&title, &identity);
7387 let mut body = crate::scrub::scrub(&raw_body, &identity);
7388 let redacted = clean_title != title || body != raw_body;
7389 let title = clean_title;
7390 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7391 if !complete {
7392 self.state.event(
7393 "github-text",
7394 "description rewrite omitted required findings",
7395 );
7396 }
7397 if !body.trim().is_empty() {
7398 body.push_str("\n\n");
7399 body.push_str(&public_pr_evidence(&self.state, head));
7400 }
7401 let marker = format!("magi:run/{}", self.state.id);
7402 if !body.trim().is_empty() && !body.contains(&marker) {
7403 body.push_str(&format!("\n\n{marker}"));
7404 }
7405 let rewrite = PrMessage {
7406 title: title.clone(),
7407 body: body.clone(),
7408 };
7409 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7410 let mut failed = crate::github_text::check_for(
7411 &self.state.config.graph,
7412 &title,
7413 &body,
7414 redecision,
7415 );
7416 if (!crate::github_text::shareable(&title)
7417 || !crate::github_text::shareable(&body))
7418 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7419 {
7420 failed.push(crate::github_text::Violation::SensitiveData);
7421 }
7422 if complete
7423 && !title.trim().is_empty()
7424 && !body.trim().is_empty()
7425 && !failed.is_empty()
7426 {
7427 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7428 title: title.clone(),
7429 body: body.clone(),
7430 failed: failed
7431 .iter()
7432 .map(|v| crate::github_text::category(*v).to_owned())
7433 .collect(),
7434 });
7435 }
7436 if complete
7437 && !title.trim().is_empty()
7438 && !raw_body.trim().is_empty()
7439 && failed.is_empty()
7440 {
7441 decision = redecision;
7442 pr = PrMessage { title, body };
7443 rewritten = true;
7444 self.state
7445 .event("github-text", "description rewrite passed");
7446 if redacted {
7447 self.state.event(
7448 "github-text",
7449 "sensitive data removed from description rewrite",
7450 );
7451 }
7452 }
7453 }
7454 }
7455 if !rewritten {
7456 self.state.event(
7457 "github-text",
7458 "description rewrite unavailable or rejected; asking the owner",
7459 );
7460 if let Some(w) =
7461 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7462 {
7463 self.retire_github_text_question("the withheld text changed");
7466 let record = crate::run::GithubTextAsk {
7467 rewrite: rejected_rewrite,
7468 fingerprint: raw_fp,
7469 title: w.title,
7470 body: w.body,
7471 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7472 question: None,
7473 asks: 0,
7474 resolved: false,
7475 chosen_title: None,
7476 choice: None,
7477 };
7478 return self.settle_withheld(winner, &pr, record).await;
7479 }
7480 }
7481 }
7482 let (title, body) =
7483 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7484 Ok(Some(PrMessage { title, body }))
7485 }
7486
7487 async fn summarize_change(
7498 &mut self,
7499 winner: &Candidate,
7500 head: Option<&str>,
7501 ) -> std::result::Result<(), String> {
7502 let Some(head) = head else {
7503 return Err("the final commit could not be read".to_owned());
7504 };
7505 let remote = self.state.config.merge.remote.clone();
7506 let tracking = format!("{remote}/{}", self.state.base_branch);
7507 if !git::rev_exists(&winner.worktree, &tracking).await {
7508 return Err(format!("the remote base `{tracking}` could not be read"));
7509 }
7510 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7511 Ok(b) if !b.is_empty() => b,
7512 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7513 };
7514 let evidence = crate::pr_outcome::summary_evidence(
7515 &self.state,
7516 crate::pr_outcome::FinalHead::At(head),
7517 );
7518 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7519 if self
7520 .state
7521 .change_summary
7522 .as_ref()
7523 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7524 {
7525 return Ok(());
7526 }
7527 if self.state.change_summary.take().is_some() {
7528 self.state.event(
7529 "pr-summary",
7530 "the final diff or evidence changed; the earlier change summary was discarded",
7531 );
7532 }
7533 if let Err(e) = write_artifact(
7535 &self.state,
7536 &format!("public-description-{fingerprint}.md"),
7537 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7538 ) {
7539 tracing::warn!("could not save public description evidence: {e:#}");
7540 }
7541 let chain = match agent::pick_chain(
7542 &self.state.config.agents,
7543 self.state.config.roles.synthesizer.as_ref(),
7544 &agent::installed,
7545 "synthesizer",
7546 ) {
7547 Ok(chain) => chain,
7548 Err(e) => {
7549 tracing::warn!("no agent for the change summary: {e:#}");
7550 self.state.event("pr-summary", "no summary agent available");
7551 return Err("no agent was available to summarize it".to_owned());
7552 }
7553 };
7554 let subjects = git::subjects(&winner.worktree, &base, head)
7555 .await
7556 .unwrap_or_default();
7557 let stat = git::diff_stat(&winner.worktree, &base, head)
7558 .await
7559 .unwrap_or_default();
7560 let stat: String = stat.chars().take(6_000).collect();
7561 let mut prompt = crate::prompt::change_summary_in(
7562 &base,
7563 head,
7564 &subjects,
7565 stat.trim(),
7566 &self.state.config.graph.github_language,
7567 );
7568 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7569 let sessions = self.state.config.graph.sessions;
7570 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7571 let prompts = self.state.config.prompts.clone();
7572 let cache = self.state.config.cache_dir();
7573 let run = self.state.id.clone();
7574 for (i, spec) in chain.into_iter().enumerate() {
7575 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7576 let job = SeatJob {
7577 spec: spec.clone(),
7578 seat,
7579 cwd: winner.worktree.clone(),
7580 prompt: prompt.clone(),
7581 timeout,
7582 allow_write: false,
7583 sessions,
7584 artifacts: agent::artifacts_dir(&self.state.dir()),
7585 stem: if i == 0 {
7586 "change-summary".to_owned()
7587 } else {
7588 format!("change-summary-{}", spec.id)
7589 },
7590 handover: None,
7591 };
7592 let ctx = WaveCtx {
7593 run: &run,
7594 node: "pr-summary",
7595 prompts: &prompts,
7596 cache: cache.as_deref(),
7597 round: None,
7598 carry_seats: false,
7599 pauses: Some((&self.pause, &self.interrupt)),
7600 };
7601 let (seat, outcome) =
7602 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7603 self.state.seats.insert(seat.key.clone(), seat);
7604 let text = match outcome {
7605 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7606 AgentOutcome::ResourceBlocked(why) => {
7607 if self.state.resource_block.is_none() {
7610 self.state.resource_block = Some(crate::run::ResourceBlock {
7611 node: "pr-summary".to_owned(),
7612 kind: "shared resource".to_owned(),
7613 detail: why,
7614 at: Timestamp::now(),
7615 });
7616 }
7617 return Err("a shared resource was busy".to_owned());
7618 }
7619 _ => continue,
7620 };
7621 let blind = &self.state.config.blind;
7622 let Some(title) = summary_title(&text) else {
7623 continue;
7624 };
7625 let body: String = summary_without_title(&text)
7626 .lines()
7627 .filter(|l| !l.contains("magi:run/"))
7628 .collect::<Vec<_>>()
7629 .join("\n");
7630 let body = blind::sanitize_prose(body.trim(), blind);
7631 let title = blind::sanitize_prose(&title, blind);
7632 if body.trim().is_empty() || title.trim().is_empty() {
7633 continue;
7634 }
7635 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7636 continue;
7637 }
7638 if git::rev_parse(&winner.worktree, "HEAD")
7640 .await
7641 .ok()
7642 .as_deref()
7643 != Some(head)
7644 {
7645 return Err("the branch moved while the summary was written".to_owned());
7646 }
7647 if git::merge_base(&winner.worktree, &tracking, head)
7648 .await
7649 .ok()
7650 .as_deref()
7651 != Some(base.as_str())
7652 || crate::pr_outcome::summary_evidence(
7653 &self.state,
7654 crate::pr_outcome::FinalHead::At(head),
7655 ) != evidence
7656 {
7657 return Err("the summary evidence changed while it was written".to_owned());
7658 }
7659 self.state.change_summary = Some(crate::run::ChangeSummary {
7660 head: head.to_owned(),
7661 base,
7662 evidence: fingerprint,
7663 title,
7664 body,
7665 });
7666 self.state.event(
7667 "pr-summary",
7668 format!("change summary written by {} for the final diff", spec.id),
7669 );
7670 if let Err(e) = self.state.save() {
7671 tracing::warn!("could not save the change summary: {e:#}");
7672 }
7673 return Ok(());
7674 }
7675 self.state
7676 .event("pr-summary", "no agent produced a change summary");
7677 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7678 }
7679
7680 async fn rewrite_pr_text(
7688 &mut self,
7689 winner: &Candidate,
7690 pr: &PrMessage,
7691 violations: &[crate::github_text::Violation],
7692 ) -> Option<String> {
7693 let prompt = format!(
7694 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7695 pr.title, pr.body
7696 );
7697 let sessions = self.state.config.graph.sessions;
7698 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7699 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7701 let key = format!("impl-{}", winner.label);
7702 if let Some(spec) = self
7703 .state
7704 .config
7705 .agents
7706 .iter()
7707 .find(|a| a.id == winner.agent)
7708 .cloned()
7709 && let Some(existing) = self.state.seats.get(&key)
7710 && existing.agent == spec.id
7711 && agent::has_session(spec.kind, existing, sessions)
7712 {
7713 attempts.push((spec, existing.clone(), true));
7714 } else {
7715 let chain = agent::pick_chain(
7716 &self.state.config.agents,
7717 self.state.config.roles.language_judge.as_ref(),
7718 &agent::installed,
7719 "language_judge",
7720 );
7721 match chain {
7722 Ok(chain) => {
7723 for spec in chain {
7724 let seat = crate::fixer::seat_for(
7725 &mut self.state,
7726 "github-text-rewrite",
7727 &spec.id,
7728 );
7729 attempts.push((spec, seat, false));
7730 }
7731 }
7732 Err(e) => {
7733 tracing::warn!("no agent for the description rewrite: {e:#}");
7734 self.state
7735 .event("github-text", "no rewrite agent available");
7736 return None;
7737 }
7738 }
7739 }
7740 let prompts = self.state.config.prompts.clone();
7741 let cache = self.state.config.cache_dir();
7742 let run = self.state.id.clone();
7743 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7744 let job = SeatJob {
7745 spec: spec.clone(),
7746 seat,
7747 cwd: winner.worktree.clone(),
7748 prompt: prompt.clone(),
7749 timeout: retry_budget(timeout_full, resumed),
7750 allow_write: false,
7751 sessions,
7752 artifacts: agent::artifacts_dir(&self.state.dir()),
7753 stem: if i == 0 {
7754 "github-text-rewrite".to_owned()
7755 } else {
7756 format!("github-text-rewrite-{}", spec.id)
7757 },
7758 handover: None,
7759 };
7760 let ctx = WaveCtx {
7761 run: &run,
7762 node: "github-text",
7763 prompts: &prompts,
7764 cache: cache.as_deref(),
7765 round: None,
7766 carry_seats: false,
7767 pauses: Some((&self.pause, &self.interrupt)),
7768 };
7769 let (seat, outcome) =
7770 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7771 self.state.seats.insert(seat.key.clone(), seat);
7772 match outcome {
7773 AgentOutcome::Ok(o)
7774 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7775 {
7776 return Some(o.text);
7777 }
7778 AgentOutcome::ResourceBlocked(_) => return None,
7779 _ => {}
7780 }
7781 }
7782 None
7783 }
7784
7785 fn retire_github_text_question(&mut self, why: &str) {
7787 let Some(id) = self
7788 .state
7789 .github_text
7790 .as_ref()
7791 .and_then(|g| g.question.clone())
7792 else {
7793 return;
7794 };
7795 let _ = ask::Questions::open().update(&id, |q| {
7796 q.abandon(why);
7797 Ok(())
7798 });
7799 }
7800
7801 async fn settle_withheld(
7805 &mut self,
7806 winner: &Candidate,
7807 pr: &PrMessage,
7808 mut g: crate::run::GithubTextAsk,
7809 ) -> Result<Option<PrMessage>> {
7810 use crate::github_text as gt;
7811 let store = ask::Questions::open();
7812 let timeout = self.state.config.graph.answer_timeout;
7813 if !g.resolved {
7814 let mut standing = g
7815 .question
7816 .as_ref()
7817 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7818 if let Some(q) = standing.as_ref()
7819 && q.status.open()
7820 && gt::expired(q, timeout)
7821 {
7822 let why = format!("no answer within {}s of asking", timeout.max(1));
7823 standing = store
7824 .update(&q.id, |q| {
7825 q.abandon(&why);
7826 Ok(())
7827 })
7828 .ok()
7829 .map(|(q, ())| q);
7830 }
7831 let mut ask_again: Option<Option<Vec<String>>> = None;
7832 match standing {
7833 None if g.asks <= 1 => {
7836 g.asks = 0;
7837 ask_again = Some(None);
7838 }
7839 None => {
7840 g.resolved = true;
7841 g.chosen_title = None;
7842 g.choice = Some(GithubTextChoice::Fallback);
7843 }
7844 Some(q) if q.status.open() => {
7845 self.park_on_github_text(&g)?;
7846 return Ok(None);
7847 }
7848 Some(q) => match gt::read_reply(&q) {
7849 gt::Reply::Fallback => {
7850 g.resolved = true;
7851 g.chosen_title = None;
7852 g.choice = Some(GithubTextChoice::Fallback);
7853 }
7854 gt::Reply::Original => {
7855 g.resolved = true;
7856 g.chosen_title = None;
7857 g.choice = Some(GithubTextChoice::Original);
7858 }
7859 gt::Reply::Rewrite => {
7860 g.resolved = true;
7861 g.chosen_title = None;
7862 g.choice = Some(if g.rewrite.is_some() {
7863 GithubTextChoice::Rewrite
7864 } else {
7865 GithubTextChoice::Fallback
7866 });
7867 }
7868 gt::Reply::Title(text) => {
7869 let first = text
7870 .lines()
7871 .map(str::trim)
7872 .find(|l| !l.is_empty())
7873 .unwrap_or("");
7874 let decision = if first.is_empty() {
7875 None
7876 } else {
7877 let candidate = PrMessage {
7878 title: first.to_owned(),
7879 body: String::new(),
7880 };
7881 self.judge_pr_language(&winner.worktree, &candidate).await
7882 };
7883 match gt::vet_title(
7884 &text,
7885 decision,
7886 self.state.config.graph.github_english(),
7887 ) {
7888 Ok(title) => {
7889 g.resolved = true;
7890 g.chosen_title = Some(title);
7891 g.choice = Some(GithubTextChoice::MyTitle);
7892 }
7893 Err(cats) if g.asks < 2 => {
7894 ask_again = Some(Some(
7895 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7896 ))
7897 }
7898 Err(_) => {
7899 g.resolved = true;
7900 g.chosen_title = None;
7901 g.choice = Some(GithubTextChoice::Fallback);
7902 }
7903 }
7904 }
7905 },
7906 }
7907 if let Some(rejected) = ask_again {
7908 let w = gt::Withheld {
7911 title: g.title,
7912 body: g.body,
7913 categories: g.categories.clone(),
7914 };
7915 let artifact = match crate::run::write_artifact(
7918 &self.state,
7919 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7920 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7921 ) {
7922 Ok(p) => Some(p),
7923 Err(e) => {
7924 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7925 None
7926 }
7927 };
7928 let (detail, comparison) = gt::question_content(
7929 &w,
7930 &pr.title,
7931 &pr.body,
7932 &self.state.id,
7933 rejected.as_deref(),
7934 artifact.as_deref(),
7935 g.rewrite.as_ref(),
7936 );
7937 let mut q = ask::Question::new(
7938 self.state.id.clone(),
7939 gt::ASK_NODE.to_owned(),
7940 gt::ASK_SEAT.to_owned(),
7941 gt::question_summary(&self.state.config.graph.language, &w),
7942 detail,
7943 gt::question_choices(&w, g.rewrite.as_ref()),
7944 );
7945 q.posting_comparison = Some(comparison);
7946 q.answer_timeout = timeout;
7947 g.question = Some(q.id.clone());
7950 g.asks += 1;
7951 self.state.github_text = Some(g.clone());
7952 self.state.event(
7953 "github-text",
7954 format!("asking the owner about the withheld text ({})", q.short()),
7955 );
7956 self.state.save()?;
7957 store
7958 .put_executable(&mut q)
7959 .context("file the posting-gate question")?;
7960 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7961 tracing::warn!(
7962 "could not notify about posting-gate question {}: {e:#}",
7963 q.short()
7964 );
7965 }
7966 self.park_on_github_text(&g)?;
7967 return Ok(None);
7968 }
7969 }
7970 let resolved = g.chosen_title.clone();
7971 let original = g.choice == Some(GithubTextChoice::Original);
7972 self.state.event(
7974 "github-text",
7975 if original {
7976 format!(
7977 "posting the original text for: {}",
7978 match (g.title, g.body) {
7979 (true, true) => "title, description",
7980 (true, false) => "title",
7981 _ => "description",
7982 }
7983 )
7984 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7985 "posting the saved rewrite title and description".to_owned()
7986 } else if resolved.is_some() {
7987 "posting the owner's replacement title".to_owned()
7988 } else {
7989 "posting the neutral text for the withheld fields".to_owned()
7990 },
7991 );
7992 self.state.github_text = Some(g.clone());
7993 self.state.save()?;
7994 if g.choice == Some(GithubTextChoice::Rewrite)
7995 && let Some(r) = g.rewrite.as_ref()
7996 {
7997 return Ok(Some(PrMessage {
7998 title: r.title.clone(),
7999 body: r.body.clone(),
8000 }));
8001 }
8002 let w = gt::Withheld {
8003 title: g.title,
8004 body: g.body,
8005 categories: g.categories.clone(),
8006 };
8007 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8008 if let Some(t) = resolved {
8009 title = t;
8010 }
8011 let english = gt::LanguageDecision {
8013 title_english: true,
8014 body_english: true,
8015 body_complete: true,
8016 };
8017 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8018 if original {
8019 if g.title {
8022 title = pr.title.clone();
8023 }
8024 if g.body {
8025 body = pr.body.clone();
8026 }
8027 }
8028 Ok(Some(PrMessage { title, body }))
8029 }
8030
8031 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8032 self.state.github_text = Some(g.clone());
8033 self.state.end_pass_parked(ParkCause::Question);
8034 self.state.parked = true;
8035 self.state.event(
8036 "github-text",
8037 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8038 );
8039 self.state.save()
8040 }
8041
8042 async fn judge_pr_language(
8045 &mut self,
8046 cwd: &Path,
8047 pr: &PrMessage,
8048 ) -> Option<crate::github_text::LanguageDecision> {
8049 self.state.config.roles.language_judge.as_ref()?;
8050 if !self.state.config.graph.language_gate() {
8051 return None;
8052 }
8053 let decision =
8054 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8055 self.state.event(
8056 "github-text",
8057 if decision.is_some() {
8058 "language decided by the language judge"
8059 } else {
8060 "language judge unavailable; using built-in heuristics"
8061 },
8062 );
8063 decision
8064 }
8065
8066 async fn merge(&mut self) -> Result<()> {
8067 if self
8082 .state
8083 .base_sync
8084 .as_ref()
8085 .is_some_and(|s| s.conflict.is_some())
8086 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8087 != Some(RunStatus::Gating)
8088 || !self.state.gate_allows_progress()
8096 {
8097 return Ok(());
8098 }
8099 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8108 return Ok(());
8109 }
8110 let Some(winner) = self.state.winner().cloned() else {
8111 return Ok(());
8112 };
8113 let repo = self.state.repo.clone();
8114 let base = self.state.base_branch.clone();
8115 let mode = self.state.config.merge.mode;
8116 let style = self.state.config.merge.style;
8117 let facts = if is_review_run(&self.state) {
8118 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8119 let start = review_base(
8120 &repo,
8121 &self.state.config.merge.remote,
8122 &base,
8123 &self.state.base_commit,
8124 &winner.branch,
8125 )
8126 .await;
8127 branch_facts(&repo, &start, &winner.branch).await
8128 } else {
8129 None
8130 };
8131 let leaked = if is_review_run(&self.state) {
8134 leaked_subjects(&self.state, &winner.branch).await
8135 } else {
8136 Some(Vec::new())
8137 };
8138 let Some(pr) = self
8142 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8143 .await?
8144 else {
8145 return Ok(());
8146 };
8147 let message = pr.commit_message();
8148
8149 let outcome = match mode {
8150 MergeMode::None => MergeOutcome {
8151 mode,
8152 ok: true,
8153 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8154 empty: false,
8155 },
8156 MergeMode::Pr | MergeMode::Local
8157 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8158 {
8159 MergeOutcome {
8160 mode,
8161 ok: false,
8162 detail: empty_candidate_detail(&self.state, &base),
8163 empty: true,
8164 }
8165 }
8166 MergeMode::Local => {
8167 let on = git::current_branch(&repo).await?;
8168 if on.as_deref() != Some(base.as_str()) {
8169 MergeOutcome {
8170 mode,
8171 ok: false,
8172 detail: format!(
8173 "{} has {} checked out, not the base branch {base}",
8174 repo.display(),
8175 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8176 ),
8177 empty: false,
8178 }
8179 } else if !git::is_clean(&repo).await? {
8180 MergeOutcome {
8181 mode,
8182 ok: false,
8183 detail: format!("{} is dirty; refusing to merge", repo.display()),
8184 empty: false,
8185 }
8186 } else {
8187 let out = match style {
8188 MergeStyle::Merge => {
8189 git::merge_no_ff(&repo, &winner.branch, &message).await?
8190 }
8191 MergeStyle::Squash => {
8192 git::merge_squash(&repo, &winner.branch, &message).await?
8193 }
8194 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8195 };
8196 MergeOutcome {
8197 mode,
8198 ok: out.ok(),
8199 detail: if out.ok() { out.stdout } else { out.stderr },
8200 empty: false,
8201 }
8202 }
8203 }
8204 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8205 self.state.pr_publication_pending = true;
8206 self.state.save()?;
8207 let out =
8208 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8209 self.state.pr_publication_pending = out.is_err();
8210 MergeOutcome {
8211 mode,
8212 ok: out.is_ok(),
8213 detail: match out {
8214 Ok(url) => url,
8215 Err(e) => format!("{e:#}"),
8216 },
8217 empty: false,
8218 }
8219 }
8220 MergeMode::Pr => {
8221 let remote = self.state.config.merge.remote.clone();
8222 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8223 if !pushed.ok() {
8224 MergeOutcome {
8225 mode,
8226 ok: false,
8227 detail: pushed.stderr,
8228 empty: false,
8229 }
8230 } else {
8231 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8237 let out = match pr_merge_plan(found) {
8238 PrPlan::Create => {
8239 gh_pr_create(
8240 &winner.worktree,
8241 &base,
8242 &winner.branch,
8243 &pr.title,
8244 &pr.body,
8245 )
8246 .await
8247 }
8248 PrPlan::Adopt { url, title } => {
8249 self.state
8250 .event("merge", format!("Pr: adopted open pull request {url}"));
8251 if self.state.github_text.as_ref().is_some_and(|g| {
8252 g.resolved
8253 && g.choice == Some(GithubTextChoice::Rewrite)
8254 && g.rewrite.is_some()
8255 }) {
8256 land::set_pr_text_verbatim(
8259 &winner.worktree,
8260 &url,
8261 &pr.title,
8262 &pr.body,
8263 )
8264 .await
8265 .map(|()| url)
8266 } else {
8267 if title != pr.title
8268 && (!is_review_run(&self.state)
8269 || leaked
8270 .as_deref()
8271 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8272 && let Err(e) =
8273 if self.state.github_text.as_ref().is_some_and(|g| {
8274 g.resolved
8275 && g.title
8276 && g.choice == Some(GithubTextChoice::Original)
8277 }) {
8278 land::set_pr_title_verbatim(
8280 &winner.worktree,
8281 &url,
8282 &pr.title,
8283 )
8284 .await
8285 } else {
8286 land::set_pr_title(
8287 &mut self.state,
8288 &winner.worktree,
8289 &url,
8290 &pr.title,
8291 )
8292 .await
8293 }
8294 {
8295 tracing::warn!("could not refresh title of {url}: {e:#}");
8296 self.state
8297 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8298 }
8299 Ok(url)
8300 }
8301 }
8302 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8303 };
8304 match out {
8305 Ok(url) => MergeOutcome {
8306 mode,
8307 ok: true,
8308 detail: url,
8309 empty: false,
8310 },
8311 Err(e) => MergeOutcome {
8312 mode,
8313 ok: false,
8314 detail: e.to_string(),
8315 empty: false,
8316 },
8317 }
8318 }
8319 }
8320 };
8321
8322 self.state.status = match (mode, outcome.ok) {
8323 (MergeMode::None, _) => RunStatus::Ready,
8324 (_, true) if outcome.empty => RunStatus::Ready,
8325 (MergeMode::Pr, true) => RunStatus::Ready,
8326 (_, true) => RunStatus::Merged,
8327 (_, false) => RunStatus::Blocked,
8328 };
8329 self.state.event(
8330 "merge",
8331 format!(
8332 "{:?}: {}",
8333 mode,
8334 outcome.detail.lines().next().unwrap_or("")
8335 ),
8336 );
8337 self.state.merge = Some(outcome);
8338 self.state.save()?;
8339
8340 if self.state.config.graph.land
8346 && mode == MergeMode::Pr
8347 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8348 {
8349 self.run_land().await?;
8350 }
8351 self.settle_questions();
8356 Ok(())
8357 }
8358
8359 async fn run_land(&mut self) -> Result<()> {
8370 crate::pr_head::ensure_owner(&self.state)?;
8371 let url = self
8372 .state
8373 .merge
8374 .as_ref()
8375 .map(|m| m.detail.clone())
8376 .unwrap_or_default();
8377 let url = url.lines().next().unwrap_or("").trim().to_owned();
8378 if !url.starts_with("http") {
8379 return Ok(());
8380 }
8381 loop {
8384 match land::land(&mut self.state, &url).await {
8385 Ok(_)
8386 if self
8387 .state
8388 .merge_repairs
8389 .last()
8390 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8391 {
8392 if self.repair_merge_review().await? {
8393 continue;
8394 }
8395 }
8396 Ok(pr) if self.state.parked => {
8397 let _ = pr;
8401 }
8402 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8403 self.state.status = RunStatus::Superseded;
8404 self.state.save()?;
8405 }
8406 Ok(pr) => {
8407 self.state.status = match pr.state {
8408 land::PrLifecycle::Merged => RunStatus::Merged,
8409 _ => RunStatus::Blocked,
8410 };
8411 if bump::should_release_bump(self.state.status)
8418 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8419 {
8420 self.state
8426 .event("bump", format!("release bump skipped: {e:#}"));
8427 }
8428 if self.state.status == RunStatus::Merged {
8431 crate::followup::after_merge(&mut self.state, &pr.url).await;
8432 }
8433 self.state.save()?;
8434 }
8435 Err(e) => {
8436 if e.is::<crate::repo_lock::ResourceBlocked>() {
8437 return Err(e);
8438 }
8439 self.state.status = RunStatus::Blocked;
8440 self.state.event("land", format!("gave up: {e}"));
8441 self.state.save()?;
8442 }
8443 }
8444 break;
8445 }
8446 Ok(())
8447 }
8448
8449 async fn repair_merge_review(&mut self) -> Result<bool> {
8452 use crate::run::MergeRepairPhase as Phase;
8453 crate::pr_head::ensure_owner(&self.state)?;
8454 let i = self.state.merge_repairs.len() - 1;
8455 let request = self.state.merge_repairs[i].clone();
8456 if request.phase == Phase::Fixing {
8457 let restored = match self.state.winner().cloned() {
8458 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8459 None => Err(anyhow::anyhow!("repair has no winning branch")),
8460 };
8461 let why = match restored {
8462 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8463 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8464 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8465 };
8466 self.hold_merge_repair(i, why)?;
8467 return Ok(false);
8468 }
8469 let failure = if i >= self.state.config.graph.land_rounds
8470 || self.state.config.graph.review_rounds == 0
8471 {
8472 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8473 } else {
8474 None
8475 };
8476 if let Some(why) = failure {
8477 self.hold_merge_repair(i, why)?;
8478 return Ok(false);
8479 }
8480 if request.phase == Phase::Requested {
8481 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8482 Ok(pr) => pr,
8483 Err(e) => {
8484 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8485 return Ok(false);
8486 }
8487 };
8488 self.state.merge_repairs[i].phase = Phase::Fixing;
8489 self.state.land_approval = None;
8490 self.state.land_armed_head = None;
8491 self.state.save()?;
8492 let evidence = serde_json::to_string_pretty(&request)?;
8495 let reason = format!(
8496 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8497 );
8498 let budget = self.state.config.graph.land_rounds;
8499 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8500 match result {
8501 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8502 Ok(land::Fixed::Failed(why)) => {
8503 self.hold_merge_repair(
8504 i,
8505 format!(
8506 "repair failed: {why}; inspect the branch and run an explicit review"
8507 ),
8508 )?;
8509 return Ok(false);
8510 }
8511 Err(e) => {
8512 self.hold_merge_repair(
8513 i,
8514 format!(
8515 "repair failed: {e:#}; inspect the branch and run an explicit review"
8516 ),
8517 )?;
8518 return Ok(false);
8519 }
8520 }
8521 self.begin_merge_repair_review(i)?;
8525 }
8526 self.review_loop().await?;
8527 self.gate().await?;
8528 if self.resource_halt()? || self.state.parked {
8529 return Ok(false);
8530 }
8531 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8532 != Some(RunStatus::Gating)
8533 || !self.state.gate_allows_progress()
8534 {
8535 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8536 return Ok(false);
8537 }
8538 let winner = self
8541 .state
8542 .winner()
8543 .cloned()
8544 .context("repair has no winning branch")?;
8545 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8546 if self.state.gate_head.as_deref() != Some(after.as_str())
8547 || !git::is_clean(&winner.worktree).await?
8548 {
8549 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8550 return Ok(false);
8551 }
8552 let expected = self
8553 .state
8554 .pr_pushed_head
8555 .as_deref()
8556 .unwrap_or(&request.head)
8557 .to_owned();
8558 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8559 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8562 let refspec = format!("{after}:refs/heads/{public}");
8563 let push = git::git_raw(
8564 &winner.worktree,
8565 &["push", &lease, &self.state.config.merge.remote, &refspec],
8566 )
8567 .await;
8568 match push {
8569 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8570 Ok(out) => {
8571 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8572 return Ok(false);
8573 }
8574 Err(e) => {
8575 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8576 return Ok(false);
8577 }
8578 }
8579 self.state.merge_repairs[i].phase = Phase::Complete;
8580 self.state.status = RunStatus::Landing;
8581 self.state.event(
8582 "repair",
8583 "new review and gate completed; fresh owner merge decision required",
8584 );
8585 self.state.save()?;
8586 Ok(true)
8587 }
8588
8589 fn merge_repair_artifacts(&self) -> PathBuf {
8590 let artifacts = agent::artifacts_dir(&self.state.dir());
8591 match self
8592 .state
8593 .merge_repairs
8594 .last()
8595 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8596 {
8597 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8598 None => artifacts,
8599 }
8600 }
8601
8602 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8603 self.state.reviews.clear();
8604 self.state.gate.clear();
8605 self.state.gate_ran = false;
8606 self.state.gate_head = None;
8607 self.state.gate_fixes.clear();
8608 self.state.pre_gate.clear();
8609 self.state.pre_gate_commit = None;
8610 self.state.contested_handoff = None;
8611 self.state.land_reruns = None;
8612 self.state.land_approval = None;
8613 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8614 self.state.status = RunStatus::Reviewing;
8615 self.state.save()
8616 }
8617
8618 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8619 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8620 self.state.merge_repairs[index].failure = Some(why.clone());
8621 self.state.status = RunStatus::Blocked;
8622 self.state.event("repair", why);
8623 self.state.save()
8624 }
8625
8626 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8630 if let Some(existing) = self.state.seats.get(key)
8631 && existing.agent == agent
8632 {
8633 return existing.clone();
8634 }
8635 let fresh = if self.state.seats.contains_key(key) {
8638 handover_seat(key, agent, self.state.next_seat_seed())
8639 } else {
8640 SeatState::new(key, agent, self.state.seed)
8641 };
8642 self.state.seats.insert(key.to_owned(), fresh.clone());
8643 fresh
8644 }
8645
8646 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8652 match self.state.seats.get(key) {
8653 Some(s) if s.agent != spec.id => {
8654 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8655 }
8656 _ => spec,
8657 }
8658 }
8659
8660 fn view(&self, c: &Candidate) -> CandidateView {
8662 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8663 .unwrap_or_default();
8664 let (patch, _) = blind::sanitize_patch(
8665 &format!("candidate {} patch", c.label),
8666 &raw,
8667 &self.state.config.blind,
8668 );
8669 CandidateView {
8670 label: c.label,
8671 branch: c.branch.clone(),
8672 summary: c.summary.clone(),
8673 stat: c.stat.clone(),
8674 patch,
8675 }
8676 }
8677
8678 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8680 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8681 prompt::judge(
8682 "(see above)",
8683 &views,
8684 self.roles.judges.len(),
8685 base_short,
8686 "en",
8687 )
8688 }
8689
8690 fn vote_prompt_full(
8696 &self,
8697 j: usize,
8698 labels: &[char],
8699 language: &str,
8700 candidates: &[Candidate],
8701 base_short: &str,
8702 ) -> String {
8703 let mut text = format!(
8704 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8705 prompt::final_vote(labels, language),
8706 self.state.instruction,
8707 self.candidate_block(candidates, base_short)
8708 );
8709 if let Some(own) = self
8710 .state
8711 .judgements
8712 .get(j)
8713 .filter(|r| !r.ranking.is_empty())
8714 {
8715 let reasons = own
8716 .reasons
8717 .iter()
8718 .map(|(k, v)| format!("- {k}: {v}"))
8719 .collect::<Vec<_>>()
8720 .join("\n");
8721 text.push_str(&format!(
8722 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8723 own.ranking.iter().collect::<String>(),
8724 if reasons.is_empty() {
8725 ""
8726 } else {
8727 ", because:\n"
8728 }
8729 ));
8730 }
8731 if !self.state.deliberation.is_empty() {
8732 text.push_str("\n# What was argued before this vote\n");
8733 for t in self.transcript(&[], j) {
8734 text.push_str(&format!(
8735 "\n## {}{}\n\n{}\n",
8736 t.who,
8737 if t.is_self { " (you)" } else { "" },
8738 t.body.trim()
8739 ));
8740 }
8741 }
8742 text
8743 }
8744
8745 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8752 let mut turns = Vec::new();
8753 for j in &self.state.judgements {
8754 if j.ranking.is_empty() {
8755 continue;
8756 }
8757 let reasons = j
8758 .reasons
8759 .iter()
8760 .map(|(k, v)| format!("- {k}: {v}"))
8761 .collect::<Vec<_>>()
8762 .join("\n");
8763 turns.push(Turn {
8764 who: format!("Judge {} (opening ranking)", j.judge),
8765 is_self: j.judge == self_idx + 1,
8766 body: format!(
8767 "Ranked {}{}{reasons}",
8768 j.ranking.iter().collect::<String>(),
8769 if reasons.is_empty() {
8770 ""
8771 } else {
8772 ", because:\n"
8773 }
8774 ),
8775 });
8776 }
8777 for t in self
8778 .state
8779 .deliberation
8780 .iter()
8781 .flat_map(|r| r.turns.iter())
8782 .chain(current)
8783 {
8784 turns.push(Turn {
8785 who: format!("Judge {}", t.judge),
8786 is_self: t.judge == self_idx + 1,
8787 body: t.body.clone(),
8788 });
8789 }
8790 turns
8791 }
8792}
8793
8794fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8796 agent::has_session(spec.kind, seat, sessions)
8797}
8798
8799fn next_untried_in_roster<'a>(
8820 roster: &'a [AgentSpec],
8821 start: usize,
8822 tried: &BTreeSet<String>,
8823) -> Option<&'a AgentSpec> {
8824 roster
8825 .get(start + 1..)?
8826 .iter()
8827 .find(|s| !tried.contains(&s.id))
8828}
8829
8830fn pick_successor<'a>(
8851 roster: &'a [AgentSpec],
8852 start: usize,
8853 tried: &BTreeSet<String>,
8854 carried: Option<&BTreeSet<String>>,
8855 others: &BTreeSet<String>,
8856) -> Option<&'a AgentSpec> {
8857 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8858 roster
8859 .get(start + 1..)
8860 .and_then(|tail| {
8861 tail.iter()
8862 .filter(free)
8863 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8864 })
8865 .or_else(|| {
8866 carried
8867 .is_some()
8868 .then(|| roster.iter().find(free))
8869 .flatten()
8870 })
8871 .or_else(|| match carried {
8872 Some(c) => next_for_seat(roster, start, tried, c),
8873 None => next_untried_in_roster(roster, start, tried),
8874 })
8875}
8876
8877fn next_for_seat<'a>(
8887 roster: &'a [AgentSpec],
8888 start: usize,
8889 round_tried: &BTreeSet<String>,
8890 carried_failed: &BTreeSet<String>,
8891) -> Option<&'a AgentSpec> {
8892 roster
8893 .get(start + 1..)?
8894 .iter()
8895 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8896 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8897}
8898
8899fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8905 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8906 return spec;
8907 };
8908 let ok = |id: &str| !h.failed.contains(id);
8909 if let Some(last) = h.last_ok.as_deref()
8910 && ok(last)
8911 && let Some(s) = roster.iter().find(|s| s.id == last)
8912 {
8913 return s.clone();
8914 }
8915 if ok(&spec.id) {
8916 return spec;
8917 }
8918 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8919 roster
8920 .get(from + 1..)
8921 .into_iter()
8922 .flatten()
8923 .chain(roster.iter())
8924 .find(|s| ok(&s.id))
8925 .cloned()
8926 .unwrap_or(spec)
8927}
8928
8929pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8933 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8934}
8935
8936const TIMED_OUT: &str = "timed out";
8939
8940impl FailClass {
8941 fn of(out: &AgentOutcome) -> Option<Self> {
8943 match out {
8944 AgentOutcome::Ok(_) => None,
8945 AgentOutcome::Quota(_) => Some(Self::Quota),
8946 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8947 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8948 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8949 AgentOutcome::ResourceBlocked(_) => None,
8952 }
8953 }
8954
8955 fn stem_word(&self) -> &'static str {
8957 match self {
8958 Self::Quota => "quota",
8959 _ => "handover",
8960 }
8961 }
8962}
8963
8964fn failure_signature(msg: &str) -> String {
8968 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8969 if let Some(code) = line
8972 .strip_prefix("process exited with no usable output (exit code ")
8973 .and_then(|s| s.strip_suffix(')'))
8974 .and_then(|s| s.parse::<i32>().ok())
8975 {
8976 line = format!("exited with some({code}) and no usable output");
8977 } else if matches!(
8978 line.as_str(),
8979 "process exited with no usable output (no exit code reported)"
8980 | "process exited with no usable output (exit code unavailable)"
8981 ) {
8982 line = "exited with none and no usable output".to_owned();
8983 }
8984 let mut out = Vec::new();
8985 for word in line.split_whitespace() {
8986 if word.contains('/') || word.contains('\\') {
8987 out.push("<path>".to_owned());
8988 continue;
8989 }
8990 let mut w = String::new();
8991 let mut in_digits = false;
8992 for c in word.chars() {
8993 if c.is_ascii_digit() {
8994 if !in_digits {
8995 w.push('#');
8996 }
8997 in_digits = true;
8998 } else {
8999 in_digits = false;
9000 w.push(c);
9001 }
9002 }
9003 out.push(w);
9004 }
9005 out.join(" ").chars().take(120).collect()
9006}
9007
9008fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9015 match cur {
9016 FailClass::Quota | FailClass::Timeout => true,
9017 FailClass::Other(_) => prev != Some(cur),
9018 }
9019}
9020
9021fn fail_reason(out: &AgentOutcome) -> String {
9023 match out {
9024 AgentOutcome::Ok(_) => String::new(),
9025 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9026 AgentOutcome::Dropped(o) => format!(
9027 "the CLI dropped the stream ({})",
9028 o.dropped
9029 .as_ref()
9030 .map(|d| d.why.as_str())
9031 .unwrap_or("it ended without delivering its answer")
9032 ),
9033 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9034 e.lines().next().unwrap_or("").chars().take(160).collect()
9035 }
9036 }
9037}
9038
9039pub(crate) fn record_handover(
9042 state: &mut RunState,
9043 node: &str,
9044 seat: &str,
9045 from: &str,
9046 to: &str,
9047 class: &FailClass,
9048 reason: &str,
9049) {
9050 let message = if *class == FailClass::Quota {
9051 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9052 } else {
9053 format!("{seat}: handed over {from} -> {to} ({reason})")
9054 };
9055 state.event(node, message);
9056 state.handovers.push(Handover {
9057 at: Timestamp::now(),
9058 node: node.to_owned(),
9059 seat: seat.to_owned(),
9060 from: from.to_owned(),
9061 to: to.to_owned(),
9062 reason: reason.to_owned(),
9063 });
9064}
9065
9066fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9080 commands.iter().any(|c| c.exit_code.is_none())
9081}
9082
9083fn verified_noop_claim(
9096 usable: bool,
9097 commands: &[agent::CommandEvidence],
9098 text: &str,
9099) -> Option<String> {
9100 (usable && !has_unconfirmed_command(commands))
9101 .then(|| verdict::verified_noop(text))
9102 .flatten()
9103}
9104
9105fn short(commit: &str) -> String {
9106 commit.chars().take(7).collect()
9107}
9108
9109fn make_executable(path: &Path) -> Result<()> {
9110 #[cfg(unix)]
9111 {
9112 use std::os::unix::fs::PermissionsExt as _;
9113 let mut perms = std::fs::metadata(path)?.permissions();
9114 perms.set_mode(0o755);
9115 std::fs::set_permissions(path, perms)?;
9116 }
9117 #[cfg(not(unix))]
9118 {
9119 let _ = path;
9120 }
9121 Ok(())
9122}
9123
9124struct WaveCtx<'a> {
9131 run: &'a str,
9134 node: &'a str,
9136 prompts: &'a Prompts,
9137 cache: Option<&'a Path>,
9139 round: Option<usize>,
9142 carry_seats: bool,
9146 pauses: Option<(&'a Pause, &'a Pause)>,
9149}
9150
9151fn repository_wait(
9153 state: &RunState,
9154 pauses: Option<(&Pause, &Pause)>,
9155) -> crate::repo_lock::WaitPolicy {
9156 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9157 let inherited = crate::repo_lock::WaitPolicy::current();
9158 crate::repo_lock::WaitPolicy::new(
9159 Duration::from_secs(state.config.graph.timeout_resource_wait),
9160 move || {
9161 inherited.cancelled()
9162 || pauses
9163 .as_ref()
9164 .is_some_and(|(a, b)| a.parked() || b.parked())
9165 },
9166 )
9167}
9168
9169fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9170 if error.is::<crate::repo_lock::ResourceBlocked>() {
9171 state.resource_block = Some(crate::run::ResourceBlock {
9172 node: "repository".into(),
9173 kind: "repository coordination".into(),
9174 detail: error.to_string(),
9175 at: Timestamp::now(),
9176 });
9177 state.event("resource", error.to_string());
9178 }
9179}
9180
9181async fn coordinate_operation(
9182 state: &mut RunState,
9183 cwd: &Path,
9184 wait: &crate::repo_lock::WaitPolicy,
9185) -> Result<Option<crate::repo_lock::SeatLease>> {
9186 let result =
9187 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9188 .await;
9189 if let Err(e) = &result {
9190 record_repository_block(state, e);
9191 }
9192 result
9193}
9194
9195async fn run_one(
9197 job: SeatJob,
9198 sem: Arc<Semaphore>,
9199 ctx: &WaveCtx<'_>,
9200 state: &mut RunState,
9201 attempt: usize,
9202) -> (SeatState, AgentOutcome) {
9203 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9204 .await
9205 .pop()
9206 .expect("one job in, one result out");
9207 (seat, out)
9208}
9209
9210async fn wave(
9216 jobs: Vec<SeatJob>,
9217 sem: Arc<Semaphore>,
9218 ctx: &WaveCtx<'_>,
9219 state: &mut RunState,
9220 attempt: usize,
9221) -> Vec<(usize, SeatState, AgentOutcome)> {
9222 let WaveCtx {
9223 run,
9224 node,
9225 prompts,
9226 cache,
9227 round,
9228 carry_seats: _,
9229 pauses,
9230 } = *ctx;
9231 for job in &jobs {
9232 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9233 }
9234 if let Err(e) = state.save() {
9235 tracing::warn!("could not persist in-progress seats: {e:#}");
9240 }
9241 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9257 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9261 let cache_guard = if let Some(cache_dir) = cache {
9262 if jobs_had_a_writer {
9263 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9264 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9265 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9266 {
9267 Ok(Some(g)) => Some(g),
9268 Ok(None) => {
9271 state.resource_block = Some(crate::run::ResourceBlock {
9272 node: node.to_owned(),
9273 kind: "shared build cache".to_owned(),
9274 detail: "the run is parking".to_owned(),
9275 at: Timestamp::now(),
9276 });
9277 return block_wave(
9278 jobs,
9279 state,
9280 node,
9281 attempt,
9282 "parked while waiting for the shared build cache",
9283 );
9284 }
9285 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9288 Err(e) => {
9289 let why = format!("{e:#}");
9293 state.resource_block = Some(crate::run::ResourceBlock {
9294 node: node.to_owned(),
9295 kind: "shared build cache".to_owned(),
9296 detail: why.clone(),
9297 at: Timestamp::now(),
9298 });
9299 state.event(
9300 "resource",
9301 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9302 );
9303 return block_wave(jobs, state, node, attempt, &why);
9304 }
9305 }
9306 } else {
9307 None
9308 }
9309 } else {
9310 None
9311 };
9312 state.seats_spawning(node, attempt);
9315 if let Err(e) = state.save() {
9316 tracing::warn!("could not persist the start of a wave: {e:#}");
9317 }
9318 let slot_limit = resource_limit
9322 + jobs
9323 .iter()
9324 .map(|j| j.timeout)
9325 .max()
9326 .unwrap_or_default()
9327 .saturating_mul(jobs.len() as u32);
9328 let mut set = tokio::task::JoinSet::new();
9329 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9332 let overlay = prompts.overlay(node);
9333 let repository_wait = repository_wait(state, pauses);
9334 for (i, mut job) in jobs.into_iter().enumerate() {
9335 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9336 if cache.is_some() {
9337 job.prompt.push('\n');
9338 job.prompt
9339 .push_str(&prompt::build_cache_note(node, job.allow_write));
9340 }
9341 let sem = Arc::clone(&sem);
9342 let run = run.to_owned();
9343 let node = node.to_owned();
9344 let attachments = if node == "implement" {
9347 state.attachments.clone()
9348 } else {
9349 Vec::new()
9350 };
9351 let cache = cache
9362 .filter(|_| job.allow_write && cache_guard.is_some())
9363 .map(Path::to_path_buf);
9364 let spawned_tx = spawned_tx.clone();
9365 let repository_wait = repository_wait.clone();
9366 let coordination = state.config.coordination.clone();
9367 let fetched_base = state.fetched_base.clone();
9368 set.spawn(async move {
9369 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9373 Ok(p) => p,
9374 Err(_) => {
9375 return (
9376 i,
9377 job.seat,
9378 AgentOutcome::ResourceBlocked(format!(
9379 "no parallel slot became free within {}s",
9380 slot_limit.as_secs()
9381 )),
9382 );
9383 }
9384 };
9385 spawned_tx.send(job.seat.key.clone()).ok();
9386 let mut seat = job.seat;
9387 let out = repository_wait
9388 .scope(agent::invoke(
9389 &job.spec,
9390 &mut seat,
9391 &Invocation {
9392 scope: crate::agent::InvocationScope::Repository,
9393 coordination: coordination.as_ref(),
9394 fetched_base: fetched_base.as_ref(),
9395 cwd: &job.cwd,
9396 prompt: &job.prompt,
9397 timeout: job.timeout,
9398 allow_write: job.allow_write,
9399 unsandboxed: false,
9400 sessions: job.sessions,
9401 artifacts: &job.artifacts,
9402 stem: &job.stem,
9403 run: &run,
9404 node: &node,
9405 cache_dir: cache.as_deref(),
9406 attachments: &attachments,
9407 writable: &[],
9408 },
9409 ))
9410 .await;
9411 let out = match out {
9412 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9413 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9414 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9422 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9423 Ok(o) => AgentOutcome::Failed(format!(
9424 "process exited with no usable output ({})",
9425 crate::notices::exit_status(o.exit_code)
9426 )),
9427 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9428 AgentOutcome::ResourceBlocked(e.to_string())
9429 }
9430 Err(e) => AgentOutcome::Failed(e.to_string()),
9431 };
9432 (i, seat, out)
9433 });
9434 }
9435 drop(spawned_tx);
9436 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9437 loop {
9438 let joined = tokio::select! {
9439 Some(key) = spawned_rx.recv() => {
9440 state.seat_spawned(&key);
9442 if let Err(e) = state.save() {
9443 tracing::warn!("could not persist a seat's start: {e:#}");
9444 }
9445 continue;
9446 }
9447 joined = set.join_next() => joined,
9448 };
9449 let Some(joined) = joined else {
9450 break;
9451 };
9452 let (i, seat, out) = match joined {
9453 Ok(v) => v,
9454 Err(e) => {
9458 tracing::error!("agent task panicked: {e}");
9459 continue;
9460 }
9461 };
9462 state.seat_finished(&seat.key);
9463 if let AgentOutcome::ResourceBlocked(why) = &out
9464 && state.resource_block.is_none()
9465 {
9466 state.resource_block = Some(crate::run::ResourceBlock {
9467 node: node.to_owned(),
9468 kind: if why.contains("repository coordination") {
9469 "repository coordination"
9470 } else {
9471 "parallel slot"
9472 }
9473 .to_owned(),
9474 detail: why.clone(),
9475 at: Timestamp::now(),
9476 });
9477 state.event("resource", format!("{node}: no agent was started - {why}"));
9478 }
9479 record_jobs(state, node, round, &seat.key, &out);
9480 if let Err(e) = state.save() {
9481 tracing::warn!("could not persist a seat's completion: {e:#}");
9482 }
9483 if collected.len() <= i {
9484 collected.resize_with(i + 1, || None);
9485 }
9486 collected[i] = Some((i, seat, out));
9487 }
9488 if state
9494 .active
9495 .values()
9496 .any(|a| a.node == node && a.attempt == attempt)
9497 {
9498 state
9499 .active
9500 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9501 if let Err(e) = state.save() {
9502 tracing::warn!("could not persist the end of a wave: {e:#}");
9503 }
9504 }
9505 if let Some(cache_dir) = cache
9512 && jobs_had_a_writer
9513 {
9514 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9515 }
9516 if let Some(guard) = cache_guard {
9517 guard.release();
9518 }
9519 collected.into_iter().flatten().collect()
9520}
9521
9522fn record_jobs(
9533 state: &mut RunState,
9534 node: &str,
9535 round: Option<usize>,
9536 seat: &str,
9537 out: &AgentOutcome,
9538) {
9539 let commands: &[agent::CommandEvidence] = match out {
9540 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9541 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9542 };
9543 let checked_at = Timestamp::now();
9544 for c in commands {
9545 state.jobs.push(JobRecord {
9546 node: node.to_owned(),
9547 round,
9548 seat: seat.to_owned(),
9549 id: c.id.clone(),
9550 description: c.description.clone(),
9551 checked_at,
9552 status: match c.exit_code {
9553 Some(0) => JobStatus::Completed,
9554 Some(_) => JobStatus::Failed,
9555 None => JobStatus::Unknown,
9556 },
9557 exit_code: c.exit_code,
9558 result_summary: c.result_summary.clone(),
9559 source: c.source.clone(),
9560 });
9561 }
9562}
9563
9564fn round_is_clean(
9585 blocking: usize,
9586 e2e_ok: bool,
9587 answered: usize,
9588 expected: usize,
9589 quota_missing: usize,
9590 policy: IncompleteReviewPolicy,
9591) -> bool {
9592 if blocking != 0 || !e2e_ok {
9593 return false;
9594 }
9595 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9596 return true;
9597 }
9598 answered > 0 && expected - answered <= quota_missing
9599}
9600
9601fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9625 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9626 return Some(RunStatus::Gating);
9627 }
9628 let last = reviews.last()?;
9629 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9630 if reviews.len() < max_rounds && !stagnant {
9631 return None;
9632 }
9633 if last.incomplete() && last.blocking == 0 {
9634 return Some(RunStatus::Blocked);
9635 }
9636 if last.e2e_status() == E2eStatus::ResourceBlocked {
9637 return None;
9638 }
9639 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9640 RunStatus::Gating
9641 } else {
9642 RunStatus::Blocked
9643 })
9644}
9645
9646pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9661 if nudged {
9662 (full / 4).max(Duration::from_secs(120)).min(full)
9663 } else {
9664 full
9665 }
9666}
9667
9668#[allow(clippy::too_many_arguments)]
9697async fn ask_json_wave<T>(
9698 jobs: Vec<SeatJob>,
9699 sem: Arc<Semaphore>,
9700 retries: usize,
9701 roster: &[AgentSpec],
9702 ctx: &WaveCtx<'_>,
9703 losses: &mut Vec<QuotaLoss>,
9704 state: &mut RunState,
9705 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9706) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9707where
9708 T: serde::de::DeserializeOwned + Send + 'static,
9709{
9710 ask_wave_with(
9711 jobs,
9712 sem,
9713 retries,
9714 roster,
9715 ctx,
9716 losses,
9717 state,
9718 &|text: &str| {
9719 let v = verdict::extract_json::<T>(text)?;
9720 validate(&v)?;
9721 Ok(v)
9722 },
9723 )
9724 .await
9725}
9726
9727#[allow(clippy::too_many_arguments)]
9736async fn ask_wave_with<T>(
9737 jobs: Vec<SeatJob>,
9738 sem: Arc<Semaphore>,
9739 retries: usize,
9740 roster: &[AgentSpec],
9741 ctx: &WaveCtx<'_>,
9742 losses: &mut Vec<QuotaLoss>,
9743 state: &mut RunState,
9744 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9745) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9746where
9747 T: Send + 'static,
9748{
9749 let n = jobs.len();
9750 let originals: Vec<SeatJob> = jobs;
9751 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9752 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9753 let mut nudges: Vec<usize> = vec![0; n];
9760 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9764 let mut tried: Vec<BTreeSet<String>> = specs
9765 .iter()
9766 .map(|s| BTreeSet::from([s.id.clone()]))
9767 .collect();
9768 let starts: Vec<usize> = specs
9769 .iter()
9770 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9771 .collect();
9772 let carry = ctx.carry_seats && !roster.is_empty();
9773 let carried: Vec<BTreeSet<String>> = originals
9776 .iter()
9777 .map(|j| {
9778 state
9779 .seat_history
9780 .get(&j.seat.key)
9781 .filter(|_| carry)
9782 .map(|h| h.failed.clone())
9783 .unwrap_or_default()
9784 })
9785 .collect();
9786 let mut prev: Vec<Option<FailClass>> = originals
9787 .iter()
9788 .map(|j| {
9789 state
9790 .seat_history
9791 .get(&j.seat.key)
9792 .filter(|_| carry)
9793 .and_then(|h| h.last_fail.clone())
9794 })
9795 .collect();
9796 let next_agent =
9797 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9798 let others: BTreeSet<String> = specs
9799 .iter()
9800 .enumerate()
9801 .filter(|(j, _)| *j != i)
9802 .map(|(_, s)| s.id.clone())
9803 .collect();
9804 pick_successor(
9805 roster,
9806 starts[i],
9807 tried,
9808 carry.then(|| &carried[i]),
9809 &others,
9810 )
9811 .cloned()
9812 };
9813 let mut fresh: Vec<Option<String>> = vec![None; n];
9814 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9815 let mut pending: Vec<usize> = (0..n).collect();
9816
9817 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9822 for round in 0..max_rounds {
9823 if pending.is_empty() {
9824 break;
9825 }
9826 let mut batch = Vec::with_capacity(pending.len());
9827 let mut renudged: Vec<&str> = Vec::new();
9828 for &i in &pending {
9829 let src = &originals[i];
9830 let full: &str = match &src.handover {
9835 Some(h) if specs[i].id != src.spec.id => h,
9836 _ => &src.prompt,
9837 };
9838 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9841 (
9842 full.to_owned(),
9843 src.timeout,
9844 format!("{}-{word}-{}", src.stem, specs[i].id),
9845 )
9846 } else if nudges[i] == 0 {
9847 (full.to_owned(), src.timeout, src.stem.clone())
9848 } else {
9849 renudged.push(src.seat.key.as_str());
9850 let why = done[i]
9851 .as_ref()
9852 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9853 .unwrap_or_else(|| "no parsable answer".to_owned());
9854 let nudge = prompt::nudge(&why);
9855 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9856 let prompt = if nudged {
9857 nudge
9858 } else {
9859 format!("{full}\n\n---\n\n{nudge}")
9860 };
9861 (
9862 prompt,
9863 retry_budget(src.timeout, nudged),
9864 format!("{}-retry{}", src.stem, nudges[i]),
9865 )
9866 };
9867 batch.push(SeatJob {
9868 spec: specs[i].clone(),
9869 seat: seats[i].clone(),
9870 cwd: src.cwd.clone(),
9871 prompt,
9872 timeout,
9873 allow_write: src.allow_write,
9874 sessions: src.sessions,
9875 artifacts: src.artifacts.clone(),
9876 stem,
9877 handover: None,
9878 });
9879 }
9880
9881 if !renudged.is_empty() {
9882 state.event(
9883 ctx.node,
9884 format!("retry {round}: re-asking {}", renudged.join(", ")),
9885 );
9886 }
9887 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9888 let mut still = Vec::new();
9889 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9890 seats[i] = seat;
9891 if let AgentOutcome::ResourceBlocked(why) = &out {
9894 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9895 continue;
9896 }
9897 let class = FailClass::of(&out);
9898 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9901 && nudges[i] < retries
9902 && !roster.is_empty();
9903 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9904 let next = should_hand_over(prev[i].as_ref(), &cur)
9905 .then(|| next_agent(i, &tried[i], &specs))
9906 .flatten();
9907 if carry {
9908 let h = state
9909 .seat_history
9910 .entry(originals[i].seat.key.clone())
9911 .or_default();
9912 h.failed.insert(specs[i].id.clone());
9913 h.last_fail = Some(cur.clone());
9914 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9915 h.last_ok = None;
9916 }
9917 if let Err(e) = state.save() {
9920 tracing::warn!("could not persist a seat's failure history: {e:#}");
9921 }
9922 }
9923 if let Some(next) = next {
9924 record_handover(
9925 state,
9926 ctx.node,
9927 &originals[i].seat.key,
9928 &specs[i].id,
9929 &next.id,
9930 &cur,
9931 &fail_reason(&out),
9932 );
9933 tried[i].insert(next.id.clone());
9934 prev[i] = Some(cur.clone());
9935 seats[i] =
9936 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9937 specs[i] = next;
9938 fresh[i] = Some(cur.stem_word().to_owned());
9939 nudges[i] = 0;
9940 done[i] = Some(Err(anyhow::anyhow!(
9941 "handed over after: {}",
9942 fail_reason(&out)
9943 )));
9944 still.push(i);
9945 continue;
9946 }
9947 }
9948 if carry
9949 && !nudge_first
9950 && let Some(cur) = class.clone()
9951 {
9952 let h = state
9954 .seat_history
9955 .entry(originals[i].seat.key.clone())
9956 .or_default();
9957 h.failed.insert(specs[i].id.clone());
9958 h.last_fail = Some(cur);
9959 }
9960 let parsed = match out {
9961 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9962 AgentOutcome::Quota(o) => {
9963 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9964 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9965 }
9966 AgentOutcome::Dropped(o) => {
9970 let why = o
9971 .dropped
9972 .as_ref()
9973 .map(|d| d.why.as_str())
9974 .unwrap_or("the CLI ended the stream without delivering its answer");
9975 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9976 }
9977 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9978 Err(anyhow::anyhow!(e))
9979 }
9980 };
9981 let quota = class == Some(FailClass::Quota);
9982 let failed = parsed.is_err();
9983 done[i] = Some(parsed);
9984 if carry && !failed {
9985 let h = state
9986 .seat_history
9987 .entry(originals[i].seat.key.clone())
9988 .or_default();
9989 h.failed.remove(&specs[i].id);
9990 h.last_ok = Some(specs[i].id.clone());
9991 h.last_fail = None;
9992 }
9993 let agent_failure = class.is_some()
10001 && !nudge_first
10002 && !roster.is_empty()
10003 && next_agent(i, &tried[i], &specs).is_some();
10004 if failed && !quota && !agent_failure && nudges[i] < retries {
10005 nudges[i] += 1;
10006 still.push(i);
10007 }
10008 }
10009 pending = still;
10010 }
10011
10012 for (i, q) in last_quota.into_iter().enumerate() {
10015 if let Some(reset) = q {
10016 losses.push(QuotaLoss {
10017 seat: originals[i].seat.key.clone(),
10018 node: ctx.node.to_owned(),
10019 at: Timestamp::now(),
10020 reset,
10021 });
10022 }
10023 }
10024
10025 seats
10026 .into_iter()
10027 .zip(done)
10028 .zip(nudges)
10029 .map(|((seat, res), attempts)| {
10030 (
10031 seat,
10032 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10033 attempts,
10034 )
10035 })
10036 .collect()
10037}
10038
10039fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10041 results
10042 .iter()
10043 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10044}
10045
10046fn block_wave(
10050 jobs: Vec<SeatJob>,
10051 state: &mut RunState,
10052 node: &str,
10053 attempt: usize,
10054 why: &str,
10055) -> Vec<(usize, SeatState, AgentOutcome)> {
10056 state.resource_wait = None;
10057 state
10058 .active
10059 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10060 if let Err(e) = state.save() {
10061 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10062 }
10063 jobs.into_iter()
10064 .enumerate()
10065 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10066 .collect()
10067}
10068
10069async fn acquire_cache_lease(
10082 state: &mut RunState,
10083 cache_dir: &Path,
10084 owner: &crate::cache::Owner,
10085 budget: Duration,
10086 context: &str,
10087 cancel: &(dyn Fn() -> bool + Sync),
10088) -> Result<Option<crate::cache::Guard>> {
10089 let home = crate::run::home();
10090 let started = Instant::now();
10091 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10092 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10093 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10094 Err(e) => {
10095 state.event(
10096 "verify",
10097 format!("{context}: could not check the shared build cache: {e:#}"),
10098 );
10099 if let Err(e2) = state.save() {
10100 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10101 }
10102 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10103 }
10104 };
10105 state.event(
10106 "verify",
10107 format!(
10108 "{context}: waiting for the shared build cache at {} ({})",
10109 cache_dir.display(),
10110 busy.describe()
10111 ),
10112 );
10113 state.resource_wait = Some(crate::run::ResourceWait {
10116 node: context.to_owned(),
10117 kind: "shared build cache".to_owned(),
10118 holder: busy.describe(),
10119 since: Timestamp::now(),
10120 limit_secs: budget.as_secs(),
10121 });
10122 if let Err(e) = state.save() {
10123 tracing::warn!("could not persist a cache wait: {e:#}");
10124 }
10125 let remaining = budget.saturating_sub(started.elapsed());
10126 let waited = crate::cache::wait_for_unless(
10127 &home,
10128 cache_dir,
10129 owner,
10130 remaining,
10131 Duration::from_secs(5),
10132 cancel,
10133 )
10134 .await;
10135 state.resource_wait = None;
10136 match waited {
10137 Ok(g) => Ok(g),
10138 Err(e) => {
10139 state.event("verify", format!("{context}: {e:#}"));
10140 if let Err(e2) = state.save() {
10141 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10142 }
10143 Err(e)
10144 }
10145 }
10146}
10147
10148#[derive(Debug)]
10153struct CacheCheckError(String);
10154
10155impl std::fmt::Display for CacheCheckError {
10156 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10157 f.write_str(&self.0)
10158 }
10159}
10160
10161impl std::error::Error for CacheCheckError {}
10162
10163#[allow(clippy::too_many_arguments)]
10184async fn with_cache_lease<'s, F, Fut>(
10185 state: &'s mut RunState,
10186 cache_dir: Option<&Path>,
10187 node: &str,
10188 seat: &str,
10189 worktree: &Path,
10190 head: &str,
10191 budget: Duration,
10192 context: &str,
10193 wait: &crate::repo_lock::WaitPolicy,
10194 body: F,
10195) -> Result<(Vec<CommandOutcome>, bool)>
10196where
10197 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10198 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10199{
10200 let started = Instant::now();
10203 let home = crate::run::home();
10204 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10205 let guard = if let Some(cache_dir) = cache_dir {
10206 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10207 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10208 wait.cancelled()
10209 })
10210 .await
10211 .and_then(|g| {
10212 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10213 }) {
10214 Ok(g) => Some(g),
10215 Err(e) => {
10216 return Ok((
10217 vec![CommandOutcome {
10218 command: "(waiting for the shared build cache)".into(),
10219 code: None,
10220 output_tail: e.to_string(),
10221 duration_ms: started.elapsed().as_millis() as u64,
10222 resource_blocked: true,
10223 }],
10224 false,
10225 ));
10226 }
10227 }
10228 } else {
10229 None
10230 };
10231 let _coordination = coordinate_operation(state, worktree, wait).await?;
10232 if let Some(cache_dir) = cache_dir {
10233 let identity = crate::cache::Identity::new(worktree, head);
10234 if let Err(e) = crate::cache::ensure_fresh(
10235 &home,
10236 cache_dir,
10237 &identity,
10238 &state.config.verify.cargo_manifests(),
10239 ) {
10240 state.event(
10241 "verify",
10242 format!(
10243 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10244 worktree.display(),
10245 short(head)
10246 ),
10247 );
10248 if let Some(guard) = guard {
10249 guard.release();
10250 }
10251 return Ok((
10252 vec![CommandOutcome {
10253 command: "(confirming the shared build cache is fresh)".into(),
10254 code: None,
10255 output_tail: e.to_string(),
10256 duration_ms: started.elapsed().as_millis() as u64,
10257 resource_blocked: true,
10258 }],
10259 false,
10260 ));
10261 }
10262 }
10263 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10264 if !timed_out_pids.is_empty() {
10265 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10266 }
10267 drop(_coordination);
10270 if let Some(guard) = guard {
10271 guard.release();
10272 }
10273 Ok((outcomes, retried))
10274}
10275
10276async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10288 wait_for_pids_with(
10289 pids,
10290 crate::proc::pid_alive,
10291 LEASE_RELEASE_POLL,
10292 LEASE_RELEASE_MAX_WAIT,
10293 )
10294 .await;
10295}
10296
10297async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10303 pids: &[u32],
10304 alive: F,
10305 poll: Duration,
10306 max_wait: Duration,
10307) {
10308 let deadline = Instant::now() + max_wait;
10309 loop {
10310 if pids.iter().all(|&pid| !alive(pid)) {
10311 return;
10312 }
10313 if Instant::now() >= deadline {
10314 return;
10315 }
10316 tokio::time::sleep(poll).await;
10317 }
10318}
10319
10320fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10327 outcomes.iter().any(|o| o.resource_blocked)
10328}
10329
10330enum GateFix {
10332 Retry,
10334 Stop,
10337 Defer,
10340}
10341
10342fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10350 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10351 red.peek().is_some()
10352 && red.all(|o| {
10353 !o.resource_blocked
10354 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10355 && !o.output_tail.trim().is_empty()
10356 })
10357}
10358
10359fn e2e_outcome_label(o: &CommandOutcome) -> String {
10363 if o.ok() {
10364 return "pass".to_owned();
10365 }
10366 let reason = if o.build_failed() {
10367 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10368 } else {
10369 format!("FAIL ({:?})", o.code)
10370 };
10371 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10372}
10373
10374async fn run_e2e_with_retry(
10382 state: &mut RunState,
10383 shell: &[String],
10384 commands: &[String],
10385 worktree: &Path,
10386 timeout: Duration,
10387 context: &str,
10388) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10389 let (mut e2e, mut timed_out_pids) = run_commands(
10390 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10391 )
10392 .await;
10393 for o in &e2e {
10394 state.event(
10395 "verify",
10396 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10397 );
10398 }
10399 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10403 if verify_retried {
10404 state.event(
10405 "verify",
10406 format!(
10407 "{context}: verify could not build/link, not a test result — retrying once \
10408 before concluding"
10409 ),
10410 );
10411 let retried = run_commands(
10412 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10413 )
10414 .await;
10415 e2e = retried.0;
10416 timed_out_pids.extend(retried.1);
10419 for o in &e2e {
10420 state.event(
10421 "verify",
10422 format!(
10423 "{context}: retry `{}` -> {}",
10424 o.command,
10425 e2e_outcome_label(o)
10426 ),
10427 );
10428 }
10429 }
10430 (e2e, verify_retried, timed_out_pids)
10431}
10432
10433#[allow(clippy::too_many_arguments)]
10449async fn run_commands(
10450 state: &mut RunState,
10451 node: &str,
10452 task: &str,
10453 attempt: usize,
10454 shell: &[String],
10455 commands: &[String],
10456 cwd: &Path,
10457 timeout: Duration,
10458) -> (Vec<CommandOutcome>, Vec<u32>) {
10459 if commands.is_empty() {
10460 return (Vec::new(), Vec::new());
10465 }
10466 let mut out = Vec::new();
10467 let mut timed_out_pids = Vec::new();
10468 let total = commands.len();
10469 for (idx, command) in commands.iter().enumerate() {
10470 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10471 if let Err(e) = state.save() {
10472 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10473 }
10474 let started = Instant::now();
10475 let mut cmd = tokio::process::Command::new(&shell[0]);
10476 cmd.quiet();
10477 cmd.args(&shell[1..])
10478 .arg(command)
10479 .current_dir(cwd)
10480 .stdin(std::process::Stdio::null())
10481 .stdout(std::process::Stdio::piped())
10482 .stderr(std::process::Stdio::piped())
10483 .kill_on_drop(true);
10484 let spawned = cmd.spawn();
10485 let (code, body) = match spawned {
10486 Ok(child) => {
10487 let pid = child.id();
10492 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10493 Ok(Ok(o)) => {
10494 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10495 body.push_str(&String::from_utf8_lossy(&o.stderr));
10496 (o.status.code(), body)
10497 }
10498 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10499 Err(_) => {
10500 if let Some(pid) = pid {
10501 timed_out_pids.push(pid);
10502 }
10503 (None, format!("timed out after {}s", timeout.as_secs()))
10504 }
10505 }
10506 }
10507 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10508 };
10509 out.push(CommandOutcome {
10510 command: command.clone(),
10511 code,
10512 output_tail: tail(&body, OUTPUT_TAIL),
10513 duration_ms: started.elapsed().as_millis() as u64,
10514 resource_blocked: false,
10515 });
10516 }
10517 state.task_finished(task);
10518 if let Err(e) = state.save() {
10519 tracing::warn!("could not persist the end of {task}: {e:#}");
10520 }
10521 (out, timed_out_pids)
10522}
10523
10524fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10536 let repo = repo.display();
10537 match style {
10538 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10539 MergeStyle::Squash => {
10540 let subject = message
10543 .lines()
10544 .next()
10545 .unwrap_or(branch)
10546 .replace(['\\', '"', '$', '`'], "");
10547 format!(
10548 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10549 )
10550 }
10551 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10552 }
10553}
10554
10555const PR_TITLE_MAX: usize = 240;
10566
10567fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10578 let line = queue::first_line(text)?;
10579 let chars: Vec<char> = line.chars().collect();
10580 let (kept, mark) = if chars.len() <= max {
10581 (line.to_owned(), "")
10582 } else if let Some(end) = sentence_end(&chars, max) {
10583 (chars[..end].iter().collect::<String>(), "")
10584 } else {
10585 let room = max.saturating_sub(3);
10586 let cut = if chars[room].is_whitespace() {
10589 room
10590 } else {
10591 chars[..room]
10592 .iter()
10593 .rposition(|c| c.is_whitespace())
10594 .unwrap_or(room)
10595 };
10596 let head: String = chars[..cut].iter().collect();
10597 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10598 (head.to_owned(), "...")
10599 };
10600 if kept.is_empty() {
10601 return None;
10602 }
10603 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10604 return None;
10605 }
10606 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10607 return None;
10608 }
10609 Some(format!("{kept}{mark}"))
10610}
10611
10612fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10616 const MIN: usize = 20;
10617 for i in MIN..max.min(chars.len()) {
10618 if !matches!(chars[i], '.' | '!' | '?') {
10619 continue;
10620 }
10621 let Some(&next) = chars.get(i + 1) else {
10622 continue;
10623 };
10624 if !next.is_whitespace() {
10625 continue;
10626 }
10627 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10628 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10629 continue;
10630 }
10631 let word: String = chars[..i]
10632 .iter()
10633 .rev()
10634 .take_while(|c| !c.is_whitespace())
10635 .collect::<Vec<_>>()
10636 .into_iter()
10637 .rev()
10638 .collect();
10639 let word = word.to_ascii_lowercase();
10640 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10641 continue;
10642 }
10643 let end = chars[..i]
10644 .iter()
10645 .rposition(|c| !c.is_whitespace())
10646 .map_or(i, |p| p + 1);
10647 return Some(end);
10648 }
10649 None
10650}
10651
10652struct PrMessage {
10657 title: String,
10658 body: String,
10659}
10660
10661impl PrMessage {
10662 fn commit_message(&self) -> String {
10666 format!("{}\n\n{}", self.title, self.body)
10667 }
10668}
10669
10670fn title_marker(line: &str) -> Option<&str> {
10672 let line = line.trim();
10673 let head = line.get(..6)?;
10674 head.eq_ignore_ascii_case("title:")
10675 .then(|| line[6..].trim())
10676}
10677
10678fn summary_title(summary: &str) -> Option<String> {
10683 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10684 let raw = title_marker(first)?;
10685 if raw.is_empty() {
10686 return None;
10687 }
10688 let title = queue::title_from(raw, PR_TITLE_MAX);
10689 let lower = title.to_ascii_lowercase();
10690 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10691 return None;
10692 }
10693 Some(title)
10694}
10695
10696const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10698
10699pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10701
10702fn is_review_run(state: &RunState) -> bool {
10706 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10707}
10708
10709fn review_title(state: &RunState) -> Option<String> {
10714 english_subject(
10715 state.reviewed_commits.as_ref()?.first()?,
10716 state.config.graph.github_english(),
10717 )
10718}
10719
10720fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10723 let raw = raw.trim();
10724 if raw.is_empty() {
10725 return None;
10726 }
10727 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10728 return None;
10729 }
10730 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10731 return None;
10732 }
10733 let title = queue::title_from(raw, PR_TITLE_MAX);
10734 let lower = title.to_ascii_lowercase();
10735 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10736 return None;
10737 }
10738 Some(title)
10739}
10740
10741#[derive(Debug, Clone, PartialEq, Eq)]
10744struct BranchFacts {
10745 commits: Vec<(String, String)>,
10747 stat: String,
10749}
10750
10751const STAT_FILE_LINES: usize = 25;
10753const COMMITS_MAX_CHARS: usize = 20_000;
10755
10756async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10759 let commits = git::commit_log(repo, base, branch).await.ok()?;
10760 if commits.is_empty() {
10761 return None;
10762 }
10763 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10764 let lines: Vec<&str> = stat.lines().collect();
10765 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10766 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10767 let more = format!(" ... {omitted} more file(s)");
10768 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10769 kept.push(&more);
10770 kept.push(lines[lines.len() - 1]);
10771 kept.join("\n")
10772 } else {
10773 lines.join("\n")
10774 };
10775 Some(BranchFacts { commits, stat })
10776}
10777
10778fn markdown_safe(text: &str) -> String {
10781 text.replace("</details>", "</details>")
10782 .replace("\x60\x60\x60", "~~~")
10783}
10784
10785fn neutral_title(state: &RunState, winner: char) -> String {
10786 format!(
10787 "chore: land candidate {} of run {}",
10788 winner.to_ascii_uppercase(),
10789 state.id
10790 )
10791}
10792
10793pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10799 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10800 ""
10801 } else {
10802 pr_title
10803 }
10804}
10805
10806pub fn landing_subject_source(state: &RunState) -> String {
10810 if is_review_run(state) {
10811 let winner = state.candidates.first().map_or('A', |c| c.label);
10812 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10813 }
10814 state.instruction.clone()
10815}
10816
10817fn summary_without_title(summary: &str) -> String {
10820 let mut lines = summary.trim().lines().peekable();
10821 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10822 lines.next();
10823 }
10824 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10825}
10826
10827#[cfg(test)]
10839fn pr_message(state: &RunState, winner: char) -> PrMessage {
10840 pr_message_with(state, winner, None)
10841}
10842
10843#[cfg(test)]
10846fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10847 let raw = pr_message_raw(
10848 state,
10849 winner,
10850 facts,
10851 crate::pr_outcome::FinalHead::Unreported,
10852 ChangeNote::NotAsked,
10853 );
10854 let id = crate::scrub::Identity::current();
10855 PrMessage {
10856 title: crate::scrub::scrub(&raw.title, &id),
10857 body: crate::scrub::scrub(&raw.body, &id),
10858 }
10859}
10860
10861#[derive(Debug, Clone, Copy)]
10863enum ChangeNote<'a> {
10864 NotAsked,
10866 Summary(&'a crate::run::ChangeSummary),
10868 Unavailable(&'a str),
10870}
10871
10872fn pr_message_raw(
10873 state: &RunState,
10874 winner: char,
10875 facts: Option<&BranchFacts>,
10876 head: crate::pr_outcome::FinalHead<'_>,
10877 change: ChangeNote<'_>,
10878) -> PrMessage {
10879 if !matches!(change, ChangeNote::NotAsked) {
10880 let (title, mut body) = match change {
10881 ChangeNote::Summary(c) => (
10882 c.title.clone(),
10883 format!("## Summary\n\n{}\n\n", c.body.trim()),
10884 ),
10885 ChangeNote::Unavailable(why) => (
10886 neutral_title(state, winner),
10887 format!(
10888 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10889 ),
10890 ),
10891 ChangeNote::NotAsked => unreachable!(),
10892 };
10893 body.push_str(&public_pr_evidence(state, head));
10894 body.push_str(&format!(
10895 "\n---\nmagi:run/{} magi:candidate-{}\n",
10896 state.id,
10897 winner.to_ascii_lowercase()
10898 ));
10899 return PrMessage { title, body };
10900 }
10901 pr_preview(state, winner, facts, head)
10902}
10903
10904fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10907 let mut body = crate::pr_outcome::review_rounds(state);
10908 body.push_str(&crate::pr_outcome::final_verification(state, head));
10909 let open = state.last_round_findings();
10910 if !open.is_empty() {
10911 body.push_str("## Open review findings\n\n");
10912 for f in open {
10913 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10914 }
10915 body.push('\n');
10916 }
10917 if let Some(c) = state.change_summary.as_ref()
10918 && head == crate::pr_outcome::FinalHead::At(&c.head)
10919 {
10920 body.push_str(&format!(
10921 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10922 c.base, c.head
10923 ));
10924 }
10925 body
10926}
10927
10928fn pr_preview(
10930 state: &RunState,
10931 winner: char,
10932 facts: Option<&BranchFacts>,
10933 head: crate::pr_outcome::FinalHead<'_>,
10934) -> PrMessage {
10935 let summary = state
10936 .candidates
10937 .iter()
10938 .find(|c| c.label == winner)
10939 .map(|c| c.summary.as_str())
10940 .unwrap_or_default();
10941 let review = is_review_run(state);
10944 let title = if review {
10945 facts
10946 .and_then(|f| {
10947 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10948 })
10949 .or_else(|| review_title(state))
10950 .or_else(|| {
10951 state
10952 .candidates
10953 .iter()
10954 .find(|c| c.label == winner)
10955 .filter(|c| !c.branch.starts_with("magi/"))
10956 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10957 })
10958 .unwrap_or_else(|| neutral_title(state, winner))
10959 } else {
10960 summary_title(summary).unwrap_or_else(|| {
10961 english_title(
10962 &state.instruction,
10963 PR_TITLE_MAX,
10964 state.config.graph.github_english(),
10965 )
10966 .unwrap_or_else(|| neutral_title(state, winner))
10967 })
10968 };
10969
10970 let mut body = String::new();
10971 let what = summary_without_title(summary);
10972 if !what.is_empty() {
10973 body.push_str("## Summary\n\n");
10974 body.push_str(&what);
10975 body.push_str("\n\n");
10976 }
10977
10978 body.push_str(&crate::pr_outcome::review_rounds(state));
10980 body.push_str(&crate::pr_outcome::final_verification(state, head));
10981 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10982
10983 let open = state.open_findings();
10984 if !open.is_empty() {
10985 body.push_str("## Open review findings\n\n");
10986 for f in &open {
10987 body.push_str(&format!(
10988 "- `{}` [{:?}] {}\n",
10989 f.id, f.severity, "Context retained in local review records."
10990 ));
10991 }
10992 body.push('\n');
10993 }
10994
10995 if let Some(fix) = fix
10996 && !fix.rejected.is_empty()
10997 {
10998 body.push_str("## Declined by the fixer\n\n");
10999 for r in &fix.rejected {
11000 body.push_str(&format!(
11001 "- `{}`: {}\n",
11002 r.id, "Reason retained in local review records."
11003 ));
11004 }
11005 body.push('\n');
11006 }
11007
11008 if review {
11009 body.push_str("## Commits under review\n\n");
11011 if let Some(facts) = facts {
11012 let mut left = COMMITS_MAX_CHARS;
11013 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11014 let mut entry = format!("- {}\n", markdown_safe(subject));
11015 for l in markdown_safe(text).lines() {
11016 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11017 }
11018 if left == 0 {
11019 body.push_str(&format!(
11020 "- ... {} more commit(s)\n",
11021 facts.commits.len() - i
11022 ));
11023 break;
11024 }
11025 if entry.len() > left {
11026 let mut end = left;
11029 while !entry.is_char_boundary(end) {
11030 end -= 1;
11031 }
11032 entry.truncate(end);
11033 entry.push_str("\n ... (truncated)\n");
11034 left = 0;
11035 } else {
11036 left -= entry.len();
11037 }
11038 body.push_str(&entry);
11039 }
11040 if !facts.stat.trim().is_empty() {
11041 body.push_str(&format!(
11042 "\n## Diff stat\n\n```\n{}\n```\n",
11043 markdown_safe(facts.stat.trim())
11044 ));
11045 }
11046 } else {
11047 match &state.reviewed_commits {
11048 Some(subjects) => {
11049 for s in subjects {
11050 body.push_str(&format!("- {}\n", s.trim()));
11051 }
11052 }
11053 None => {
11054 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11057 body.push_str(rest.trim());
11058 body.push('\n');
11059 }
11060 }
11061 }
11062 } else {
11063 let task = state.instruction.trim();
11064 let task = if task.is_empty() {
11065 "(empty task)"
11066 } else {
11067 task
11068 };
11069 body.push_str(&format!(
11070 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11071 task.replace("</details>", "</details>")
11072 ));
11073 }
11074
11075 body.push_str(&format!(
11076 "\n---\nmagi:run/{} magi:candidate-{}\n",
11077 state.id,
11078 winner.to_ascii_lowercase()
11079 ));
11080
11081 PrMessage { title, body }
11082}
11083
11084fn seeded_instruction(state: &RunState) -> String {
11088 match refs::describe(&state.seeds) {
11089 Some(facts) => format!(
11090 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11091 Candidates start from the unmerged branch named above, when there \
11092 is one, and carry any unmerged commit named by sha as a \
11093 cherry-pick. Check that this is what the task meant before \
11094 building on it.",
11095 state.instruction
11096 ),
11097 None => state.instruction.clone(),
11098 }
11099}
11100
11101async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11105 let base = &state.base_branch;
11106 let mut against = base.clone();
11107 if mode == MergeMode::Pr {
11108 let remote = &state.config.merge.remote;
11112 let tracking = format!("{remote}/{base}");
11113 let fetched = git::fetch(repo, remote, base).await;
11114 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11115 against = tracking;
11116 } else {
11117 return false;
11118 }
11119 }
11120 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11121}
11122
11123fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11126 let mut detail = format!(
11127 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11128 nothing to open a pull request for"
11129 );
11130 match refs::describe(&state.seeds) {
11131 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11132 None => detail.push_str(
11133 "\nThe task names no existing branch or commit; if it means to land work \
11134 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11135 ),
11136 }
11137 detail
11138}
11139
11140#[derive(Debug, PartialEq, Eq)]
11143enum PrPlan {
11144 Create,
11145 Adopt { url: String, title: String },
11146 Stop(String),
11147}
11148
11149fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11152 match found {
11153 Ok(land::OpenPr::None) => PrPlan::Create,
11154 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11155 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11156 "several open pull requests exist for this branch, not picking one: {}",
11157 urls.join(" ")
11158 )),
11159 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11160 }
11161}
11162
11163pub(crate) async fn gh_pr_create(
11165 cwd: &Path,
11166 base: &str,
11167 head: &str,
11168 title: &str,
11169 body: &str,
11170) -> Result<String> {
11171 let out = tokio::process::Command::new("gh")
11172 .args([
11173 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11174 ])
11175 .current_dir(cwd)
11176 .quiet()
11177 .stdin(std::process::Stdio::null())
11178 .output()
11179 .await
11180 .context("spawn gh")?;
11181 if out.status.success() {
11182 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11183 } else {
11184 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11185 }
11186}
11187
11188async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11190 match std::fs::symlink_metadata(path) {
11191 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11192 git::worktree_remove_clean(repo, path).await?;
11195 return Ok(());
11196 }
11197 Err(e) => return Err(e.into()),
11198 Ok(_) => {}
11199 }
11200 if !git::is_clean(path).await? {
11201 bail!(
11202 "worktree has uncommitted work; refusing removal: {}",
11203 path.display()
11204 );
11205 }
11206 if !git::worktree_remove_clean(repo, path).await? {
11207 bail!("worktree removal refused: {}", path.display());
11208 }
11209 Ok(())
11210}
11211
11212pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11221 if crate::pr_recovery::sources_retained(home, &state.id) {
11222 return Ok(Vec::new());
11223 }
11224 if !state.candidate_preservation.is_empty() {
11225 bail!("candidate preservation incomplete; refusing cleanup");
11226 }
11227 let repo = state.repo.clone();
11228 let wait = repository_wait(state, None);
11231 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11232 Ok(lease) => lease,
11233 Err(e) => {
11234 state.save_under(home)?;
11235 return Err(e);
11236 }
11237 };
11238 let root = state.worktree_root();
11239 let winner = state.tally.as_ref().map(|t| t.winner);
11240 let mut removed = Vec::new();
11241
11242 for i in 0..state.candidates.len() {
11243 let c = state.candidates[i].clone();
11244 let is_winner = Some(c.label) == winner;
11245 if is_winner && !drop_winner {
11246 continue;
11247 }
11248 let existed = c.worktree.exists();
11249 remove_preserved_worktree(&repo, &c.worktree).await?;
11252 if existed {
11253 removed.push(c.worktree.to_string_lossy().into_owned());
11254 }
11255 let handed_over = state.released_branches.contains(&c.branch);
11258 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11259 if !git::branch_delete(&repo, &c.branch).await? {
11260 bail!("candidate branch removal refused");
11261 }
11262 removed.push(c.branch.clone());
11263 }
11264 state.candidates[i].folded = true;
11265 }
11266
11267 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11268 let path = name.path();
11269 let keep = !drop_winner
11270 && winner.is_some_and(|w| {
11271 path.file_name()
11272 .is_some_and(|n| n == format!("cand-{w}").as_str())
11273 });
11274 if keep {
11275 continue;
11276 }
11277 remove_preserved_worktree(&repo, &path).await?;
11278 removed.push(path.to_string_lossy().into_owned());
11279 }
11280
11281 remove_if_empty(&root);
11290
11291 if state.enabled_worktree_config && drop_winner {
11292 git::release_worktree_config(&repo).await.ok();
11296 state.enabled_worktree_config = false;
11297 }
11298 state.save_under(home)?;
11299 Ok(removed)
11300}
11301
11302fn remove_if_empty(dir: &Path) {
11313 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11314 std::fs::remove_dir(dir).ok();
11315 }
11316}
11317
11318pub fn worst_open(state: &RunState) -> Option<Severity> {
11320 state
11321 .reviews
11322 .last()?
11323 .reviews
11324 .iter()
11325 .flat_map(|r| r.findings.iter())
11326 .map(|f| f.severity)
11327 .max()
11328}
11329
11330async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11334 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11335 git::acquire_worktree_config(repo).await?;
11336 state.enabled_worktree_config = true;
11337 }
11338 Ok(())
11339}
11340
11341#[cfg(test)]
11342mod tests {
11343 #[tokio::test]
11344 async fn launcher_base_fetch_requires_remote_success() {
11345 let tmp = tempfile::tempdir().unwrap();
11346 let repo = tmp.path().join("repo");
11347 let remote = tmp.path().join("remote");
11348 std::fs::create_dir(&repo).unwrap();
11349 std::fs::create_dir(&remote).unwrap();
11350 git::git(&remote, &["init", "--bare", "-b", "main"])
11351 .await
11352 .unwrap();
11353 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11354 git::git(
11355 &repo,
11356 &[
11357 "-c",
11358 "user.name=Fixture",
11359 "-c",
11360 "user.email=fixture@example.invalid",
11361 "commit",
11362 "--allow-empty",
11363 "-m",
11364 "fixture",
11365 ],
11366 )
11367 .await
11368 .unwrap();
11369 git::git(
11370 &repo,
11371 &["remote", "add", "origin", remote.to_str().unwrap()],
11372 )
11373 .await
11374 .unwrap();
11375 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11376 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11377 assert_eq!(
11378 resolve_base(&repo, "main", "origin", None)
11379 .await
11380 .unwrap()
11381 .commit,
11382 head
11383 );
11384 #[cfg(unix)]
11385 {
11386 let policy: crate::config::Coordination = toml::from_str(
11387 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11388 ).unwrap();
11389 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11390 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11391 let held = crate::repo_lock::SeatLease::operation_with(
11392 &repo,
11393 Some(&policy),
11394 &crate::repo_lock::WaitPolicy::default(),
11395 )
11396 .await
11397 .unwrap();
11398 assert!(held.is_some());
11399 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11400 tokio::pin!(fetch);
11401 assert!(
11402 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11403 .await
11404 .is_err()
11405 );
11406 assert!(
11407 !repo.join(".git/FETCH_HEAD").exists(),
11408 "no fetch before admission"
11409 );
11410 drop(held);
11411 let fetched = fetch.await.unwrap();
11412 assert_eq!(fetched.commit, head);
11413 assert_eq!(fetched.remote, "origin");
11414 assert_eq!(fetched.branch, "main");
11415 }
11416 git::git(
11418 &repo,
11419 &[
11420 "remote",
11421 "set-url",
11422 "origin",
11423 tmp.path().join("missing").to_str().unwrap(),
11424 ],
11425 )
11426 .await
11427 .unwrap();
11428 let error = resolve_base(&repo, "main", "origin", None)
11429 .await
11430 .unwrap_err()
11431 .to_string();
11432 assert!(error.contains("cannot read origin/main"));
11433 assert!(error.contains("refusing to branch off"));
11434 }
11435
11436 #[test]
11437 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11438 let leaked = vec!["chore(deps): update a crate".to_owned()];
11439 let own = "fix(daemon): apply a chosen action";
11440 assert!(should_retitle("", own, &leaked));
11441 assert!(should_retitle(
11442 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11443 own,
11444 &leaked
11445 ));
11446 assert!(should_retitle(
11447 "chore: land candidate A of run 1",
11448 own,
11449 &leaked
11450 ));
11451 assert!(should_retitle(
11452 "magi: candidate A (uncommitted work)",
11453 own,
11454 &leaked
11455 ));
11456 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11457 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11458 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11459 }
11460
11461 #[test]
11462 fn pr_merge_plan_creates_adopts_or_stops() {
11463 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11464 assert_eq!(
11465 pr_merge_plan(Ok(land::OpenPr::One {
11466 url: "u".into(),
11467 title: "t".into()
11468 })),
11469 PrPlan::Adopt {
11470 url: "u".into(),
11471 title: "t".into()
11472 }
11473 );
11474 let PrPlan::Stop(many) =
11475 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11476 else {
11477 panic!("many must stop");
11478 };
11479 assert!(many.contains('a') && many.contains('b'));
11480 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11481 panic!("a failed lookup must stop");
11482 };
11483 assert!(err.contains("bad token"));
11484 }
11485
11486 use super::*;
11487 use crate::run::GateStatus;
11488 use std::collections::BTreeMap;
11489 use std::time::Duration;
11490
11491 fn conductor() -> AgentSpec {
11492 AgentSpec {
11493 id: "conductor".to_owned(),
11494 kind: crate::config::AgentKind::Command,
11495 model: None,
11496 command: vec!["true".to_owned()],
11497 extra_args: Vec::new(),
11498 env: BTreeMap::new(),
11499 prompt_delivery: None,
11500 }
11501 }
11502
11503 fn spec(id: &str) -> AgentSpec {
11504 AgentSpec {
11505 id: id.to_owned(),
11506 kind: crate::config::AgentKind::Command,
11507 model: None,
11508 command: vec!["true".to_owned()],
11509 extra_args: Vec::new(),
11510 env: BTreeMap::new(),
11511 prompt_delivery: None,
11512 }
11513 }
11514
11515 fn ids(xs: &[&str]) -> BTreeSet<String> {
11516 xs.iter().map(|s| (*s).to_owned()).collect()
11517 }
11518
11519 #[test]
11520 fn pick_successor_skips_an_agent_another_seat_holds() {
11521 let roster = [spec("a"), spec("b"), spec("c")];
11523 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11524 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11525 }
11526
11527 #[test]
11528 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11529 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11531 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11532 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11533 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11534 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11535 }
11536
11537 #[test]
11538 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11539 let roster = [spec("a"), spec("b")];
11540 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11541 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11542 let carried = ids(&["b"]);
11543 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11544 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11545 }
11546
11547 #[test]
11548 fn pick_successor_returns_none_without_any_untried_successor() {
11549 let roster = [spec("a"), spec("b")];
11550 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11551 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11552 let carried = ids(&["a"]);
11553 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11554 }
11555
11556 #[test]
11557 fn pick_successor_rescue_avoids_another_seats_occupant() {
11558 let roster = [spec("a"), spec("b"), spec("c")];
11560 let carried = ids(&["b", "c"]);
11561 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11562 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11563 }
11564
11565 #[test]
11566 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11567 let roster = [spec("a"), spec("b"), spec("c")];
11568 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11569 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11570 }
11571
11572 #[test]
11573 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11574 let roster = [spec("a"), spec("b"), spec("c")];
11575 let failed = ids(&["a", "b", "c"]);
11576 let mut tried = ids(&["b"]);
11578 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11579 assert_eq!(first.id, "a");
11580 tried.insert(first.id.clone());
11581 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11582 assert_eq!(second.id, "c");
11583 tried.insert(second.id.clone());
11584 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11585 }
11586
11587 #[test]
11588 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11589 let roster = [spec("a"), spec("b")];
11590 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11591 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11592 }
11593
11594 #[test]
11595 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11596 let roster = [spec("a"), spec("b"), spec("c")];
11597 let h = SeatHistory {
11598 failed: ids(&["a"]),
11599 last_ok: Some("b".to_owned()),
11600 last_fail: None,
11601 };
11602 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11603 let gone = SeatHistory {
11605 last_ok: Some("zzz".to_owned()),
11606 ..h.clone()
11607 };
11608 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11609 let failed = SeatHistory {
11610 failed: ids(&["a", "b"]),
11611 last_ok: Some("b".to_owned()),
11612 last_fail: None,
11613 };
11614 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11615 }
11616
11617 #[test]
11618 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11619 let first = SeatState::new("review-1", "alpha", 7);
11620 let next = handover_seat("review-1", "gamma", 7);
11621 assert_ne!(first.claude_session, next.claude_session);
11622 }
11623
11624 #[test]
11625 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11626 let mut state = state_with_summary("x", "y");
11627 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11628 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11629 assert_ne!(a.claude_session, b.claude_session);
11630 }
11631
11632 #[test]
11633 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11634 let roster = [spec("a"), spec("b")];
11635 let h = SeatHistory {
11636 failed: ids(&["a", "b"]),
11637 ..SeatHistory::default()
11638 };
11639 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11640 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11641 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11642 }
11643
11644 #[test]
11651 fn failure_signature_ignores_numbers_and_paths() {
11652 assert_eq!(
11653 failure_signature("exited with Some(2) and no usable output"),
11654 failure_signature("exited with Some(137) and no usable output")
11655 );
11656 assert_eq!(
11657 failure_signature("process exited with no usable output (exit code 2)"),
11658 failure_signature("process exited with no usable output (exit code 137)")
11659 );
11660 for (old, new) in [
11661 (
11662 "exited with Some(2) and no usable output",
11663 "process exited with no usable output (exit code 2)",
11664 ),
11665 (
11666 "exited with None and no usable output",
11667 "process exited with no usable output (exit code unavailable)",
11668 ),
11669 ] {
11670 let previous = FailClass::Other(failure_signature(old));
11671 let current = FailClass::Other(failure_signature(new));
11672 assert_eq!(previous, current, "resumption retains the failure class");
11673 assert!(!should_hand_over(Some(&previous), ¤t));
11674 }
11675 assert_eq!(
11676 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11677 failure_signature("cannot open /var/x.txt: denied")
11678 );
11679 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11680 }
11681
11682 #[test]
11683 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11684 let other = FailClass::Other("x".into());
11685 assert!(should_hand_over(None, &FailClass::Quota));
11686 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11687 assert!(should_hand_over(
11688 Some(&FailClass::Timeout),
11689 &FailClass::Timeout
11690 ));
11691 assert!(should_hand_over(None, &other));
11692 assert!(!should_hand_over(Some(&other), &other));
11693 assert!(should_hand_over(
11694 Some(&other),
11695 &FailClass::Other("y".into())
11696 ));
11697 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11699 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11700 }
11701
11702 #[test]
11703 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11704 let a = SeatState::new("judge-1", "alpha", 7);
11705 let b = handover_seat("judge-1", "beta", 7);
11706 assert_ne!(a.claude_session, b.claude_session);
11707 assert_eq!(b.turns, 0);
11708 }
11709
11710 #[test]
11711 fn a_timeout_is_classified_apart_from_other_failures() {
11712 assert_eq!(
11713 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11714 Some(FailClass::Timeout)
11715 );
11716 assert!(matches!(
11717 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11718 Some(FailClass::Other(_))
11719 ));
11720 }
11721
11722 #[test]
11723 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11724 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11725 let tried = BTreeSet::from(["beta".to_owned()]);
11726 let next = next_untried_in_roster(&roster, 1, &tried);
11729 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11730 }
11731
11732 #[test]
11733 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11734 let roster = vec![spec("alpha"), spec("beta")];
11735 let tried = BTreeSet::from(["beta".to_owned()]);
11736 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11740 }
11741
11742 #[test]
11743 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11744 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11745 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11746 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11750 }
11751
11752 #[test]
11753 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11754 let roster = vec![spec("a"), spec("a"), spec("b")];
11755 let tried = BTreeSet::from(["a".to_owned()]);
11756 let next = next_untried_in_roster(&roster, 0, &tried);
11757 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11758 }
11759
11760 #[test]
11761 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11762 let roster = vec![spec("a"), spec("b")];
11763 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11764 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11765 }
11766
11767 #[test]
11768 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11769 let dir = tempfile::tempdir().unwrap();
11770 let bay = dir.path().join("ffff");
11771
11772 remove_if_empty(&bay);
11774 assert!(!bay.exists());
11775
11776 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11779 remove_if_empty(&bay);
11780 assert!(bay.exists(), "non-empty directory must survive");
11781
11782 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11784 remove_if_empty(&bay);
11785 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11786 }
11787
11788 #[test]
11797 fn a_full_panel_that_found_nothing_is_clean() {
11798 assert!(round_is_clean(
11799 0,
11800 true,
11801 2,
11802 2,
11803 0,
11804 IncompleteReviewPolicy::Block
11805 ));
11806 }
11807
11808 #[test]
11809 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11810 assert!(!round_is_clean(
11811 0,
11812 true,
11813 1,
11814 2,
11815 0,
11816 IncompleteReviewPolicy::Block
11817 ));
11818 }
11819
11820 #[test]
11821 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11822 assert!(!round_is_clean(
11823 1,
11824 true,
11825 1,
11826 2,
11827 0,
11828 IncompleteReviewPolicy::Warn
11829 ));
11830 }
11831
11832 #[test]
11833 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11834 assert!(round_is_clean(
11835 0,
11836 true,
11837 1,
11838 2,
11839 0,
11840 IncompleteReviewPolicy::Warn
11841 ));
11842 }
11843
11844 #[test]
11845 fn a_full_panel_with_an_open_finding_is_not_clean() {
11846 assert!(!round_is_clean(
11847 1,
11848 true,
11849 2,
11850 2,
11851 0,
11852 IncompleteReviewPolicy::Block
11853 ));
11854 }
11855
11856 #[test]
11857 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11858 assert!(!round_is_clean(
11859 0,
11860 false,
11861 2,
11862 2,
11863 0,
11864 IncompleteReviewPolicy::Block
11865 ));
11866 }
11867
11868 #[test]
11875 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11876 assert!(round_is_clean(
11879 0,
11880 true,
11881 1,
11882 2,
11883 1,
11884 IncompleteReviewPolicy::Block
11885 ));
11886 }
11887
11888 #[test]
11889 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11890 assert!(!round_is_clean(
11893 0,
11894 true,
11895 1,
11896 2,
11897 0,
11898 IncompleteReviewPolicy::Block
11899 ));
11900 }
11901
11902 #[test]
11903 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11904 assert!(!round_is_clean(
11905 1,
11906 true,
11907 1,
11908 2,
11909 1,
11910 IncompleteReviewPolicy::Block
11911 ));
11912 assert!(!round_is_clean(
11913 0,
11914 false,
11915 1,
11916 2,
11917 1,
11918 IncompleteReviewPolicy::Block
11919 ));
11920 }
11921
11922 #[test]
11923 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11924 assert!(!round_is_clean(
11928 0,
11929 true,
11930 0,
11931 2,
11932 2,
11933 IncompleteReviewPolicy::Block
11934 ));
11935 }
11936
11937 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11938 CommandOutcome {
11939 command: "test".to_owned(),
11940 code,
11941 output_tail: String::new(),
11942 duration_ms: 0,
11943 resource_blocked,
11944 }
11945 }
11946
11947 #[test]
11948 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11949 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11950 assert!(
11951 !verify_inconclusive(&[outcome(Some(1), false)]),
11952 "an ordinary failure is still evidence about the patch"
11953 );
11954 assert!(verify_inconclusive(&[outcome(None, true)]));
11955 assert!(
11956 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11957 "one inconclusive outcome taints the whole batch"
11958 );
11959 assert!(!verify_inconclusive(&[]));
11960 }
11961
11962 #[tokio::test]
11963 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11964 let calls = std::sync::atomic::AtomicUsize::new(0);
11968 let started = Instant::now();
11969 wait_for_pids_with(
11970 &[123],
11971 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11972 Duration::from_millis(5),
11973 Duration::from_secs(5),
11974 )
11975 .await;
11976 assert!(
11977 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11978 "must keep checking rather than deciding on the first answer"
11979 );
11980 assert!(
11981 started.elapsed() < Duration::from_secs(1),
11982 "must return the moment it is confirmed dead, not wait out the ceiling"
11983 );
11984 }
11985
11986 #[tokio::test]
11987 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11988 let started = Instant::now();
11989 wait_for_pids_with(
11990 &[123],
11991 |_| true, Duration::from_millis(5),
11993 Duration::from_millis(30),
11994 )
11995 .await;
11996 let elapsed = started.elapsed();
11997 assert!(
11998 elapsed >= Duration::from_millis(30),
11999 "must not give up before its own ceiling: {elapsed:?}"
12000 );
12001 assert!(
12002 elapsed < Duration::from_secs(1),
12003 "must not wait past its own ceiling either: {elapsed:?}"
12004 );
12005 }
12006
12007 #[tokio::test]
12008 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12009 let started = Instant::now();
12010 wait_for_pids_with(
12011 &[],
12012 |_| true,
12013 Duration::from_secs(5),
12014 Duration::from_secs(5),
12015 )
12016 .await;
12017 assert!(
12018 started.elapsed() < Duration::from_millis(200),
12019 "an empty pid list has nothing to confirm"
12020 );
12021 }
12022
12023 fn review_round(
12029 clean: bool,
12030 blocking: usize,
12031 answered: usize,
12032 expected: usize,
12033 progressed: bool,
12034 e2e_ok: bool,
12035 ) -> ReviewRound {
12036 ReviewRound {
12037 round: 1,
12038 head: "h".to_owned(),
12039 verified_head: None,
12040 verified_at: None,
12041 reviews: Vec::new(),
12042 e2e: vec![CommandOutcome {
12043 command: "test".to_owned(),
12044 code: Some(if e2e_ok { 0 } else { 1 }),
12045 output_tail: String::new(),
12046 duration_ms: 0,
12047 resource_blocked: false,
12048 }],
12049 verify_retried: false,
12050 e2e_deferred: false,
12051 e2e_defer_reason: None,
12052 fix: None,
12053 blocking,
12054 answered,
12055 expected,
12056 clean,
12057 progressed,
12058 vote_split: false,
12059 reconsideration: Vec::new(),
12060 verdict: None,
12061 }
12062 }
12063
12064 #[test]
12065 fn review_conclusion_is_none_when_nothing_has_run() {
12066 assert_eq!(review_conclusion(&[], 3), None);
12067 }
12068
12069 #[test]
12070 fn review_conclusion_is_none_while_rounds_remain() {
12071 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12072 assert_eq!(review_conclusion(&rounds, 3), None);
12073 }
12074
12075 #[test]
12076 fn review_conclusion_is_gating_once_a_round_is_clean() {
12077 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12078 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12079 }
12080
12081 #[test]
12082 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12083 let rounds = vec![
12084 review_round(false, 1, 2, 2, true, true),
12085 review_round(false, 1, 2, 2, true, true),
12086 ];
12087 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12088 }
12089
12090 #[test]
12091 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12092 let rounds = vec![
12093 review_round(false, 1, 2, 2, true, true),
12094 review_round(false, 1, 2, 2, true, false),
12095 ];
12096 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12097 }
12098
12099 #[test]
12100 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12101 let mut blocked = review_round(false, 1, 2, 2, true, false);
12108 blocked.e2e[0].resource_blocked = true;
12109 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12110 assert_eq!(review_conclusion(&rounds, 2), None);
12111 }
12112
12113 #[test]
12114 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12115 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12117 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12118 }
12119
12120 #[test]
12121 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12122 let rounds = vec![
12123 review_round(false, 1, 2, 2, false, true),
12124 review_round(false, 1, 2, 2, false, true),
12125 ];
12126 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12127 }
12128
12129 fn secs(n: u64) -> Duration {
12130 Duration::from_secs(n)
12131 }
12132
12133 #[tokio::test]
12136 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12137 ask_test_home();
12138 let tmp = tempfile::tempdir().unwrap();
12139 let repo = tmp.path().join("repo");
12140 std::fs::create_dir_all(&repo).unwrap();
12141 init_repo(&repo);
12142 let g = |args: &[&str]| {
12143 let out = std::process::Command::new("git")
12144 .args(args)
12145 .current_dir(&repo)
12146 .quiet()
12147 .output()
12148 .unwrap();
12149 assert!(
12150 out.status.success(),
12151 "{}",
12152 String::from_utf8_lossy(&out.stderr)
12153 );
12154 };
12155 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12156 g(&["checkout", "-b", "historical"]);
12157 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12158 g(&["add", "."]);
12159 g(&["commit", "-m", "old alternative"]);
12160 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12161 g(&["checkout", "-b", "magi/task-selected", &base]);
12162 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12163 g(&["add", "."]);
12164 g(&["commit", "-m", "measurement infrastructure"]);
12165 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12166 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12167 g(&["add", "."]);
12168 g(&["commit", "-m", "dependent implementation"]);
12169 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12170 g(&["checkout", "main"]);
12171 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12172 g(&["add", "."]);
12173 g(&["commit", "-m", "unrelated base change"]);
12174 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12175 let origin = tmp.path().join("origin.git");
12176 g(&[
12177 "clone",
12178 "--bare",
12179 repo.to_str().unwrap(),
12180 origin.to_str().unwrap(),
12181 ]);
12182 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12183 g(&["fetch", "origin"]);
12184 let selection = crate::pr_recovery::Selection {
12185 url: "https://example.test/project/pull/1".into(),
12186 branch: "magi/task-selected".into(),
12187 head: head.clone(),
12188 source_head: None,
12189 disposition: Some(crate::pr_recovery::Disposition::Review),
12190 instruction: "preserve measured behavior".into(),
12191 approved_at: jiff::Timestamp::now(),
12192 };
12193 let mut template = runner_at(RunStatus::Reviewing);
12194 template.state.repo = repo.clone();
12195 template.state.base_commit = current.clone();
12196 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12197 template.state.origin = Some(crate::run::Origin::queue("selected"));
12198 template.state.pr_head = Some(selection.branch.clone());
12199 template.state.pr_recovery = Some(selection.clone());
12200 let mut runner =
12201 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12202 .await
12203 .unwrap();
12204 let wt = runner.state.winner().unwrap().worktree.clone();
12205 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12206 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12207 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12208 assert!(runner.state.seeds.is_empty());
12209 assert_eq!(runner.state.instruction, "preserve measured behavior");
12210 runner.sync_to_base().await.unwrap();
12211 assert_eq!(
12212 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12213 "prerequisite"
12214 );
12215 assert_eq!(
12216 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12217 "measured implementation"
12218 );
12219 assert_eq!(
12220 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12221 "current main change"
12222 );
12223 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12224 assert_eq!(
12225 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12226 head
12227 );
12228
12229 let mut partial = runner_at(RunStatus::Reviewing);
12231 partial.state.repo = repo.clone();
12232 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12233 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12234 run: Some("exit 1".into()),
12235 ..Default::default()
12236 }];
12237 let selection = runner.state.pr_recovery.clone().unwrap();
12238 partial.state.pr_recovery = Some(selection.clone());
12239 partial.state.pr_head = Some(selection.branch.clone());
12240 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12241 let id = partial.state.id.clone();
12242 let error = match Runner::open_selected(
12243 &repo,
12244 partial.state,
12245 partial.roles,
12246 1,
12247 runner.state.base_commit.clone(),
12248 selection,
12249 )
12250 .await
12251 {
12252 Ok(_) => panic!("setup must fail"),
12253 Err(e) => e,
12254 };
12255 assert_eq!(
12256 error
12257 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12258 .unwrap()
12259 .run,
12260 id
12261 );
12262 let failed = RunState::load(&id).unwrap();
12263 assert_eq!(failed.status, RunStatus::Blocked);
12264 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12265 assert!(Runner::resume(&id).is_err());
12266 runner.state.status = RunStatus::Ready;
12267 runner.state.driver_exited = true;
12268 runner.state.save().unwrap();
12269 let mut retry = runner_at(RunStatus::Reviewing);
12270 retry.state.repo = repo.clone();
12271 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12272 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12273 reselected.approved_at = jiff::Timestamp::now();
12274 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12275 retry.state.pr_head = Some(reselected.branch.clone());
12276 retry.state.pr_recovery = Some(reselected.clone());
12277 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12278 .await
12279 .unwrap();
12280 let restarted = Runner::open_selected(
12281 &repo,
12282 retry.state,
12283 retry.roles,
12284 1,
12285 runner.state.base_commit.clone(),
12286 reselected,
12287 )
12288 .await
12289 .unwrap();
12290 assert_eq!(
12291 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12292 .await
12293 .unwrap(),
12294 head
12295 );
12296 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12297 assert_eq!(
12298 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12299 head
12300 );
12301 }
12302
12303 #[tokio::test]
12304 #[cfg(unix)]
12305 async fn fallback_and_interrupted_restoration_require_coordination() {
12306 use std::os::fd::AsRawFd as _;
12307 ask_test_home();
12308 let tmp = tempfile::tempdir().unwrap();
12309 let mut runner = registered_runner(tmp.path()).await;
12310 runner.state.config.graph.timeout_resource_wait = 1;
12311 let winner = runner.state.candidates[0].clone();
12312 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12313 let leftover = winner.worktree.join("leftover");
12314 std::fs::write(&leftover, "untrusted edits").unwrap();
12315 let next = tmp.path().join("next-fixer");
12316 let mut failed = spec("first");
12317 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12318 let mut successor = spec("second");
12319 successor.command = vec![
12320 "sh".into(),
12321 "-c".into(),
12322 format!("touch '{}'; echo done", next.display()),
12323 ];
12324 let attempts = vec![
12325 (failed.clone(), "first".into()),
12326 (successor.clone(), "second".into()),
12327 ];
12328 let lock_path = runner.state.repo.join(".git/updater.lock");
12329 let external = std::fs::File::create(&lock_path).unwrap();
12330 assert_eq!(
12331 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12332 0
12333 );
12334 let artifacts = tmp.path().join("artifacts");
12335 let build = |spec, seat| SeatJob {
12336 spec,
12337 seat,
12338 cwd: winner.worktree.clone(),
12339 prompt: "fixture".into(),
12340 timeout: Duration::from_secs(10),
12341 allow_write: true,
12342 sessions: false,
12343 artifacts: artifacts.clone(),
12344 stem: "fix".into(),
12345 handover: None,
12346 };
12347 let (_, _, out) = runner
12348 .ask_fixer_among(
12349 attempts.clone(),
12350 "graft",
12351 None,
12352 Some((&winner.worktree, &head)),
12353 build,
12354 )
12355 .await;
12356 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12357 assert!(leftover.exists(), "contention must prevent clean");
12358 assert!(!next.exists());
12359 assert!(runner.state.handovers.is_empty());
12360 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12362 from_run: "earlier".into(),
12363 generation: 1,
12364 mode: crate::recovery::Mode::VerifyRepair,
12365 reasons: Vec::new(),
12366 reviewed_head: None,
12367 rounds: Vec::new(),
12368 open_findings: Vec::new(),
12369 evidence: None,
12370 criteria_inherited: true,
12371 repair: crate::recovery::Repair::Started {
12372 before: head.clone(),
12373 },
12374 });
12375 assert!(
12376 runner
12377 .recovery_repair(&winner)
12378 .await
12379 .unwrap_err()
12380 .is::<crate::repo_lock::ResourceBlocked>()
12381 );
12382 assert!(matches!(
12383 runner.state.recovery.as_ref().unwrap().repair,
12384 crate::recovery::Repair::Started { .. }
12385 ));
12386 assert!(leftover.exists());
12387 drop(external);
12388 runner.recovery_repair(&winner).await.unwrap();
12389 assert!(!leftover.exists());
12390 assert!(matches!(
12391 runner.state.recovery.as_ref().unwrap().repair,
12392 crate::recovery::Repair::Done { .. }
12393 ));
12394 std::fs::remove_file(&lock_path).unwrap();
12396 std::fs::create_dir(&lock_path).unwrap();
12397 std::fs::write(&leftover, "untrusted edits").unwrap();
12398 let (_, _, out) = runner
12399 .ask_fixer_among(
12400 attempts,
12401 "graft",
12402 None,
12403 Some((&winner.worktree, &head)),
12404 build,
12405 )
12406 .await;
12407 assert!(matches!(out, AgentOutcome::Failed(_)));
12408 assert!(leftover.exists());
12409 assert!(!next.exists());
12410 assert_eq!(
12411 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12412 head
12413 );
12414 }
12415
12416 #[cfg(unix)]
12417 async fn registered_runner(root: &Path) -> Runner {
12418 ask_test_home();
12419 let repo = root.join("repo");
12420 std::fs::create_dir_all(&repo).unwrap();
12421 init_repo(&repo);
12422 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12423 let mut runner = runner_at(RunStatus::Prep);
12424 runner.state.repo = repo.clone();
12425 runner.state.config.coordination = crate::config::test_coordination();
12426 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12427 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12428 runner.state.config.graph.advise = false;
12429 runner.roles.implementers = vec![spec("implement")];
12430 runner.roles.judges = vec![spec("judge")];
12431 runner.prep().await.unwrap();
12432 runner
12433 }
12434
12435 #[tokio::test]
12436 #[cfg(unix)]
12437 async fn parked_implementer_persists_completed_work() {
12438 for registered in [false, true] {
12439 let tmp = tempfile::tempdir().unwrap();
12440 let mut runner = registered_runner(tmp.path()).await;
12441 if !registered {
12442 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12443 }
12444 let marker = tmp.path().join("started");
12445 let release = tmp.path().join("release");
12446 let mut implementer = spec("implement");
12447 implementer.command = vec![
12448 "sh".into(),
12449 "-c".into(),
12450 format!(
12451 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12452 marker.display(),
12453 release.display()
12454 ),
12455 ];
12456 runner.roles.implementers = vec![implementer.clone()];
12457 runner.state.config.agents = vec![implementer];
12458 let pause = runner.pause.clone();
12459 let (result, ()) = tokio::join!(runner.implement(), async {
12460 tokio::time::timeout(Duration::from_secs(10), async {
12461 while !marker.exists() {
12462 tokio::time::sleep(Duration::from_millis(10)).await;
12463 }
12464 })
12465 .await
12466 .unwrap();
12467 pause.park();
12468 std::fs::write(&release, "go").unwrap();
12469 });
12470 result.unwrap();
12471 let candidate = &runner.state.candidates[0];
12472 assert!(candidate.commits > 0, "{candidate:?}");
12473 assert!(candidate.summary.contains("saved-result"));
12474 assert_eq!(candidate.files, 1);
12475 let before = candidate.commits;
12476 runner.implement().await.unwrap();
12477 assert_eq!(runner.state.candidates[0].commits, before);
12478 }
12479 }
12480
12481 #[tokio::test]
12482 #[cfg(unix)]
12483 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12484 let tmp = tempfile::tempdir().unwrap();
12485 let mut runner = registered_runner(tmp.path()).await;
12486 let winner = runner.state.candidates[0].clone();
12487 let cache = tmp.path().join("cache");
12488 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12489 runner.state.config.graph.timeout_resource_wait = 0;
12490 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12491 let crate::cache::AcquireOutcome::Acquired(held) =
12492 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12493 else {
12494 panic!("cache must be free")
12495 };
12496 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12497 from_run: "earlier".into(),
12498 generation: 1,
12499 mode: crate::recovery::Mode::VerifyRepair,
12500 reasons: Vec::new(),
12501 reviewed_head: None,
12502 rounds: Vec::new(),
12503 open_findings: Vec::new(),
12504 evidence: None,
12505 criteria_inherited: true,
12506 repair: crate::recovery::Repair::NotStarted,
12507 });
12508 runner.recovery_repair(&winner).await.unwrap();
12509 assert!(matches!(
12510 runner.state.recovery.as_ref().unwrap().repair,
12511 crate::recovery::Repair::WaitingForRestore { .. }
12512 ));
12513 drop(held);
12514 runner.state.resource_block = None;
12515 runner.state.config.graph.timeout_resource_wait = 1;
12516 runner.recovery_repair(&winner).await.unwrap();
12517 assert!(matches!(
12518 runner.state.recovery.as_ref().unwrap().repair,
12519 crate::recovery::Repair::Done { .. }
12520 ));
12521 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12522 }
12523
12524 #[tokio::test]
12525 #[cfg(unix)]
12526 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12527 use std::os::fd::AsRawFd as _;
12528 for graft in [false, true] {
12529 let tmp = tempfile::tempdir().unwrap();
12530 let mut runner = registered_runner(tmp.path()).await;
12531 runner.state.config.graph.timeout_resource_wait = 1;
12532 let winner = runner.state.candidates[0].clone();
12533 let marker = tmp.path().join("fixer-started");
12534 let mut fixer = spec("fixer");
12535 fixer.command = vec![
12536 "sh".into(),
12537 "-c".into(),
12538 format!("echo started >> '{}'; echo done", marker.display()),
12539 ];
12540 runner.roles.fixer = Some(vec![fixer]);
12541 if graft {
12542 let mut loser = winner.clone();
12543 loser.label = 'B';
12544 loser.index = 1;
12545 runner.state.candidates.push(loser);
12546 runner.state.config.graph.graft_runner_up = true;
12547 runner.state.tally = Some(Tally {
12548 first_choice: BTreeMap::new(),
12549 borda: BTreeMap::new(),
12550 winner: winner.label,
12551 rankings: 1,
12552 unanimous_initial: true,
12553 deliberated: false,
12554 changed_votes: 0,
12555 unanimous_final: true,
12556 tie_break: None,
12557 judges: 1,
12558 present: 1,
12559 quorum: 1,
12560 met_quorum: true,
12561 uncontested: None,
12562 });
12563 runner.state.judgements.push(crate::run::Judgement {
12564 judge: 1,
12565 seat: "judge-1".into(),
12566 agent: "judge".into(),
12567 ranking: vec![winner.label, 'B'],
12568 reasons: BTreeMap::new(),
12569 confidence: None,
12570 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12571 order: vec![0, 1],
12572 failed: None,
12573 duration_ms: 0,
12574 });
12575 } else {
12576 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12577 from_run: "earlier".into(),
12578 generation: 1,
12579 mode: crate::recovery::Mode::VerifyRepair,
12580 reasons: Vec::new(),
12581 reviewed_head: None,
12582 rounds: Vec::new(),
12583 open_findings: Vec::new(),
12584 evidence: None,
12585 criteria_inherited: true,
12586 repair: crate::recovery::Repair::NotStarted,
12587 });
12588 }
12589 let reader = crate::repo_lock::SeatLease::for_checkout(
12590 &winner.worktree,
12591 false,
12592 crate::config::test_coordination().as_ref(),
12593 )
12594 .await
12595 .unwrap();
12596 let pause = runner.pause.clone();
12597 let (result, ()) = tokio::join!(
12598 async {
12599 if graft {
12600 runner.graft().await
12601 } else {
12602 runner.recovery_repair(&winner).await
12603 }
12604 },
12605 async {
12606 tokio::time::sleep(Duration::from_millis(100)).await;
12607 pause.park();
12608 }
12609 );
12610 result.unwrap();
12611 assert!(!marker.exists(), "a blocked seat must not spawn");
12612 runner.state = RunState::load(&runner.state.id).unwrap();
12614 if graft {
12615 assert_eq!(
12616 runner.state.graft.as_ref().unwrap().outcome,
12617 GraftOutcome::WaitingForRestore
12618 );
12619 } else {
12620 assert!(matches!(
12621 runner.state.recovery.as_ref().unwrap().repair,
12622 crate::recovery::Repair::WaitingForRestore { .. }
12623 ));
12624 }
12625 drop(reader);
12626 runner.pause = Pause::new();
12627 runner.state.resource_block = None;
12628 let external =
12630 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12631 assert_eq!(
12632 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12633 0
12634 );
12635 let leftover = winner.worktree.join("leftover");
12636 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12637 let result = if graft {
12638 runner.graft().await
12639 } else {
12640 runner.recovery_repair(&winner).await
12641 };
12642 assert!(
12643 result
12644 .unwrap_err()
12645 .is::<crate::repo_lock::ResourceBlocked>()
12646 );
12647 assert!(leftover.exists());
12648 assert!(!marker.exists());
12649 drop(external);
12650 runner.state.resource_block = None;
12651 let result = if graft {
12652 runner.graft().await
12653 } else {
12654 runner.recovery_repair(&winner).await
12655 };
12656 result.unwrap();
12657 assert!(!leftover.exists(), "restore must precede retry");
12658 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12659 if graft {
12660 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12661 } else {
12662 assert!(matches!(
12663 runner.state.recovery.as_ref().unwrap().repair,
12664 crate::recovery::Repair::Done { .. }
12665 ));
12666 }
12667 }
12668 }
12669
12670 #[tokio::test]
12671 #[cfg(unix)]
12672 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12673 use std::os::fd::AsRawFd as _;
12674 let tmp = tempfile::tempdir().unwrap();
12675 let mut runner = registered_runner(tmp.path()).await;
12676 runner.state.config.graph.timeout_resource_wait = 1;
12677 let winner = runner.state.candidates[0].clone();
12678 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12679 runner.state.tally = Some(Tally {
12680 first_choice: BTreeMap::new(),
12681 borda: BTreeMap::new(),
12682 winner: winner.label,
12683 rankings: 0,
12684 unanimous_initial: false,
12685 deliberated: false,
12686 changed_votes: 0,
12687 unanimous_final: false,
12688 tie_break: None,
12689 judges: 0,
12690 present: 0,
12691 quorum: 0,
12692 met_quorum: true,
12693 uncontested: Some("fixture".into()),
12694 });
12695 runner.state.graft = Some(GraftRecord {
12696 pre: pre.clone(),
12697 post: None,
12698 lines: None,
12699 limit: None,
12700 agent: None,
12701 outcome: GraftOutcome::Running,
12702 requested: Vec::new(),
12703 adoption: None,
12704 adoption_error: None,
12705 });
12706 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12707 git::commit_all(&winner.worktree, "unfinished graft")
12708 .await
12709 .unwrap();
12710 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12711 let leftover = winner.worktree.join("leftover");
12712 std::fs::write(&leftover, "untrusted edits").unwrap();
12713 let path = runner.state.repo.join(".git/updater.lock");
12714 let external = std::fs::File::create(&path).unwrap();
12715 assert_eq!(
12716 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12717 0
12718 );
12719 assert!(
12720 runner
12721 .graft()
12722 .await
12723 .unwrap_err()
12724 .is::<crate::repo_lock::ResourceBlocked>()
12725 );
12726 assert!(matches!(
12727 runner.state.graft.as_ref().unwrap().outcome,
12728 GraftOutcome::Running
12729 ));
12730 assert_eq!(
12731 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12732 unfinished
12733 );
12734 assert!(leftover.exists());
12735 drop(external);
12736 std::fs::remove_file(&path).unwrap();
12737 std::fs::create_dir(&path).unwrap();
12738 assert!(runner.graft().await.is_err());
12739 assert!(matches!(
12740 runner.state.graft.as_ref().unwrap().outcome,
12741 GraftOutcome::Running
12742 ));
12743 assert_eq!(
12744 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12745 unfinished
12746 );
12747 assert!(leftover.exists());
12748 std::fs::remove_dir(&path).unwrap();
12749 runner.graft().await.unwrap();
12750 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12751 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12752 assert!(!leftover.exists());
12753 }
12754
12755 #[tokio::test]
12756 #[cfg(unix)]
12757 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12758 use std::os::fd::AsRawFd as _;
12759 let tmp = tempfile::tempdir().unwrap();
12760 let mut runner = registered_runner(tmp.path()).await;
12761 runner.state.prep_complete = Some(false);
12762 runner.state.config.graph.timeout_resource_wait = 1;
12763 runner.state.save().unwrap();
12764 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12765 assert_eq!(
12766 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12767 0
12768 );
12769 runner.execute().await.unwrap();
12770 let disk = RunState::load(&runner.state.id).unwrap();
12771 assert!(disk.driver_exited);
12772 assert_eq!(disk.driver_pid, None);
12773 assert_eq!(runner.state.status, RunStatus::Blocked);
12774 assert_eq!(
12775 runner.state.resource_block.as_ref().unwrap().kind,
12776 "repository coordination"
12777 );
12778 assert!(runner.state.prep_failure.is_none());
12779 assert!(runner.state.seats.is_empty());
12780 runner.pause.park();
12781 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12784 .await
12785 .unwrap()
12786 .unwrap();
12787 assert!(runner.state.parked);
12788 assert!(runner.state.prep_failure.is_none());
12789 let disk = RunState::load(&runner.state.id).unwrap();
12790 assert!(disk.driver_exited);
12791 assert_eq!(disk.driver_pid, None);
12792 }
12793
12794 #[tokio::test]
12795 #[cfg(unix)]
12796 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12797 use std::os::fd::AsRawFd as _;
12798 let tmp = tempfile::tempdir().unwrap();
12799 let mut runner = registered_runner(tmp.path()).await;
12800 runner.state.prep_complete = Some(false);
12801 runner.state.config.graph.timeout_resource_wait = 30;
12802 runner.state.save().unwrap();
12803 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12804 assert_eq!(
12805 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12806 0
12807 );
12808 assert!(
12809 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12810 .await
12811 .is_err()
12812 );
12813 let disk = RunState::load(&runner.state.id).unwrap();
12814 assert!(disk.driver_exited);
12815 assert_eq!(disk.driver_pid, None);
12816 assert!(disk.status.resumable());
12817 assert!(runner.state.driver_exited);
12818 assert_eq!(runner.state.driver_pid, None);
12819 runner.state.save().unwrap();
12820 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12821 }
12822
12823 #[tokio::test]
12824 #[cfg(unix)]
12825 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12826 let tmp = tempfile::tempdir().unwrap();
12827 let mut runner = registered_runner(tmp.path()).await;
12828 let winner = runner.state.candidates[0].clone();
12829 let cache = tmp.path().join("cache");
12830 let started = tmp.path().join("pre-gate-started");
12831 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12832 runner.state.config.verify.gate =
12833 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12834 let home = crate::run::home();
12835 let owner =
12836 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12837 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12838 crate::cache::AcquireOutcome::Acquired(g) => g,
12839 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12840 };
12841 let mut gate = Box::pin(runner.run_gate(&winner));
12842 assert!(
12843 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12844 .await
12845 .is_err()
12846 );
12847 assert!(!started.exists(), "pre-gate must wait for the cache too");
12848 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12852 let editor = crate::repo_lock::SeatLease::checkout_with(
12853 &winner.worktree,
12854 true,
12855 crate::config::test_coordination().as_ref(),
12856 &wait,
12857 )
12858 .await
12859 .unwrap();
12860 drop(editor);
12861 guard.release();
12862 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12863 .await
12864 .unwrap()
12865 .unwrap();
12866 drop(gate);
12867 assert!(outcomes.iter().all(CommandOutcome::ok));
12868 assert!(started.exists());
12869 }
12870
12871 #[tokio::test]
12872 #[cfg(unix)]
12873 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12874 use std::os::fd::AsRawFd as _;
12875
12876 ask_test_home();
12877 let tmp = tempfile::tempdir().unwrap();
12878 let repo = tmp.path().join("repo");
12879 std::fs::create_dir_all(&repo).unwrap();
12880 init_repo(&repo);
12881 let common = repo.join(".git");
12882 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12883 let mut runner = runner_at(RunStatus::Prep);
12884 runner.state.repo = repo.clone();
12885 runner.state.config.coordination = crate::config::test_coordination();
12886 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12887 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12888 runner.state.config.graph.advise = false;
12889 runner.roles.implementers = vec![spec("implement")];
12890 runner.roles.judges = vec![spec("judge")];
12891 runner.prep().await.unwrap();
12892 let winner = runner.state.candidates[0].clone();
12893 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12896 let completed = tmp.path().join("seat-completed");
12897 runner.roles.implementers[0].command = vec![
12898 "sh".into(),
12899 "-c".into(),
12900 format!("touch '{}'; echo completed", completed.display()),
12901 ];
12902 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12903 assert_eq!(
12904 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12905 0
12906 );
12907 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12908 let mut implement = Box::pin(runner.implement());
12909 tokio::select! {
12910 result = &mut implement => panic!("rescue must wait: {result:?}"),
12911 result = tokio::time::timeout(Duration::from_secs(10), async {
12912 while !completed.exists() {
12913 tokio::time::sleep(Duration::from_millis(20)).await;
12914 }
12915 }) => result.unwrap(),
12916 }
12917 assert!(
12918 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12919 .await
12920 .is_err()
12921 );
12922 assert_eq!(
12923 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12924 before
12925 );
12926 assert_eq!(
12927 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12928 0
12929 );
12930 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12931 .await
12932 .unwrap()
12933 .unwrap();
12934 drop(implement);
12935 assert_ne!(
12936 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12937 before
12938 );
12939 assert!(git::is_clean(&winner.worktree).await.unwrap());
12940
12941 let started = tmp.path().join("pre-gate-started");
12942 let proceed = tmp.path().join("pre-gate-proceed");
12943 let checked = tmp.path().join("gate-ran");
12944 runner.state.config.verify.pre_gate = vec![format!(
12945 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12946 started.display(),
12947 proceed.display()
12948 )];
12949 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12950 assert_eq!(
12951 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12952 0
12953 );
12954 let mut gate = Box::pin(runner.run_gate(&winner));
12955 assert!(
12956 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12957 .await
12958 .is_err()
12959 );
12960 assert!(!started.exists());
12961 assert!(!checked.exists());
12962 assert!(git::is_clean(&winner.worktree).await.unwrap());
12963 assert_eq!(
12964 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12965 0
12966 );
12967 tokio::select! {
12968 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12969 result = tokio::time::timeout(Duration::from_secs(10), async {
12970 while !started.exists() {
12971 tokio::time::sleep(Duration::from_millis(20)).await;
12972 }
12973 }) => result.unwrap(),
12974 }
12975 assert_eq!(
12976 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12977 -1,
12978 "mechanical edits must hold the real lock"
12979 );
12980 std::fs::write(proceed, "ready").unwrap();
12981 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12982 .await
12983 .unwrap()
12984 .unwrap();
12985 drop(gate);
12986 assert!(outcomes.iter().all(CommandOutcome::ok));
12987 assert!(checked.exists());
12988 assert!(runner.state.pre_gate_commit.is_some());
12989 assert!(git::is_clean(&winner.worktree).await.unwrap());
12990 assert_eq!(
12991 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12992 0,
12993 "gate completion must release coordination"
12994 );
12995 drop(outsider);
12996 std::fs::remove_file(common.join("updater.lock")).unwrap();
12997 std::fs::create_dir(common.join("updater.lock")).unwrap();
12998 std::fs::remove_file(&started).unwrap();
12999 std::fs::remove_file(&checked).unwrap();
13000 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13001 let error = runner.run_gate(&winner).await.unwrap_err();
13002 assert!(
13003 error
13004 .to_string()
13005 .contains("acquire mandatory repository lock")
13006 );
13007 assert!(
13008 !started.exists(),
13009 "failed acquisition must prevent pre-gate"
13010 );
13011 assert!(!checked.exists(), "failed acquisition must prevent gate");
13012 assert!(git::is_clean(&winner.worktree).await.unwrap());
13013 assert_eq!(
13014 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13015 head
13016 );
13017 }
13018
13019 #[tokio::test]
13020 #[cfg(unix)]
13021 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13022 use std::os::fd::AsRawFd as _;
13023 use std::time::Duration;
13024
13025 ask_test_home();
13026 let tmp = tempfile::tempdir().unwrap();
13027 let repo = tmp.path().join("repo");
13028 std::fs::create_dir_all(&repo).unwrap();
13029 init_repo(&repo);
13030 let common = repo.join(".git");
13031 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13032 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13033 assert_eq!(
13034 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13035 0
13036 );
13037 let mut runner = runner_at(RunStatus::Prep);
13038 runner.state.repo = repo.clone();
13039 runner.state.config.coordination = crate::config::test_coordination();
13040 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13041 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13042 runner.state.config.graph.advise = true;
13043 runner.state.config.graph.advisors = 1;
13044 runner.roles.implementers = vec![spec("implement")];
13045 runner.roles.judges = vec![spec("judge")];
13046 let root = runner.state.worktree_root();
13047 let started = tmp.path().join("setup-started");
13048 let proceed = tmp.path().join("setup-proceed");
13049 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13050 run: Some(format!(
13051 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13052 started.display(),
13053 proceed.display()
13054 )),
13055 ..Default::default()
13056 }];
13057 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13058 let config = git::git(&repo, &["config", "--local", "--list"])
13059 .await
13060 .unwrap();
13061 let mut prep = Box::pin(runner.prep());
13062 assert!(
13063 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13064 .await
13065 .is_err()
13066 );
13067 assert!(
13068 !root.exists(),
13069 "contention must prevent even checkout creation"
13070 );
13071 assert!(!started.exists(), "contention must prevent setup");
13072 assert_eq!(
13073 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13074 branches
13075 );
13076 assert_eq!(
13077 git::git(&repo, &["config", "--local", "--list"])
13078 .await
13079 .unwrap(),
13080 config
13081 );
13082 drop(lock);
13083 tokio::select! {
13084 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13085 result = tokio::time::timeout(Duration::from_secs(10), async {
13086 while !started.exists() {
13087 tokio::time::sleep(Duration::from_millis(20)).await;
13088 }
13089 }) => result.unwrap(),
13090 }
13091 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13092 assert_eq!(
13093 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13094 -1
13095 );
13096 assert!(
13097 tokio::time::timeout(
13098 Duration::from_millis(100),
13099 crate::repo_lock::SeatLease::for_checkout(
13100 &repo,
13101 true,
13102 crate::config::test_coordination().as_ref()
13103 )
13104 )
13105 .await
13106 .is_err(),
13107 "preparation must also exclude local editors"
13108 );
13109 std::fs::write(proceed, "ready").unwrap();
13110 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13111 .await
13112 .unwrap()
13113 .unwrap();
13114 drop(prep);
13115 assert_eq!(runner.state.prep_complete, Some(true));
13116 assert!(root.join("judge-1").exists());
13117 assert!(root.join("advisor-1").exists());
13118 assert_eq!(
13119 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13120 0
13121 );
13122 }
13123
13124 #[tokio::test]
13125 #[cfg(unix)]
13126 async fn preparation_lock_failure_prevents_repository_changes() {
13127 ask_test_home();
13128 let tmp = tempfile::tempdir().unwrap();
13129 let repo = tmp.path().join("repo");
13130 std::fs::create_dir_all(&repo).unwrap();
13131 init_repo(&repo);
13132 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13133 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13136 let mut runner = runner_at(RunStatus::Prep);
13137 runner.state.repo = repo.clone();
13138 runner.state.config.coordination = crate::config::test_coordination();
13139 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13140 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13141 runner.roles.implementers = vec![spec("implement")];
13142 let error = runner.prep().await.unwrap_err();
13143 assert!(
13144 error
13145 .to_string()
13146 .contains("acquire mandatory repository lock")
13147 );
13148 assert!(!runner.state.worktree_root().exists());
13149 assert!(runner.state.candidates.is_empty());
13150 assert!(!repo.join(".git/config.worktree").exists());
13151 assert_eq!(
13152 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13153 "* main"
13154 );
13155 }
13156
13157 #[tokio::test]
13158 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13159 ask_test_home();
13160 let tmp = tempfile::tempdir().unwrap();
13161 let repo = tmp.path().join("repo");
13162 std::fs::create_dir_all(&repo).unwrap();
13163 init_repo(&repo);
13164 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13165 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13166 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13167 git::git(&repo, &["add", "."]).await.unwrap();
13168 git::git(&repo, &["commit", "-m", "required seed"])
13169 .await
13170 .unwrap();
13171 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13172 git::git(&repo, &["checkout", "main"]).await.unwrap();
13173 let mut runner = runner_at(RunStatus::Prep);
13174 runner.state.repo = repo.clone();
13175 runner.state.base_commit = base.clone();
13176 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13177 runner.state.config.graph.advise = false;
13178 runner.state.config.blind.commit_msg_hook = false;
13179 runner.state.seeds = vec![refs::Seed {
13180 token: seed.clone(),
13181 sha: seed.clone(),
13182 kind: refs::SeedKind::Unmerged,
13183 branch: false,
13184 detail: String::new(),
13185 }];
13186 runner.roles.implementers = vec![spec("one"), spec("two")];
13187 runner.roles.judges = vec![spec("judge")];
13188 let label = blind::assign_labels(2, runner.state.seed)[0];
13189 let branch = runner.state.branch_for(label);
13190 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13191 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13192 .await
13193 .unwrap();
13194 runner.state.candidates.push(Candidate {
13195 index: 0,
13196 label,
13197 agent: "one".into(),
13198 branch,
13199 worktree,
13200 summary: String::new(),
13201 stat: String::new(),
13202 files: 0,
13203 commits: 0,
13204 empty: false,
13205 failed: None,
13206 verified_noop: None,
13207 duration_ms: 0,
13208 folded: false,
13209 });
13210 runner.state.save().unwrap();
13211 runner.state = RunState::load(&runner.state.id).unwrap();
13213 assert_eq!(runner.state.prep_complete, Some(false));
13214 runner.prep().await.unwrap();
13215 assert_eq!(runner.state.prep_complete, Some(true));
13216 assert_eq!(runner.state.candidates.len(), 2);
13217 for candidate in &runner.state.candidates {
13218 assert_eq!(
13219 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13220 git::tree_of(&repo, &seed).await.unwrap()
13221 );
13222 assert_eq!(
13223 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13224 "required commit"
13225 );
13226 }
13227 assert!(runner.state.worktree_root().join("judge-1").exists());
13228 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13229 runner.prep().await.unwrap();
13230 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13231
13232 runner.state.prep_complete = Some(false);
13233 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13234 run: Some(format!(
13235 "test -f '{}'",
13236 tmp.path().join("dependency-ready").display()
13237 )),
13238 ..Default::default()
13239 }];
13240 assert!(runner.prep().await.is_err());
13241 assert!(runner.state.prep_failure.is_some());
13242 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13243 runner.state = RunState::load(&runner.state.id).unwrap();
13244 assert!(
13245 runner.prep().await.is_err(),
13246 "correcting the cause alone must not auto-retry"
13247 );
13248 let failed_at = runner.state.events.last().unwrap().at;
13249 assert!(
13250 runner.retry_preparation(failed_at).is_err(),
13251 "old approvals cannot unlock the hold"
13252 );
13253 let mut task = crate::queue::Task::new(
13254 "retry".into(),
13255 "retry setup".into(),
13256 repo.clone(),
13257 crate::queue::Source::Human,
13258 );
13259 task.start(runner.state.id.clone());
13260 task.hold_machine(Some("setup failed".into()));
13261 task.release_operator("cli", None);
13262 runner
13263 .retry_preparation(task.resumption.as_ref().unwrap().at)
13264 .unwrap();
13265 runner.prep().await.unwrap();
13266 assert!(runner.state.prep_failure.is_none());
13267 assert_eq!(runner.state.prep_complete, Some(true));
13268 assert_eq!(runner.state.candidates.len(), 2);
13269 assert!(runner.state.worktree_root().join("judge-1").exists());
13270 for candidate in &runner.state.candidates {
13271 assert_eq!(
13272 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13273 git::tree_of(&repo, &seed).await.unwrap()
13274 );
13275 }
13276 }
13277
13278 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13279 let mut heads = Vec::new();
13280 for candidate in candidates {
13281 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13282 }
13283 heads
13284 }
13285
13286 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13287 ask_test_home();
13288 let tmp = tempfile::tempdir().unwrap();
13289 let repo = tmp.path().join("repo");
13290 std::fs::create_dir(&repo).unwrap();
13291 init_repo(&repo);
13292 let mut runner = runner_at(RunStatus::Reviewing);
13293 runner.state.repo = repo.clone();
13294 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13295 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13296 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13297 let branch = format!("candidate-{label}");
13298 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13299 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13300 .await
13301 .unwrap();
13302 if label == 'B' {
13303 std::fs::write(worktree.join("change"), "winner").unwrap();
13304 git::rescue_commit(&worktree, "candidate change")
13305 .await
13306 .unwrap();
13307 }
13308 runner.state.candidates.push(Candidate {
13309 index,
13310 label,
13311 agent: "alpha".into(),
13312 branch,
13313 worktree,
13314 summary: "completed reply".into(),
13315 stat: String::new(),
13316 files: usize::from(label == 'B'),
13317 commits: usize::from(label == 'B'),
13318 empty: label == 'A',
13319 failed: None,
13320 verified_noop: None,
13321 duration_ms: 1,
13322 folded: false,
13323 });
13324 }
13325 runner.state.judge_skipped = true;
13326 runner.state.tally = Some(Tally {
13327 first_choice: BTreeMap::new(),
13328 borda: BTreeMap::new(),
13329 winner: 'B',
13330 rankings: 0,
13331 unanimous_initial: false,
13332 deliberated: false,
13333 changed_votes: 0,
13334 unanimous_final: false,
13335 tie_break: None,
13336 judges: 0,
13337 present: 0,
13338 quorum: 0,
13339 met_quorum: true,
13340 uncontested: Some("only B produced a change".into()),
13341 });
13342 runner.state.save().unwrap();
13343 (tmp, runner)
13344 }
13345
13346 #[tokio::test]
13347 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13348 let (_tmp, mut runner) = preservation_resume_fixture().await;
13349 runner.implement().await.unwrap();
13351 assert!(runner.state.candidate_preservation.is_empty());
13352 assert!(
13353 !runner
13354 .state
13355 .dir()
13356 .join("artifacts/candidate-A-recovery")
13357 .exists()
13358 );
13359 runner.fold_losers().await.unwrap();
13360 assert!(!runner.state.candidates[0].worktree.exists());
13361 for schema in [14, crate::run::SCHEMA] {
13362 runner.state.schema = schema;
13363 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13364 runner.state.save().unwrap();
13365 runner.state = RunState::load(&runner.state.id).unwrap();
13366 runner.implement().await.unwrap();
13367 assert!(runner.state.candidate_preservation.is_empty());
13368 assert!(runner.state.candidates[0].folded);
13369 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13370 }
13371 }
13372
13373 #[tokio::test]
13374 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13375 let (_tmp, mut runner) = preservation_resume_fixture().await;
13376 let wt = runner.state.candidates[0].worktree.clone();
13377 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13378 runner.state.schema = 14;
13379 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13380 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13381 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13382 for _ in 0..2 {
13383 assert!(runner.implement().await.is_err());
13384 assert_eq!(runner.state.status, RunStatus::Blocked);
13385 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13386 assert!(
13387 runner.state.candidates[0].empty,
13388 "old verdict must remain unusable"
13389 );
13390 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13391 assert!(runner.state.judge_skipped);
13392 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13393 assert!(
13394 crate::run::read_artifact(&runner.state, "cand-A.patch")
13395 .unwrap()
13396 .contains("owned edits")
13397 );
13398 assert!(wt.exists());
13399 assert_eq!(
13400 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13401 .await
13402 .unwrap(),
13403 1
13404 );
13405 runner.state = RunState::load(&runner.state.id).unwrap();
13406 }
13407 }
13408
13409 #[tokio::test]
13410 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13411 let (_tmp, mut runner) = preservation_resume_fixture().await;
13412 let wt = runner.state.candidates[0].worktree.clone();
13413 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13414 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13415 let index_lock = PathBuf::from(
13416 git::git(
13417 &wt,
13418 &[
13419 "rev-parse",
13420 "--path-format=absolute",
13421 "--git-path",
13422 "index.lock",
13423 ],
13424 )
13425 .await
13426 .unwrap(),
13427 );
13428 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13429 runner.state.schema = 14;
13430 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13431 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13432 assert!(runner.implement().await.is_err());
13433 assert!(runner.after_implement().is_err());
13434 assert!(runner.fold_losers().await.is_err());
13435 assert_eq!(
13436 std::fs::read_to_string(wt.join("recovered")).unwrap(),
13437 "failed seat edits"
13438 );
13439 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13440 runner.state = RunState::load(&runner.state.id).unwrap();
13441 std::fs::remove_file(index_lock).unwrap();
13442 assert!(runner.implement().await.is_err());
13444 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13445 assert!(runner.state.candidates[0].empty);
13446 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13447 assert!(
13448 crate::run::read_artifact(&runner.state, "cand-A.patch")
13449 .unwrap()
13450 .contains("failed seat edits")
13451 );
13452 assert!(wt.exists());
13453 }
13454
13455 #[tokio::test]
13456 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13457 let (_tmp, mut runner) = preservation_resume_fixture().await;
13458 let repo = runner.state.repo.clone();
13459 let common = git::git(
13460 &repo,
13461 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13462 )
13463 .await
13464 .unwrap();
13465 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13466 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13467 assert!(runner.fold_losers().await.is_err());
13468 assert!(!runner.state.candidates[0].worktree.exists());
13469 assert!(!runner.state.candidates[0].folded);
13470 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13471 runner.state = RunState::load(&runner.state.id).unwrap();
13472 std::fs::remove_file(branch_lock).unwrap();
13473 let judge = runner.state.worktree_root().join("judge-1");
13474 git::worktree_add_detached(&repo, &judge, "main")
13475 .await
13476 .unwrap();
13477 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13478 runner.roles.judges = vec![spec("judge")];
13479 assert!(runner.fold_losers().await.is_err());
13480 runner.state = RunState::load(&runner.state.id).unwrap();
13481 assert!(
13482 runner.state.candidates[0].folded,
13483 "candidate progress saved before judge failure"
13484 );
13485 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13486 assert!(judge.join("uncommitted").exists());
13487 std::fs::remove_file(judge.join("uncommitted")).unwrap();
13488 runner.fold_losers().await.unwrap();
13489 assert!(!judge.exists());
13490 runner.fold_losers().await.unwrap();
13491 }
13492
13493 #[tokio::test]
13494 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13495 let (_tmp, mut runner) = preservation_resume_fixture().await;
13496 runner.state.candidates.retain(|c| c.label == 'B');
13497 runner.state.tally = None;
13498 runner.state.judge_skipped = false;
13499 runner.state.prep_complete = Some(true);
13500 runner.state.config.graph.advise = false;
13501 runner.state.config.graph.selected_implementer = Some("alpha".into());
13502 let wt = runner.state.candidates[0].worktree.clone();
13503 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13504 runner
13505 .state
13506 .candidate_preservation
13507 .insert('B', "awaiting persistence".into());
13508 let index_lock = git::git(
13509 &wt,
13510 &[
13511 "rev-parse",
13512 "--path-format=absolute",
13513 "--git-path",
13514 "index.lock",
13515 ],
13516 )
13517 .await
13518 .unwrap();
13519 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13520 runner.state.save().unwrap();
13521 assert!(runner.execute_graph().await.is_err());
13522 assert_eq!(runner.state.status, RunStatus::Blocked);
13523 assert!(runner.state.candidate_preservation.contains_key(&'B'));
13524 std::fs::remove_file(index_lock).unwrap();
13525 for _ in 0..2 {
13528 runner.state = RunState::load(&runner.state.id).unwrap();
13529 runner.pause.park();
13530 runner.execute_graph().await.unwrap();
13531 assert_eq!(runner.state.status, RunStatus::Implementing);
13532 assert!(runner.state.candidate_preservation.is_empty());
13533 assert_eq!(runner.state.candidates[0].commits, 2);
13534 assert!(runner.state.tally.is_none());
13535 }
13536 runner.implement().await.unwrap();
13537 assert_eq!(runner.state.status, RunStatus::Judging);
13538 runner.state.status = RunStatus::Blocked;
13540 runner.state.candidates[0].failed = Some("implementation failed".into());
13541 runner.state.save().unwrap();
13542 let error = runner.execute_graph().await.unwrap_err();
13543 assert!(
13544 error
13545 .to_string()
13546 .contains("selected implementation is held")
13547 );
13548 }
13549
13550 #[tokio::test]
13551 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13552 let (_tmp, mut runner) = preservation_resume_fixture().await;
13553 let repo = runner.state.repo.clone();
13554 let candidate = runner.state.candidates[0].clone();
13555 let registered = git::worktree_holding(&repo, &candidate.branch)
13558 .await
13559 .unwrap()
13560 .unwrap();
13561 assert_eq!(
13562 registered.canonicalize().unwrap(),
13563 candidate.worktree.canonicalize().unwrap()
13564 );
13565 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13566 assert_eq!(
13567 git::worktree_holding(&repo, &candidate.branch)
13568 .await
13569 .unwrap(),
13570 Some(registered)
13571 );
13572 assert!(
13573 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13574 "the stale registration initially keeps the branch checked out"
13575 );
13576 runner.fold_losers().await.unwrap();
13577 assert!(runner.state.candidates[0].folded);
13578 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13579 assert_eq!(
13580 git::worktree_holding(&repo, &candidate.branch)
13581 .await
13582 .unwrap(),
13583 None
13584 );
13585 runner.state = RunState::load(&runner.state.id).unwrap();
13586 runner.fold_losers().await.unwrap();
13587 }
13588
13589 #[tokio::test]
13590 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13591 let (_tmp, mut runner) = preservation_resume_fixture().await;
13592 let repo = runner.state.repo.clone();
13593 let candidate = runner.state.candidates[0].clone();
13594 let winner = runner.state.candidates[1].clone();
13595 let registered = git::worktree_holding(&repo, &candidate.branch)
13596 .await
13597 .unwrap()
13598 .unwrap();
13599 assert_eq!(
13600 registered.canonicalize().unwrap(),
13601 candidate.worktree.canonicalize().unwrap()
13602 );
13603 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13604 assert_eq!(
13605 git::worktree_holding(&repo, &candidate.branch)
13606 .await
13607 .unwrap(),
13608 Some(registered)
13609 );
13610 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13611
13612 for _ in 0..2 {
13613 fold_run(&mut runner.state, false, &crate::run::home())
13614 .await
13615 .unwrap();
13616 runner.state = RunState::load(&runner.state.id).unwrap();
13617 assert!(runner.state.candidates[0].folded);
13618 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13619 assert_eq!(
13620 git::worktree_holding(&repo, &candidate.branch)
13621 .await
13622 .unwrap(),
13623 None
13624 );
13625 assert!(!runner.state.candidates[1].folded);
13626 assert!(winner.worktree.exists());
13627 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13628 }
13629 }
13630
13631 #[tokio::test]
13632 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13633 ask_test_home();
13634 let tmp = tempfile::tempdir().unwrap();
13635 let repo = tmp.path().join("repo");
13636 std::fs::create_dir(&repo).unwrap();
13637 init_repo(&repo);
13638 let mut runner = runner_at(RunStatus::Implementing);
13639 runner.state.repo = repo.clone();
13640 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13641 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13642 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13643 let branch = format!("candidate-{label}");
13644 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13645 git::worktree_add_branch(&repo, &wt, &branch, "main")
13646 .await
13647 .unwrap();
13648 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13649 git::git(&wt, &["add", "README.md"]).await.unwrap();
13650 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13651 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13652 runner.state.candidates.push(Candidate {
13653 index,
13654 label,
13655 agent: "alpha".to_owned(),
13656 branch,
13657 worktree: wt,
13658 summary: "completed implementation".to_owned(),
13659 stat: String::new(),
13660 files: 0,
13661 commits: 0,
13662 empty: false,
13663 failed: None,
13664 verified_noop: None,
13665 duration_ms: 42,
13666 folded: false,
13667 });
13668 runner
13669 .state
13670 .candidate_preservation
13671 .insert(label, "awaiting persistence".to_owned());
13672 }
13673 let wt = runner.state.candidates[0].worktree.clone();
13674 let index_lock = git::git(
13675 &wt,
13676 &[
13677 "rev-parse",
13678 "--path-format=absolute",
13679 "--git-path",
13680 "index.lock",
13681 ],
13682 )
13683 .await
13684 .unwrap();
13685 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13686 runner.state.save().unwrap();
13687 assert!(runner.persist_candidates().await.is_err());
13688 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13689 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13690 assert_eq!(
13691 runner.state.candidates[1].commits, 1,
13692 "other results are still collected"
13693 );
13694 assert!(
13695 !runner.state.candidates[0].empty,
13696 "failed rescue is not no change"
13697 );
13698 assert_eq!(
13699 runner.state.candidates[0].summary,
13700 "completed implementation"
13701 );
13702 assert!(runner.after_implement().is_err());
13703 let home = tmp.path().join("fold-home");
13704 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13705 assert!(wt.join("new-file").exists());
13706 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13707 assert!(recovery.join("staged.patch").exists());
13708 assert!(recovery.join("unstaged.patch").exists());
13709 assert_eq!(
13710 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13711 "untracked edits\n"
13712 );
13713 runner.state.begin_pass();
13714 runner.state.end_pass_parked(ParkCause::Upgrade {
13715 from: "1.0".into(),
13716 to: Some("1.1".into()),
13717 });
13718 runner.state.save().unwrap();
13719 runner.state = RunState::load(&runner.state.id).unwrap();
13720 assert!(matches!(
13721 runner.state.passes.last().unwrap().end,
13722 Some(crate::run::PassEnd::Parked {
13723 cause: ParkCause::Upgrade { .. }
13724 })
13725 ));
13726 std::fs::remove_file(index_lock).unwrap();
13727 runner.implement().await.unwrap();
13729 assert!(runner.state.candidate_preservation.is_empty());
13730 assert!(
13731 runner
13732 .state
13733 .candidates
13734 .iter()
13735 .all(|c| c.commits == 1 && !c.empty)
13736 );
13737 assert!(git::is_clean(&wt).await.unwrap());
13738 assert!(
13739 crate::run::read_artifact(&runner.state, "cand-A.patch")
13740 .unwrap()
13741 .contains("later tracked edits")
13742 );
13743 }
13744
13745 #[tokio::test]
13746 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13747 ask_test_home();
13748 let tmp = tempfile::tempdir().unwrap();
13749 let repo = tmp.path().join("repo");
13750 std::fs::create_dir(&repo).unwrap();
13751 init_repo(&repo);
13752 let wt = tmp.path().join("candidate");
13753 git::worktree_add_branch(&repo, &wt, "candidate", "main")
13754 .await
13755 .unwrap();
13756 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13757 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13758 assert!(wt.join("uncommitted").exists());
13759 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13760 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13761 let mut runner = runner_at(RunStatus::Implementing);
13762 runner.state.repo = repo.clone();
13763 runner.state.base_commit = "missing-base".to_owned();
13764 runner.state.candidates.push(Candidate {
13765 index: 0,
13766 label: 'A',
13767 agent: "alpha".into(),
13768 branch: "candidate".into(),
13769 worktree: wt.clone(),
13770 summary: "completed".into(),
13771 stat: String::new(),
13772 files: 0,
13773 commits: 0,
13774 empty: false,
13775 failed: None,
13776 verified_noop: None,
13777 duration_ms: 1,
13778 folded: false,
13779 });
13780 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13781 assert!(
13782 fold_run(&mut runner.state, true, &tmp.path().join("home"))
13783 .await
13784 .is_err()
13785 );
13786 assert!(!runner.state.candidates[0].folded);
13787 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13788 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13789 runner
13790 .state
13791 .candidate_preservation
13792 .insert('A', "pending".into());
13793 assert!(runner.persist_candidates().await.is_err());
13794 assert!(!runner.state.candidates[0].empty);
13795 assert!(!runner.state.candidates[0].folded);
13796 assert!(wt.exists());
13797 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13798 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13799 std::fs::create_dir_all(&patch_path).unwrap();
13800 assert!(
13801 runner.persist_candidates().await.is_err(),
13802 "artifact failure stays pending"
13803 );
13804 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13805 assert!(!runner.state.candidates[0].empty);
13806 std::fs::remove_dir(patch_path).unwrap();
13807 runner.persist_candidates().await.unwrap();
13808 assert!(
13809 runner.state.candidates[0].empty,
13810 "only successful inspection confirms no change"
13811 );
13812 }
13813
13814 fn init_repo(dir: &Path) {
13815 let run = |args: &[&str]| {
13816 let out = std::process::Command::new("git")
13817 .args(args)
13818 .current_dir(dir)
13819 .quiet()
13820 .output()
13821 .expect("spawn git");
13822 assert!(
13823 out.status.success(),
13824 "git {args:?} failed: {}",
13825 String::from_utf8_lossy(&out.stderr)
13826 );
13827 };
13828 run(&["init", "-b", "main"]);
13829 run(&["config", "user.name", "magi test"]);
13830 run(&["config", "user.email", "magi@example.com"]);
13831 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13832 run(&["add", "-A"]);
13833 run(&["commit", "-m", "init"]);
13834 }
13835
13836 fn ask_test_home() {
13844 crate::run::pin_test_home();
13845 }
13846
13847 fn runner_at(status: RunStatus) -> Runner {
13850 let mut state = RunState::new(
13851 PathBuf::from("/nonexistent/repo"),
13852 "main".to_owned(),
13853 "deadbeef".to_owned(),
13854 "task".to_owned(),
13855 Config::default(),
13856 );
13857 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13860 state.id = format!(
13861 "20000101-000000-{:04x}",
13862 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13863 );
13864 state.status = status;
13865 Runner {
13866 state,
13867 roles: ResolvedRoles {
13868 implementers: Vec::new(),
13869 judges: Vec::new(),
13870 reviewers: Vec::new(),
13871 fixer: None,
13872 conductor: conductor(),
13873 implementer_roster: Vec::new(),
13874 judge_roster: Vec::new(),
13875 reviewer_roster: Vec::new(),
13876 },
13877 sem: Arc::new(Semaphore::new(1)),
13878 pause: Pause::new(),
13879 interrupt: Pause::new(),
13880 }
13881 }
13882
13883 #[test]
13884 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13885 crate::run::pin_test_home();
13886 let mut runner = runner_at(RunStatus::Gating);
13887 runner.state.save().unwrap();
13888 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13891 assert!(runner.claim_driver().is_err());
13892 assert!(runner.state.driver_exited);
13893 assert_eq!(runner.state.driver_pid, None);
13894 let disk = RunState::load(&runner.state.id).unwrap();
13895 assert!(disk.driver_exited);
13896 assert_eq!(disk.driver_pid, None);
13897 assert_eq!(disk.status, RunStatus::Gating);
13898 }
13899
13900 #[tokio::test]
13901 async fn an_early_graph_error_releases_its_driver() {
13902 crate::run::pin_test_home();
13903 let mut runner = runner_at(RunStatus::Blocked);
13904 runner.state.config.graph.selected_implementer = Some("selected".into());
13905 runner.state.save().unwrap();
13906 assert!(runner.execute().await.is_err());
13907 let disk = RunState::load(&runner.state.id).unwrap();
13908 assert!(disk.driver_exited);
13909 assert_eq!(disk.driver_pid, None);
13910 assert_eq!(disk.status, RunStatus::Blocked);
13911 }
13912
13913 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13914 crate::run::MergeRepair {
13915 question: "repair-question".into(),
13916 head: "old-head".into(),
13917 pr: "https://github.com/o/r/pull/42".into(),
13918 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13919 gate: vec![outcome(Some(0), false)],
13920 gate_head: Some("old-head".into()),
13921 gate_fixes: Vec::new(),
13922 pre_gate: Vec::new(),
13923 phase,
13924 failure: None,
13925 }
13926 }
13927
13928 #[tokio::test]
13929 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13930 crate::run::pin_test_home();
13931 let mut runner = runner_at(RunStatus::Landing);
13932 runner.state.merge_repairs.push(merge_repair_request(
13933 crate::run::MergeRepairPhase::Requested,
13934 ));
13935 runner.state.save().unwrap();
13936 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13937 assert!(
13938 runner
13939 .execute()
13940 .await
13941 .unwrap_err()
13942 .to_string()
13943 .contains("already running")
13944 );
13945 assert_eq!(
13946 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13947 crate::run::MergeRepairPhase::Requested
13948 );
13949 drop(claim);
13950 let mut disk = runner.state.clone();
13951 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13952 disk.save().unwrap();
13953 assert!(
13954 runner
13955 .execute()
13956 .await
13957 .unwrap_err()
13958 .to_string()
13959 .contains("decision changed")
13960 );
13961 assert_eq!(
13962 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13963 crate::run::MergeRepairPhase::Complete
13964 );
13965 }
13966
13967 #[test]
13968 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13969 crate::run::pin_test_home();
13970 let mut runner = runner_at(RunStatus::Landing);
13971 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13972 runner.state.reviews = request.reviews.clone();
13973 runner.state.gate = request.gate.clone();
13974 runner.state.gate_ran = true;
13975 runner.state.land_approval = Some(crate::run::LandApproval {
13976 question: "old".into(),
13977 head: "old-head".into(),
13978 });
13979 runner.state.merge_repairs.push(request);
13980 runner.begin_merge_repair_review(0).unwrap();
13981 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13982 assert!(!runner.state.gate_status().ok());
13983 assert!(runner.state.land_approval.is_none());
13984 let disk = RunState::load(&runner.state.id).unwrap();
13985 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13986 assert!(disk.merge_repairs[0].gate[0].ok());
13987 assert_eq!(
13988 disk.merge_repairs[0].phase,
13989 crate::run::MergeRepairPhase::Reviewing
13990 );
13991 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13992 assert!(disk.reviews.is_empty());
13993 }
13994
13995 #[tokio::test]
13996 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13997 crate::run::pin_test_home();
13998 for phase in [
13999 crate::run::MergeRepairPhase::Fixing,
14000 crate::run::MergeRepairPhase::Requested,
14001 ] {
14002 let mut runner = runner_at(RunStatus::Landing);
14003 runner.state.config.graph.land_rounds = 0;
14004 runner.state.merge_repairs.push(merge_repair_request(phase));
14005 runner.state.save().unwrap();
14006 assert!(!runner.repair_merge_review().await.unwrap());
14007 assert_eq!(runner.state.status, RunStatus::Blocked);
14008 assert_eq!(
14009 runner.state.merge_repairs[0].phase,
14010 crate::run::MergeRepairPhase::Failed
14011 );
14012 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14013 assert!(
14014 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14015 "interrupted"
14016 } else {
14017 "budget exhausted"
14018 })
14019 );
14020 assert!(runner.state.merge.is_none());
14021 }
14022 }
14023
14024 #[test]
14028 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14029 crate::run::pin_test_home();
14030 let mut runner = runner_at(RunStatus::Implementing);
14031 let interrupt = Pause::new();
14032 runner.watch_interrupt(interrupt.clone());
14033
14034 interrupt.park_because("task a1b2 asked to run first");
14035
14036 assert!(runner.park_here().expect("park_here"));
14037 assert!(runner.state.parked);
14038 let last = runner.state.events.last().expect("a park event");
14039 assert_eq!(last.node, "park");
14040 assert!(
14041 last.message.contains("task a1b2 asked to run first"),
14042 "expected the interrupt reason in {:?}",
14043 last.message
14044 );
14045 }
14046
14047 #[test]
14050 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14051 crate::run::pin_test_home();
14052 let mut runner = runner_at(RunStatus::Implementing);
14053 runner.state.begin_pass();
14054 let shutdown = Pause::new();
14055 runner.on_pause(shutdown.clone());
14056 let upgrade = ParkCause::Upgrade {
14057 from: "0.121.0".to_owned(),
14058 to: Some("0.121.1".to_owned()),
14059 };
14060 shutdown.park_for(upgrade.clone());
14061 shutdown.park_for(ParkCause::Operator);
14062
14063 assert!(runner.park_here().expect("park_here"));
14064 assert_eq!(
14065 runner.state.passes[0].end,
14066 Some(crate::run::PassEnd::Parked { cause: upgrade })
14067 );
14068 let last = runner.state.events.last().expect("a park event");
14069 assert!(last.message.contains("0.121.0"), "{}", last.message);
14070 }
14071
14072 #[test]
14080 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14081 crate::run::pin_test_home();
14082 let mut runner = runner_at(RunStatus::Implementing);
14083 let shutdown = Pause::new();
14084 runner.on_pause(shutdown.clone());
14085 let interrupt = Pause::new();
14086 runner.watch_interrupt(interrupt.clone());
14087
14088 assert!(!runner.park_here().expect("park_here"));
14090 assert!(!runner.state.parked);
14091
14092 interrupt.park_because("test");
14094 assert!(!shutdown.parked());
14095 assert!(runner.park_here().expect("park_here"));
14096 }
14097
14098 #[tokio::test]
14112 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14113 crate::run::pin_test_home();
14114 let mut runner = runner_at(RunStatus::Implementing);
14115 let interrupt = Pause::new();
14116 runner.watch_interrupt(interrupt.clone());
14117
14118 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14119 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14120
14121 let node = async move {
14125 started_tx.send(()).expect("send started");
14126 finish_rx.await.expect("recv finish");
14127 "node finished"
14128 };
14129
14130 let interrupter = async move {
14131 started_rx.await.expect("recv started");
14132 interrupt.park_because("higher-priority task waiting");
14134 tokio::task::yield_now().await;
14138 finish_tx.send(()).expect("send finish");
14139 };
14140
14141 let (node_result, ()) = tokio::join!(node, interrupter);
14142 assert_eq!(
14143 node_result, "node finished",
14144 "the in-flight call ran to completion"
14145 );
14146
14147 assert!(runner.park_here().expect("park_here"));
14150 assert!(runner.state.parked);
14151 }
14152
14153 #[test]
14159 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14160 crate::run::pin_test_home();
14161 let mut runner = runner_at(RunStatus::Judging);
14162 runner.state.config.agents = vec![conductor()];
14166 runner.state.candidates = vec![Candidate {
14167 index: 0,
14168 label: 'A',
14169 agent: "alpha".to_owned(),
14170 branch: "magi/x/A".to_owned(),
14171 worktree: PathBuf::from("/nonexistent/worktree"),
14172 summary: "did the thing".to_owned(),
14173 stat: "1 file changed".to_owned(),
14174 files: 1,
14175 commits: 1,
14176 empty: false,
14177 failed: None,
14178 verified_noop: None,
14179 duration_ms: 1234,
14180 folded: false,
14181 }];
14182 let run_id = runner.state.id.clone();
14183
14184 let interrupt = Pause::new();
14185 runner.watch_interrupt(interrupt.clone());
14186 interrupt.park_because("task c3d4 asked to run first");
14187 assert!(runner.park_here().expect("park_here"));
14188
14189 let resumed = Runner::resume(&run_id).expect("resume");
14190 assert_eq!(resumed.state.candidates.len(), 1);
14191 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14192 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14193 assert_eq!(resumed.state.status, runner.state.status);
14194 assert!(
14195 resumed.state.parked,
14196 "still parked until `execute` actually walks the graph again"
14197 );
14198 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14199 }
14200
14201 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14203 let mut q = ask::Question::new(
14204 run.to_owned(),
14205 "implement".to_owned(),
14206 "impl-A".to_owned(),
14207 "Which storage backend should the cache use?".to_owned(),
14208 String::new(),
14209 vec!["SQLite".to_owned(), "Redis".to_owned()],
14210 );
14211 store.put(&mut q).unwrap();
14212 q
14213 }
14214
14215 #[test]
14216 fn a_failed_runs_open_question_is_abandoned() {
14217 ask_test_home();
14218 let store = ask::Questions::open();
14219 let mut runner = runner_at(RunStatus::Failed);
14220 let run = runner.state.id.clone();
14221 let q = ask_open_question(&store, &run);
14222
14223 runner.settle_questions();
14224
14225 let back = store.get(&q.id).unwrap();
14226 assert!(
14227 !back.status.open(),
14228 "the seat that asked died with the run; nobody is left to read an answer"
14229 );
14230 assert!(
14231 back.detail.contains(&run) && back.detail.contains("failed"),
14232 "the reason names what the run became, not just that it is gone: {}",
14233 back.detail
14234 );
14235 }
14236
14237 #[test]
14238 fn a_merged_runs_open_question_is_abandoned_too() {
14239 ask_test_home();
14240 let store = ask::Questions::open();
14241 for status in [RunStatus::Merged, RunStatus::Ready] {
14244 let mut runner = runner_at(status);
14245 let run = runner.state.id.clone();
14246 let q = ask_open_question(&store, &run);
14247
14248 runner.settle_questions();
14249
14250 let back = store.get(&q.id).unwrap();
14251 assert!(
14252 !back.status.open(),
14253 "{status:?} run's question must not outlive the run"
14254 );
14255 }
14256 }
14257
14258 #[test]
14259 fn a_still_resumable_runs_open_question_is_left_alone() {
14260 ask_test_home();
14261 let store = ask::Questions::open();
14262 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14268 let mut runner = runner_at(status);
14269 let run = runner.state.id.clone();
14270 let q = ask_open_question(&store, &run);
14271
14272 runner.settle_questions();
14273
14274 let back = store.get(&q.id).unwrap();
14275 assert!(
14276 back.status.open(),
14277 "{status:?} is still alive; the question must still be waiting"
14278 );
14279 }
14280 }
14281
14282 #[test]
14283 fn settle_questions_never_touches_an_already_answered_question() {
14284 ask_test_home();
14285 let store = ask::Questions::open();
14286 let mut runner = runner_at(RunStatus::Failed);
14287 let run = runner.state.id.clone();
14288 let mut q = ask_open_question(&store, &run);
14289 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14290 .unwrap();
14291 store.put(&mut q).unwrap();
14292
14293 runner.settle_questions();
14298 runner.settle_questions();
14299
14300 let back = store.get(&q.id).unwrap();
14301 assert_eq!(
14302 back.status,
14303 ask::QuestionStatus::Answered,
14304 "a real answer is a decision on record, never overwritten by a sweep"
14305 );
14306 }
14307
14308 #[tokio::test]
14319 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14320 crate::run::pin_test_home();
14321 let tmp = tempfile::tempdir().expect("tempdir");
14322 let repo = tmp.path().join("repo");
14323 std::fs::create_dir_all(&repo).unwrap();
14324 init_repo(&repo);
14325
14326 let mut config = Config::default();
14327 config.graph.worktree_root = Some(tmp.path().join("wt"));
14328
14329 let mut state = RunState::new(
14330 repo.clone(),
14331 "main".to_owned(),
14332 "deadbeef".to_owned(),
14333 "task".to_owned(),
14334 config,
14335 );
14336 let root = state.worktree_root();
14337 let wt_a = root.join("cand-A");
14338 let wt_b = root.join("cand-B");
14339 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14340 .await
14341 .expect("worktree A");
14342 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14343 .await
14344 .expect("worktree B");
14345
14346 state.candidates = vec![
14347 Candidate {
14348 index: 0,
14349 label: 'A',
14350 agent: "alpha".to_owned(),
14351 branch: "magi/x/A".to_owned(),
14352 worktree: wt_a.clone(),
14353 summary: String::new(),
14354 stat: String::new(),
14355 files: 0,
14356 commits: 0,
14357 empty: false,
14358 failed: None,
14359 verified_noop: None,
14360 duration_ms: 0,
14361 folded: false,
14362 },
14363 Candidate {
14364 index: 1,
14365 label: 'B',
14366 agent: "beta".to_owned(),
14367 branch: "magi/x/B".to_owned(),
14368 worktree: wt_b.clone(),
14369 summary: String::new(),
14370 stat: String::new(),
14371 files: 0,
14372 commits: 0,
14373 empty: false,
14374 failed: None,
14375 verified_noop: None,
14376 duration_ms: 0,
14377 folded: false,
14378 },
14379 ];
14380 state.tally = Some(Tally {
14381 first_choice: BTreeMap::from([('A', 1)]),
14382 borda: BTreeMap::new(),
14383 winner: 'A',
14384 rankings: 1,
14385 unanimous_initial: true,
14386 deliberated: false,
14387 changed_votes: 0,
14388 unanimous_final: true,
14389 tie_break: None,
14390 judges: 1,
14391 present: 1,
14392 quorum: 1,
14393 met_quorum: true,
14394 uncontested: None,
14395 });
14396 state.status = RunStatus::Ready;
14397
14398 fold_run(&mut state, false, &crate::run::home())
14399 .await
14400 .expect("fold_run");
14401
14402 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14403 assert!(
14404 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14405 "the unmerged winner's branch survives"
14406 );
14407 assert!(
14408 !state.candidates[0].folded,
14409 "the winner is not marked folded"
14410 );
14411
14412 assert!(!wt_b.exists(), "the loser's worktree is removed");
14413 assert!(
14414 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14415 "the loser's branch is removed"
14416 );
14417 assert!(state.candidates[1].folded, "the loser is marked folded");
14418 }
14419
14420 #[tokio::test]
14421 #[cfg(unix)]
14422 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14423 use std::os::fd::AsRawFd as _;
14424 use std::time::Duration;
14425
14426 let tmp = tempfile::tempdir().unwrap();
14427 let repo = tmp.path().join("repo");
14428 std::fs::create_dir_all(&repo).unwrap();
14429 init_repo(&repo);
14430 let common = repo.join(".git");
14431 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14432 let home = tmp.path().join("home");
14433 let mut config = Config::default();
14434 config.coordination = crate::config::test_coordination();
14435 config.graph.worktree_root = Some(tmp.path().join("wt"));
14436 config.graph.timeout_resource_wait = 1;
14437 let mut state = RunState::new(
14438 repo.clone(),
14439 "main".into(),
14440 "deadbeef".into(),
14441 "task".into(),
14442 config,
14443 );
14444 let root = state.worktree_root();
14445 let candidate = root.join("cand-A");
14446 let reviewer = root.join("review-1");
14447 let branch = state.branch_for('A');
14448 git::worktree_add_branch(&repo, &candidate, &branch, "main")
14449 .await
14450 .unwrap();
14451 git::worktree_add_detached(&repo, &reviewer, "main")
14452 .await
14453 .unwrap();
14454 git::acquire_worktree_config(&repo).await.unwrap();
14455 state.enabled_worktree_config = true;
14456 state.status = RunStatus::Ready;
14457 state.candidates.push(Candidate {
14458 index: 0,
14459 label: 'A',
14460 agent: "implement".into(),
14461 branch: branch.clone(),
14462 worktree: candidate.clone(),
14463 summary: String::new(),
14464 stat: String::new(),
14465 files: 0,
14466 commits: 0,
14467 empty: false,
14468 failed: None,
14469 verified_noop: None,
14470 duration_ms: 0,
14471 folded: false,
14472 });
14473 state.save_under(&home).unwrap();
14474 let config_before = std::fs::read(common.join("config")).unwrap();
14475 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14476 .await
14477 .unwrap();
14478 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14481 assert_eq!(
14482 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14483 0
14484 );
14485 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14486 assert!(
14487 tokio::time::timeout(Duration::from_millis(200), &mut fold)
14488 .await
14489 .is_err()
14490 );
14491 assert!(candidate.exists());
14492 assert!(reviewer.exists());
14493 assert!(git::branch_exists(&repo, &branch).await.unwrap());
14494 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14495 assert_eq!(
14496 git::git(&repo, &["worktree", "list", "--porcelain"])
14497 .await
14498 .unwrap(),
14499 worktrees_before
14500 );
14501 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14502 .await
14503 .unwrap()
14504 .unwrap_err();
14505 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14506 drop(fold);
14507 assert!(candidate.exists() && reviewer.exists());
14508 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14509 assert_eq!(
14510 state.resource_block.as_ref().unwrap().kind,
14511 "repository coordination"
14512 );
14513 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14514 drop(external);
14515 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14516 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14517 .await
14518 .unwrap()
14519 .unwrap();
14520 drop(fold);
14521 assert!(removed.contains(&branch));
14522 assert!(!root.exists());
14523 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14524 assert!(state.candidates[0].folded);
14525 assert!(!state.enabled_worktree_config);
14526 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14527 let enabled = git::git_raw(
14528 &repo,
14529 &["config", "--local", "--get", "extensions.worktreeConfig"],
14530 )
14531 .await
14532 .unwrap();
14533 assert!(!enabled.ok(), "the last config reference was released");
14534 }
14535
14536 #[tokio::test]
14539 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14540 let tmp = tempfile::tempdir().expect("tempdir");
14541 let repo = tmp.path().join("repo");
14542 std::fs::create_dir_all(&repo).unwrap();
14543 init_repo(&repo);
14544 let home = tmp.path().join("home");
14545
14546 let mut config = Config::default();
14547 config.graph.worktree_root = Some(tmp.path().join("wt"));
14548 let mut state = RunState::new(
14549 repo.clone(),
14550 "main".to_owned(),
14551 "deadbeef".to_owned(),
14552 "task".to_owned(),
14553 config,
14554 );
14555 git::git(&repo, &["branch", "magi/x/A", "main"])
14557 .await
14558 .expect("branch");
14559 state.candidates = vec![Candidate {
14560 index: 0,
14561 label: 'A',
14562 agent: "alpha".to_owned(),
14563 branch: "magi/x/A".to_owned(),
14564 worktree: state.worktree_root().join("cand-A"),
14565 summary: String::new(),
14566 stat: String::new(),
14567 files: 0,
14568 commits: 0,
14569 empty: false,
14570 failed: None,
14571 verified_noop: None,
14572 duration_ms: 0,
14573 folded: true,
14574 }];
14575 state.released_to = Some("20260901-000000-new1".to_owned());
14576 state.released_branches = vec!["magi/x/A".to_owned()];
14577
14578 fold_run(&mut state, true, &home).await.expect("fold_run");
14579
14580 assert!(
14581 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14582 "the handed-over branch survives a fold"
14583 );
14584 }
14585
14586 #[tokio::test]
14590 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14591 let tmp = tempfile::tempdir().expect("tempdir");
14592 let repo = tmp.path().join("repo");
14593 std::fs::create_dir_all(&repo).unwrap();
14594 init_repo(&repo);
14595 let run = |args: &[&str]| {
14596 let out = std::process::Command::new("git")
14597 .quiet()
14598 .args(args)
14599 .current_dir(&repo)
14600 .output()
14601 .expect("spawn git");
14602 assert!(out.status.success(), "git {args:?}");
14603 };
14604 run(&["branch", "magi/x/A"]);
14605 run(&["checkout", "-q", "-b", "magi/x/B"]);
14606 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14607 run(&["add", "-A"]);
14608 run(&["commit", "-q", "-m", "work"]);
14609 run(&["checkout", "-q", "main"]);
14610
14611 let probe = RunState::new(
14615 repo.clone(),
14616 "main".to_owned(),
14617 "deadbeef".to_owned(),
14618 "task".to_owned(),
14619 Config::default(),
14620 );
14621 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14622 let bare = tmp.path().join("origin.git");
14623 let out = std::process::Command::new("git")
14624 .quiet()
14625 .args(["init", "-q", "--bare"])
14626 .arg(&bare)
14627 .output()
14628 .expect("spawn git");
14629 assert!(out.status.success(), "git init --bare");
14630 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14631 run(&["push", "-q", "origin", "main"]);
14632
14633 let mut state = RunState::new(
14634 repo.clone(),
14635 "main".to_owned(),
14636 "deadbeef".to_owned(),
14637 "task".to_owned(),
14638 Config::default(),
14639 );
14640 state.seeds = vec![refs::Seed {
14641 token: "magi/27b2/A".to_owned(),
14642 kind: refs::SeedKind::Unresolved,
14643 sha: String::new(),
14644 branch: true,
14645 detail: "no branch or commit named magi/27b2/A".to_owned(),
14646 }];
14647
14648 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14649 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14650 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14651 let detail = empty_candidate_detail(&state, "main");
14652 assert!(detail.starts_with("empty candidate"), "{detail}");
14653 assert!(detail.contains("magi/27b2/A"), "{detail}");
14654 }
14655
14656 #[tokio::test]
14665 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14666 let tmp = tempfile::tempdir().expect("tempdir");
14667 let repo = tmp.path().join("repo");
14668 std::fs::create_dir_all(&repo).unwrap();
14669 init_repo(&repo);
14670
14671 let mut config = Config::default();
14672 config.merge.mode = MergeMode::Local;
14673
14674 let mut state = RunState::new(
14675 repo.clone(),
14676 "main".to_owned(),
14677 "deadbeef".to_owned(),
14678 "task".to_owned(),
14679 config,
14680 );
14681 state.candidates = vec![Candidate {
14682 index: 0,
14683 label: 'A',
14684 agent: "alpha".to_owned(),
14685 branch: "does-not-exist".to_owned(),
14686 worktree: repo.clone(),
14687 summary: String::new(),
14688 stat: String::new(),
14689 files: 0,
14690 commits: 0,
14691 empty: false,
14692 failed: None,
14693 verified_noop: None,
14694 duration_ms: 0,
14695 folded: false,
14696 }];
14697 state.tally = Some(Tally {
14698 first_choice: BTreeMap::from([('A', 1)]),
14699 borda: BTreeMap::new(),
14700 winner: 'A',
14701 rankings: 1,
14702 unanimous_initial: true,
14703 deliberated: false,
14704 changed_votes: 0,
14705 unanimous_final: true,
14706 tie_break: None,
14707 judges: 0,
14708 present: 0,
14709 quorum: 0,
14710 met_quorum: true,
14711 uncontested: Some("only candidate A produced a change".to_owned()),
14712 });
14713 state.reviews = vec![ReviewRound {
14714 round: 1,
14715 head: "deadbeef".to_owned(),
14716 verified_head: None,
14717 verified_at: None,
14718 reviews: Vec::new(),
14719 e2e: Vec::new(),
14720 fix: None,
14721 blocking: 0,
14722 answered: 0,
14723 expected: 0,
14724 clean: true,
14725 verify_retried: false,
14726 e2e_deferred: false,
14727 e2e_defer_reason: None,
14728 progressed: false,
14729 vote_split: false,
14730 reconsideration: Vec::new(),
14731 verdict: None,
14732 }];
14733 state.gate = vec![CommandOutcome {
14734 command: "test".to_owned(),
14735 code: Some(0),
14736 output_tail: String::new(),
14737 duration_ms: 0,
14738 resource_blocked: false,
14739 }];
14740 state.gate_ran = true;
14741 state.status = RunStatus::Ready;
14746 state.merge = Some(MergeOutcome {
14747 mode: MergeMode::Local,
14748 ok: false,
14749 detail: "already concluded".to_owned(),
14750 empty: false,
14751 });
14752
14753 let mut runner = Runner {
14754 state,
14755 roles: ResolvedRoles {
14756 implementers: Vec::new(),
14757 judges: Vec::new(),
14758 reviewers: Vec::new(),
14759 fixer: None,
14760 conductor: conductor(),
14761 implementer_roster: Vec::new(),
14762 judge_roster: Vec::new(),
14763 reviewer_roster: Vec::new(),
14764 },
14765 sem: Arc::new(Semaphore::new(1)),
14766 pause: Pause::new(),
14767 interrupt: Pause::new(),
14768 };
14769
14770 runner.merge().await.expect("merge");
14771
14772 assert_eq!(
14773 runner.state.status,
14774 RunStatus::Ready,
14775 "a concluded run's status must not change on reentry"
14776 );
14777 assert_eq!(
14778 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14779 Some("already concluded"),
14780 "merge must not run again once the node already recorded an outcome"
14781 );
14782 }
14783
14784 #[tokio::test]
14793 async fn merge_refuses_a_gate_that_has_not_actually_run() {
14794 let tmp = tempfile::tempdir().expect("tempdir");
14795 let repo = tmp.path().join("repo");
14796 std::fs::create_dir_all(&repo).unwrap();
14797 init_repo(&repo);
14798
14799 let mut config = Config::default();
14800 config.merge.mode = MergeMode::Local;
14801
14802 let mut state = RunState::new(
14803 repo.clone(),
14804 "main".to_owned(),
14805 "deadbeef".to_owned(),
14806 "task".to_owned(),
14807 config,
14808 );
14809 state.candidates = vec![Candidate {
14810 index: 0,
14811 label: 'A',
14812 agent: "alpha".to_owned(),
14813 branch: "does-not-exist".to_owned(),
14814 worktree: repo.clone(),
14815 summary: String::new(),
14816 stat: String::new(),
14817 files: 0,
14818 commits: 0,
14819 empty: false,
14820 failed: None,
14821 verified_noop: None,
14822 duration_ms: 0,
14823 folded: false,
14824 }];
14825 state.tally = Some(Tally {
14826 first_choice: BTreeMap::from([('A', 1)]),
14827 borda: BTreeMap::new(),
14828 winner: 'A',
14829 rankings: 1,
14830 unanimous_initial: true,
14831 deliberated: false,
14832 changed_votes: 0,
14833 unanimous_final: true,
14834 tie_break: None,
14835 judges: 0,
14836 present: 0,
14837 quorum: 0,
14838 met_quorum: true,
14839 uncontested: Some("only candidate A produced a change".to_owned()),
14840 });
14841 state.reviews = vec![ReviewRound {
14842 round: 1,
14843 head: "deadbeef".to_owned(),
14844 verified_head: None,
14845 verified_at: None,
14846 reviews: Vec::new(),
14847 e2e: Vec::new(),
14848 fix: None,
14849 blocking: 0,
14850 answered: 0,
14851 expected: 0,
14852 clean: true,
14853 verify_retried: false,
14854 e2e_deferred: false,
14855 e2e_defer_reason: None,
14856 progressed: false,
14857 vote_split: false,
14858 reconsideration: Vec::new(),
14859 verdict: None,
14860 }];
14861 state.gate = Vec::new();
14863 state.gate_ran = false;
14864 state.status = RunStatus::Gating;
14865
14866 let mut runner = Runner {
14867 state,
14868 roles: ResolvedRoles {
14869 implementers: Vec::new(),
14870 judges: Vec::new(),
14871 reviewers: Vec::new(),
14872 fixer: None,
14873 conductor: conductor(),
14874 implementer_roster: Vec::new(),
14875 judge_roster: Vec::new(),
14876 reviewer_roster: Vec::new(),
14877 },
14878 sem: Arc::new(Semaphore::new(1)),
14879 pause: Pause::new(),
14880 interrupt: Pause::new(),
14881 };
14882
14883 runner.merge().await.expect("merge");
14884
14885 assert!(
14886 runner.state.merge.is_none(),
14887 "an empty gate must never be read as a passing one: {:?}",
14888 runner.state.merge
14889 );
14890 }
14891
14892 #[tokio::test]
14899 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14900 let tmp = tempfile::tempdir().expect("tempdir");
14901 let repo = tmp.path().join("repo");
14902 std::fs::create_dir_all(&repo).unwrap();
14903 init_repo(&repo);
14904
14905 let config = Config::default();
14907
14908 let mut state = RunState::new(
14909 repo.clone(),
14910 "main".to_owned(),
14911 "deadbeef".to_owned(),
14912 "task".to_owned(),
14913 config,
14914 );
14915 state.candidates = vec![Candidate {
14916 index: 0,
14917 label: 'A',
14918 agent: "alpha".to_owned(),
14919 branch: "does-not-exist".to_owned(),
14920 worktree: repo.clone(),
14921 summary: String::new(),
14922 stat: String::new(),
14923 files: 0,
14924 commits: 0,
14925 empty: false,
14926 failed: None,
14927 verified_noop: None,
14928 duration_ms: 0,
14929 folded: false,
14930 }];
14931 state.tally = Some(Tally {
14932 first_choice: BTreeMap::from([('A', 1)]),
14933 borda: BTreeMap::new(),
14934 winner: 'A',
14935 rankings: 1,
14936 unanimous_initial: true,
14937 deliberated: false,
14938 changed_votes: 0,
14939 unanimous_final: true,
14940 tie_break: None,
14941 judges: 0,
14942 present: 0,
14943 quorum: 0,
14944 met_quorum: true,
14945 uncontested: Some("only candidate A produced a change".to_owned()),
14946 });
14947 state.reviews = vec![ReviewRound {
14948 round: 1,
14949 head: "deadbeef".to_owned(),
14950 verified_head: None,
14951 verified_at: None,
14952 reviews: Vec::new(),
14953 e2e: Vec::new(),
14954 fix: None,
14955 blocking: 0,
14956 answered: 0,
14957 expected: 0,
14958 clean: true,
14959 verify_retried: false,
14960 e2e_deferred: false,
14961 e2e_defer_reason: None,
14962 progressed: false,
14963 vote_split: false,
14964 reconsideration: Vec::new(),
14965 verdict: None,
14966 }];
14967
14968 let mut runner = Runner {
14969 state,
14970 roles: ResolvedRoles {
14971 implementers: Vec::new(),
14972 judges: Vec::new(),
14973 reviewers: Vec::new(),
14974 fixer: None,
14975 conductor: conductor(),
14976 implementer_roster: Vec::new(),
14977 judge_roster: Vec::new(),
14978 reviewer_roster: Vec::new(),
14979 },
14980 sem: Arc::new(Semaphore::new(1)),
14981 pause: Pause::new(),
14982 interrupt: Pause::new(),
14983 };
14984
14985 runner.gate().await.expect("gate");
14986 assert!(
14987 runner.state.gate_ran,
14988 "zero configured commands is still a real attempt, not an unrun gate"
14989 );
14990 assert!(runner.state.gate.is_empty());
14991 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
14992 assert_ne!(
14993 runner.state.status,
14994 RunStatus::Blocked,
14995 "a gate with nothing to check must not read as failed"
14996 );
14997
14998 runner.merge().await.expect("merge");
14999 assert_eq!(
15000 runner.state.status,
15001 RunStatus::Ready,
15002 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15003 );
15004 }
15005
15006 #[tokio::test]
15017 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15018 crate::run::pin_test_home();
15019 let home = crate::run::home();
15020
15021 let tmp = tempfile::tempdir().expect("tempdir");
15022 let repo = tmp.path().join("repo");
15023 std::fs::create_dir_all(&repo).unwrap();
15024 init_repo(&repo);
15025 let cache_dir = tmp.path().join("target");
15028
15029 let mut config = Config::default();
15030 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15031 config.graph.timeout_verify = Some(2);
15034 config.graph.timeout_resource_wait = 2;
15035
15036 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15037 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15038 .expect("no io error acquiring directly")
15039 {
15040 crate::cache::AcquireOutcome::Acquired(g) => g,
15041 crate::cache::AcquireOutcome::Busy(b) => {
15042 panic!("expected the direct acquire to win the lease first: {b:?}")
15043 }
15044 };
15045
15046 let mut state = RunState::new(
15047 repo.clone(),
15048 "main".to_owned(),
15049 "deadbeef".to_owned(),
15050 "task".to_owned(),
15051 config,
15052 );
15053 state.candidates = vec![Candidate {
15054 index: 0,
15055 label: 'A',
15056 agent: "alpha".to_owned(),
15057 branch: "does-not-exist".to_owned(),
15058 worktree: repo.clone(),
15059 summary: String::new(),
15060 stat: String::new(),
15061 files: 0,
15062 commits: 0,
15063 empty: false,
15064 failed: None,
15065 verified_noop: None,
15066 duration_ms: 0,
15067 folded: false,
15068 }];
15069 state.tally = Some(Tally {
15070 first_choice: BTreeMap::from([('A', 1)]),
15071 borda: BTreeMap::new(),
15072 winner: 'A',
15073 rankings: 1,
15074 unanimous_initial: true,
15075 deliberated: false,
15076 changed_votes: 0,
15077 unanimous_final: true,
15078 tie_break: None,
15079 judges: 0,
15080 present: 0,
15081 quorum: 0,
15082 met_quorum: true,
15083 uncontested: Some("only candidate A produced a change".to_owned()),
15084 });
15085 state.reviews = vec![ReviewRound {
15086 round: 1,
15087 head: "deadbeef".to_owned(),
15088 verified_head: None,
15089 verified_at: None,
15090 reviews: Vec::new(),
15091 e2e: Vec::new(),
15092 fix: None,
15093 blocking: 0,
15094 answered: 0,
15095 expected: 0,
15096 clean: true,
15097 verify_retried: false,
15098 e2e_deferred: false,
15099 e2e_defer_reason: None,
15100 progressed: false,
15101 vote_split: false,
15102 reconsideration: Vec::new(),
15103 verdict: None,
15104 }];
15105
15106 let mut runner = Runner {
15107 state,
15108 roles: ResolvedRoles {
15109 implementers: Vec::new(),
15110 judges: Vec::new(),
15111 reviewers: Vec::new(),
15112 fixer: None,
15113 conductor: conductor(),
15114 implementer_roster: Vec::new(),
15115 judge_roster: Vec::new(),
15116 reviewer_roster: Vec::new(),
15117 },
15118 sem: Arc::new(Semaphore::new(1)),
15119 pause: Pause::new(),
15120 interrupt: Pause::new(),
15121 };
15122
15123 let started = std::time::Instant::now();
15124 runner.gate().await.expect("gate");
15125 assert!(
15126 started.elapsed() < Duration::from_secs(1),
15127 "a gate with nothing to run must never wait on a lease it never needed"
15128 );
15129 assert!(
15130 runner.state.gate_ran,
15131 "zero commands is still a real, immediate attempt"
15132 );
15133 assert!(runner.state.gate.is_empty());
15134 assert_ne!(
15135 runner.state.status,
15136 RunStatus::Blocked,
15137 "must not read as resource-blocked on a lease it never asked for"
15138 );
15139 }
15140
15141 #[tokio::test]
15151 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15152 crate::run::pin_test_home();
15153
15154 let tmp = tempfile::tempdir().expect("tempdir");
15155 let repo = tmp.path().join("repo");
15156 std::fs::create_dir_all(&repo).unwrap();
15157 init_repo(&repo);
15158
15159 let mut config = Config::default();
15160 config.verify.gate = vec![
15161 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15162 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15163 .to_owned(),
15164 ];
15165
15166 let mut state = RunState::new(
15167 repo.clone(),
15168 "main".to_owned(),
15169 "deadbeef".to_owned(),
15170 "task".to_owned(),
15171 config,
15172 );
15173 let run_id = state.id.clone();
15174 state.candidates = vec![Candidate {
15175 index: 0,
15176 label: 'A',
15177 agent: "alpha".to_owned(),
15178 branch: "does-not-exist".to_owned(),
15179 worktree: repo.clone(),
15180 summary: String::new(),
15181 stat: String::new(),
15182 files: 0,
15183 commits: 0,
15184 empty: false,
15185 failed: None,
15186 verified_noop: None,
15187 duration_ms: 0,
15188 folded: false,
15189 }];
15190 state.tally = Some(Tally {
15191 first_choice: BTreeMap::from([('A', 1)]),
15192 borda: BTreeMap::new(),
15193 winner: 'A',
15194 rankings: 1,
15195 unanimous_initial: true,
15196 deliberated: false,
15197 changed_votes: 0,
15198 unanimous_final: true,
15199 tie_break: None,
15200 judges: 0,
15201 present: 0,
15202 quorum: 0,
15203 met_quorum: true,
15204 uncontested: Some("only candidate A produced a change".to_owned()),
15205 });
15206 state.reviews = vec![ReviewRound {
15207 round: 1,
15208 head: "deadbeef".to_owned(),
15209 verified_head: None,
15210 verified_at: None,
15211 reviews: Vec::new(),
15212 e2e: Vec::new(),
15213 fix: None,
15214 blocking: 0,
15215 answered: 0,
15216 expected: 0,
15217 clean: true,
15218 verify_retried: false,
15219 e2e_deferred: false,
15220 e2e_defer_reason: None,
15221 progressed: false,
15222 vote_split: false,
15223 reconsideration: Vec::new(),
15224 verdict: None,
15225 }];
15226
15227 let mut runner = Runner {
15228 state,
15229 roles: ResolvedRoles {
15230 implementers: Vec::new(),
15231 judges: Vec::new(),
15232 reviewers: Vec::new(),
15233 fixer: None,
15234 conductor: conductor(),
15235 implementer_roster: Vec::new(),
15236 judge_roster: Vec::new(),
15237 reviewer_roster: Vec::new(),
15238 },
15239 sem: Arc::new(Semaphore::new(1)),
15240 pause: Pause::new(),
15241 interrupt: Pause::new(),
15242 };
15243
15244 let started_marker = repo.join("started.marker");
15245 let release_marker = repo.join("release.marker");
15246 let poller = tokio::spawn(async move {
15247 for _ in 0..100 {
15252 if started_marker.exists()
15253 && let Ok(s) = crate::run::RunState::load(&run_id)
15254 && let Some(a) = s.active.get("gate")
15255 {
15256 std::fs::write(&release_marker, b"go").expect("release marker");
15257 return Some(a.clone());
15258 }
15259 tokio::time::sleep(Duration::from_millis(50)).await;
15260 }
15261 None
15262 });
15263
15264 runner.gate().await.expect("gate");
15265 let captured = poller.await.expect("poller task");
15266 let captured = captured.expect(
15267 "the poller never saw a `gate` task entry in run.json while the command was \
15268 still blocked on its own release marker",
15269 );
15270
15271 assert_eq!(captured.task.as_deref(), Some("gate"));
15272 assert_eq!(captured.node, "gate");
15273 assert_eq!(captured.index, Some(1));
15274 assert_eq!(captured.total, Some(1));
15275 assert!(
15276 captured
15277 .command
15278 .as_deref()
15279 .is_some_and(|c| c.contains("started.marker")),
15280 "{captured:?}"
15281 );
15282
15283 assert!(
15284 runner.state.active.is_empty(),
15285 "the entry must be cleared once the command actually finished: {:?}",
15286 runner.state.active
15287 );
15288 assert!(runner.state.gate_ran);
15289 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15290 }
15291
15292 #[tokio::test]
15295 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15296 use crate::verdict::{Finding, ReviewVote, Severity};
15297 crate::run::pin_test_home();
15298 let tmp = tempfile::tempdir().expect("tempdir");
15299 let repo = tmp.path().join("repo");
15300 std::fs::create_dir_all(&repo).unwrap();
15301 init_repo(&repo);
15302
15303 for (severity, vote, expect) in [
15304 (Severity::Major, ReviewVote::Reject, true),
15305 (Severity::Minor, ReviewVote::Reject, false),
15306 (Severity::Major, ReviewVote::Approve, false),
15307 ] {
15308 let mut round = review_round(false, 1, 1, 1, false, true);
15309 round.reviews = vec![ReviewRecord {
15310 reviewer: 1,
15311 agent: "alpha".to_owned(),
15312 summary: String::new(),
15313 findings: vec![Finding {
15314 id: "R1-1-1".to_owned(),
15315 severity,
15316 file: None,
15317 line: None,
15318 title: "t".to_owned(),
15319 detail: String::new(),
15320 }],
15321 vote: Some(vote),
15322 failed: None,
15323 duration_ms: 0,
15324 attempts: 0,
15325 }];
15326 let mut state = RunState::new(
15327 repo.clone(),
15328 "main".to_owned(),
15329 "deadbeef".to_owned(),
15330 "task".to_owned(),
15331 Config::default(),
15332 );
15333 state.reviews = vec![round];
15334 let mut runner = Runner {
15335 state,
15336 roles: ResolvedRoles {
15337 implementers: Vec::new(),
15338 judges: Vec::new(),
15339 reviewers: Vec::new(),
15340 fixer: None,
15341 conductor: conductor(),
15342 implementer_roster: Vec::new(),
15343 judge_roster: Vec::new(),
15344 reviewer_roster: Vec::new(),
15345 },
15346 sem: Arc::new(Semaphore::new(1)),
15347 pause: Pause::new(),
15348 interrupt: Pause::new(),
15349 };
15350 let shell = runner.state.config.shell();
15351 runner
15352 .stop_reviewing("round budget spent", &shell, &repo)
15353 .await
15354 .expect("stop_reviewing");
15355 assert_eq!(runner.state.status, RunStatus::Gating);
15356 assert_eq!(
15357 runner.state.contested_handoff.is_some(),
15358 expect,
15359 "{severity:?} + {vote:?}"
15360 );
15361 }
15362 }
15363
15364 #[tokio::test]
15377 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15378 crate::run::pin_test_home();
15379 let home = crate::run::home();
15380
15381 let tmp = tempfile::tempdir().expect("tempdir");
15382 let repo = tmp.path().join("repo");
15383 std::fs::create_dir_all(&repo).unwrap();
15384 init_repo(&repo);
15385 let head = crate::git::rev_parse(&repo, "HEAD")
15386 .await
15387 .expect("rev-parse");
15388 let cache_dir = tmp.path().join("target");
15391
15392 let mut config = Config::default();
15393 config.verify.e2e = vec![format!(
15394 "CARGO_TARGET_DIR='{}' test -f README.md",
15395 cache_dir.display()
15396 )];
15397 config.graph.review_rounds = 1;
15398 config.graph.timeout_verify = Some(2);
15401 config.graph.timeout_resource_wait = 2;
15402
15403 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15404 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15405 .expect("no io error acquiring directly")
15406 {
15407 crate::cache::AcquireOutcome::Acquired(g) => g,
15408 crate::cache::AcquireOutcome::Busy(b) => {
15409 panic!("expected the direct acquire to win the lease first: {b:?}")
15410 }
15411 };
15412
15413 let mut state = RunState::new(
15414 repo.clone(),
15415 "main".to_owned(),
15416 head.clone(),
15417 "task".to_owned(),
15418 config,
15419 );
15420 state.candidates = vec![Candidate {
15421 index: 0,
15422 label: 'A',
15423 agent: "alpha".to_owned(),
15424 branch: "does-not-exist".to_owned(),
15425 worktree: repo.clone(),
15426 summary: String::new(),
15427 stat: String::new(),
15428 files: 0,
15429 commits: 0,
15430 empty: false,
15431 failed: None,
15432 verified_noop: None,
15433 duration_ms: 0,
15434 folded: false,
15435 }];
15436 state.tally = Some(Tally {
15437 first_choice: BTreeMap::from([('A', 1)]),
15438 borda: BTreeMap::new(),
15439 winner: 'A',
15440 rankings: 1,
15441 unanimous_initial: true,
15442 deliberated: false,
15443 changed_votes: 0,
15444 unanimous_final: true,
15445 tie_break: None,
15446 judges: 0,
15447 present: 0,
15448 quorum: 0,
15449 met_quorum: true,
15450 uncontested: Some("only candidate A produced a change".to_owned()),
15451 });
15452 state.reviews = vec![ReviewRound {
15456 round: 1,
15457 head: head.clone(),
15458 verified_head: None,
15459 verified_at: None,
15460 reviews: Vec::new(),
15461 e2e: Vec::new(),
15462 fix: None,
15463 blocking: 1,
15464 answered: 1,
15465 expected: 1,
15466 clean: false,
15467 verify_retried: false,
15468 e2e_deferred: true,
15469 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15470 progressed: false,
15471 vote_split: false,
15472 reconsideration: Vec::new(),
15473 verdict: None,
15474 }];
15475
15476 let mut runner = Runner {
15477 state,
15478 roles: ResolvedRoles {
15479 implementers: Vec::new(),
15480 judges: Vec::new(),
15481 reviewers: Vec::new(),
15482 fixer: None,
15483 conductor: conductor(),
15484 implementer_roster: Vec::new(),
15485 judge_roster: Vec::new(),
15486 reviewer_roster: Vec::new(),
15487 },
15488 sem: Arc::new(Semaphore::new(1)),
15489 pause: Pause::new(),
15490 interrupt: Pause::new(),
15491 };
15492
15493 let shell = runner.state.config.shell();
15494 runner
15495 .stop_reviewing("round budget spent", &shell, &repo)
15496 .await
15497 .expect("stop_reviewing");
15498
15499 let last = runner.state.reviews.last().expect("round record");
15500 assert_eq!(
15501 last.e2e_status(),
15502 E2eStatus::ResourceBlocked,
15503 "the shared cache is still held; the attempt must read as blocked, not deferred or \
15504 failed: {last:?}"
15505 );
15506 assert_eq!(
15507 last.verified_head.as_deref(),
15508 Some(head.as_str()),
15509 "which commit this attempt targeted is known even though nothing finished checking \
15510 it"
15511 );
15512 let first_attempt_at = last
15513 .verified_at
15514 .expect("when this attempt ran is known too");
15515 assert_ne!(
15516 runner.state.status,
15517 RunStatus::Blocked,
15518 "contention is evidence about the machine, not the patch — it must not settle the \
15519 run as blocked: {:?}",
15520 runner.state.status
15521 );
15522 assert!(
15523 !runner
15524 .state
15525 .events
15526 .iter()
15527 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15528 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15529 runner.state.events
15530 );
15531
15532 runner
15537 .stop_reviewing("round budget spent", &shell, &repo)
15538 .await
15539 .expect("stop_reviewing retry");
15540 assert_eq!(
15541 runner.state.reviews.len(),
15542 1,
15543 "no new round was started: {:?}",
15544 runner.state.reviews
15545 );
15546 let last = runner.state.reviews.last().expect("round record");
15547 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15548 assert!(
15549 last.verified_at.expect("still known") > first_attempt_at,
15550 "a second reentry must be a fresh attempt, not a stale copy of the first"
15551 );
15552 assert_ne!(runner.state.status, RunStatus::Blocked);
15553
15554 held.release();
15555 }
15556
15557 #[tokio::test]
15569 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15570 crate::run::pin_test_home();
15571 let home = crate::run::home();
15572
15573 let tmp = tempfile::tempdir().expect("tempdir");
15574 let repo = tmp.path().join("repo");
15575 std::fs::create_dir_all(&repo).unwrap();
15576 init_repo(&repo);
15577 let head = crate::git::rev_parse(&repo, "HEAD")
15578 .await
15579 .expect("rev-parse");
15580 let cache_dir = tmp.path().join("target");
15581
15582 let mut config = Config::default();
15583 config.verify.e2e = vec![format!(
15584 "CARGO_TARGET_DIR='{}' test -f README.md",
15585 cache_dir.display()
15586 )];
15587 config.graph.review_rounds = 1;
15588 config.graph.timeout_verify = Some(2);
15589 config.graph.timeout_resource_wait = 2;
15590
15591 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15592 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15593 .expect("no io error acquiring directly")
15594 {
15595 crate::cache::AcquireOutcome::Acquired(g) => g,
15596 crate::cache::AcquireOutcome::Busy(b) => {
15597 panic!("expected the direct acquire to win the lease first: {b:?}")
15598 }
15599 };
15600
15601 let mut state = RunState::new(
15602 repo.clone(),
15603 "main".to_owned(),
15604 head.clone(),
15605 "task".to_owned(),
15606 config,
15607 );
15608 state.candidates = vec![Candidate {
15609 index: 0,
15610 label: 'A',
15611 agent: "alpha".to_owned(),
15612 branch: "does-not-exist".to_owned(),
15613 worktree: repo.clone(),
15614 summary: String::new(),
15615 stat: String::new(),
15616 files: 0,
15617 commits: 0,
15618 empty: false,
15619 failed: None,
15620 verified_noop: None,
15621 duration_ms: 0,
15622 folded: false,
15623 }];
15624 state.tally = Some(Tally {
15625 first_choice: BTreeMap::from([('A', 1)]),
15626 borda: BTreeMap::new(),
15627 winner: 'A',
15628 rankings: 1,
15629 unanimous_initial: true,
15630 deliberated: false,
15631 changed_votes: 0,
15632 unanimous_final: true,
15633 tie_break: None,
15634 judges: 0,
15635 present: 0,
15636 quorum: 0,
15637 met_quorum: true,
15638 uncontested: Some("only candidate A produced a change".to_owned()),
15639 });
15640 state.reviews = vec![ReviewRound {
15644 round: 1,
15645 head: head.clone(),
15646 verified_head: Some(head.clone()),
15647 verified_at: Some(jiff::Timestamp::now()),
15648 reviews: Vec::new(),
15649 e2e: vec![CommandOutcome {
15650 command: format!(
15651 "CARGO_TARGET_DIR='{}' test -f README.md",
15652 cache_dir.display()
15653 ),
15654 code: None,
15655 output_tail: "waiting for the shared build cache".to_owned(),
15656 duration_ms: 0,
15657 resource_blocked: true,
15658 }],
15659 fix: None,
15660 blocking: 1,
15661 answered: 1,
15662 expected: 1,
15663 clean: false,
15664 verify_retried: false,
15665 e2e_deferred: false,
15666 e2e_defer_reason: None,
15667 progressed: false,
15668 vote_split: false,
15669 reconsideration: Vec::new(),
15670 verdict: None,
15671 }];
15672
15673 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15674 let mut runner = Runner {
15675 state,
15676 roles: ResolvedRoles {
15677 implementers: Vec::new(),
15678 judges: Vec::new(),
15679 reviewers: Vec::new(),
15680 fixer: None,
15681 conductor: conductor(),
15682 implementer_roster: Vec::new(),
15683 judge_roster: Vec::new(),
15684 reviewer_roster: Vec::new(),
15685 },
15686 sem: Arc::new(Semaphore::new(1)),
15687 pause: Pause::new(),
15688 interrupt: Pause::new(),
15689 };
15690
15691 runner.review_loop().await.expect("review_loop");
15696
15697 assert_eq!(
15698 runner.state.reviews.len(),
15699 1,
15700 "no new round was started on top of the unresolved one: {:?}",
15701 runner.state.reviews
15702 );
15703 let last = &runner.state.reviews[0];
15704 assert_eq!(
15705 last.e2e_status(),
15706 E2eStatus::ResourceBlocked,
15707 "still contended: {last:?}"
15708 );
15709 assert!(
15710 last.verified_at.expect("still known") > first_attempt_at,
15711 "review_loop must have actually retried the check, not left it exactly as found"
15712 );
15713 assert_ne!(
15714 runner.state.status,
15715 RunStatus::Blocked,
15716 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15717 runner.state.status
15718 );
15719
15720 held.release();
15721 }
15722
15723 #[tokio::test]
15724 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15725 crate::run::pin_test_home();
15726 let tmp = tempfile::tempdir().expect("tempdir");
15727 let repo = tmp.path().join("repo");
15728 std::fs::create_dir_all(&repo).unwrap();
15729 init_repo(&repo);
15730
15731 let mut config = Config::default();
15732 config.merge.mode = MergeMode::Pr;
15733 config.graph.land = true;
15734 config.graph.land_approval = false;
15735
15736 let mut state = RunState::new(
15737 repo.clone(),
15738 "main".to_owned(),
15739 "deadbeef".to_owned(),
15740 "task".to_owned(),
15741 config,
15742 );
15743 state.candidates = vec![Candidate {
15744 index: 0,
15745 label: 'A',
15746 agent: "alpha".to_owned(),
15747 branch: "does-not-exist".to_owned(),
15748 worktree: repo.clone(),
15749 summary: String::new(),
15750 stat: String::new(),
15751 files: 0,
15752 commits: 0,
15753 empty: false,
15754 failed: None,
15755 verified_noop: None,
15756 duration_ms: 0,
15757 folded: false,
15758 }];
15759 state.tally = Some(Tally {
15760 first_choice: BTreeMap::from([('A', 1)]),
15761 borda: BTreeMap::new(),
15762 winner: 'A',
15763 rankings: 1,
15764 unanimous_initial: true,
15765 deliberated: false,
15766 changed_votes: 0,
15767 unanimous_final: true,
15768 tie_break: None,
15769 judges: 0,
15770 present: 0,
15771 quorum: 0,
15772 met_quorum: true,
15773 uncontested: Some("only candidate A produced a change".to_owned()),
15774 });
15775 state.reviews = vec![ReviewRound {
15776 round: 1,
15777 head: "deadbeef".to_owned(),
15778 verified_head: None,
15779 verified_at: None,
15780 reviews: Vec::new(),
15781 e2e: Vec::new(),
15782 fix: None,
15783 blocking: 0,
15784 answered: 0,
15785 expected: 0,
15786 clean: true,
15787 verify_retried: false,
15788 e2e_deferred: false,
15789 e2e_defer_reason: None,
15790 progressed: false,
15791 vote_split: false,
15792 reconsideration: Vec::new(),
15793 verdict: None,
15794 }];
15795 state.gate = vec![CommandOutcome {
15796 command: "test".to_owned(),
15797 code: Some(0),
15798 output_tail: String::new(),
15799 duration_ms: 0,
15800 resource_blocked: false,
15801 }];
15802 state.gate_ran = true;
15803 state.status = RunStatus::Landing;
15807 state.merge = Some(MergeOutcome {
15808 mode: MergeMode::Pr,
15809 ok: true,
15810 detail: "https://example.invalid/x/y/pull/1".to_owned(),
15811 empty: false,
15812 });
15813
15814 ask_test_home();
15818 state.save().expect("save landing run");
15821 let store = ask::Questions::open();
15822 let q = ask_open_question(&store, &state.id);
15823
15824 let mut runner = Runner {
15825 state,
15826 roles: ResolvedRoles {
15827 implementers: Vec::new(),
15828 judges: Vec::new(),
15829 reviewers: Vec::new(),
15830 fixer: None,
15831 conductor: conductor(),
15832 implementer_roster: Vec::new(),
15833 judge_roster: Vec::new(),
15834 reviewer_roster: Vec::new(),
15835 },
15836 sem: Arc::new(Semaphore::new(1)),
15837 pause: Pause::new(),
15838 interrupt: Pause::new(),
15839 };
15840
15841 runner.execute().await.expect("execute");
15846
15847 assert_eq!(
15848 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15849 Some("https://example.invalid/x/y/pull/1"),
15850 "reentry must not push again or open a second pull request over the \
15851 one `land` is already watching"
15852 );
15853 assert_ne!(
15854 runner.state.status,
15855 RunStatus::Landing,
15856 "land could not actually reach the fake pull request, so it must \
15857 have given up rather than left the run silently parked forever"
15858 );
15859 assert_eq!(runner.state.status, RunStatus::Blocked);
15863 assert!(
15864 store.get(&q.id).unwrap().status.open(),
15865 "Blocked is still alive; settle_questions must have been a no-op here"
15866 );
15867 }
15868
15869 fn state_with_round(round: ReviewRound) -> RunState {
15870 let mut s = RunState::new(
15871 PathBuf::from("/repo"),
15872 "main".to_owned(),
15873 "abc1234".to_owned(),
15874 "add retries".to_owned(),
15875 Config::default(),
15876 );
15877 s.reviews = vec![round];
15878 s
15879 }
15880
15881 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15882 crate::verdict::Finding {
15883 id: id.to_owned(),
15884 severity,
15885 file: None,
15886 line: None,
15887 title: title.to_owned(),
15888 detail: String::new(),
15889 }
15890 }
15891
15892 #[test]
15893 fn pr_body_names_open_findings_and_declined_ones() {
15894 let round = ReviewRound {
15895 round: 2,
15896 head: "deadbee".to_owned(),
15897 verified_head: None,
15898 verified_at: None,
15899 reviews: vec![ReviewRecord {
15900 attempts: 0,
15901 reviewer: 1,
15902 agent: "alpha".to_owned(),
15903 summary: String::new(),
15904 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15905 vote: None,
15906 failed: None,
15907 duration_ms: 0,
15908 }],
15909 e2e: vec![CommandOutcome {
15910 command: "cargo test".to_owned(),
15911 code: Some(0),
15912 output_tail: String::new(),
15913 duration_ms: 0,
15914 resource_blocked: false,
15915 }],
15916 verify_retried: false,
15917 e2e_deferred: false,
15918 e2e_defer_reason: None,
15919 fix: Some(FixRecord {
15920 agent: "alpha".to_owned(),
15921 addressed: Vec::new(),
15922 rejected: vec![crate::verdict::Rejection {
15923 id: "R1-1-1".to_owned(),
15924 why: "not reachable from any caller".to_owned(),
15925 }],
15926 notes: String::new(),
15927 committed: true,
15928 failed: None,
15929 duration_ms: 0,
15930 continuation: None,
15931 }),
15932 blocking: 0,
15933 answered: 1,
15934 expected: 1,
15935 clean: false,
15936 progressed: true,
15937 vote_split: false,
15938 reconsideration: Vec::new(),
15939 verdict: None,
15940 };
15941 let state = state_with_round(round);
15942 let body = pr_message(&state, 'A').body;
15943
15944 assert!(body.contains("add retries"), "the task must still be there");
15945 assert!(body.contains("R2-1-1"), "{body}");
15946 assert!(!body.contains("unused import"), "{body}");
15947 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15948 assert!(
15949 !body.contains("not reachable from any caller"),
15950 "the reason it was declined: {body}"
15951 );
15952 }
15953
15954 #[test]
15955 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15956 let round = ReviewRound {
15957 round: 1,
15958 head: "deadbee".to_owned(),
15959 verified_head: None,
15960 verified_at: None,
15961 reviews: vec![ReviewRecord {
15962 attempts: 0,
15963 reviewer: 1,
15964 agent: "alpha".to_owned(),
15965 summary: String::new(),
15966 findings: Vec::new(),
15967 vote: None,
15968 failed: None,
15969 duration_ms: 0,
15970 }],
15971 e2e: Vec::new(),
15972 verify_retried: false,
15973 e2e_deferred: false,
15974 e2e_defer_reason: None,
15975 fix: None,
15976 blocking: 0,
15977 answered: 1,
15978 expected: 1,
15979 clean: true,
15980 progressed: false,
15981 vote_split: false,
15982 reconsideration: Vec::new(),
15983 verdict: None,
15984 };
15985 let state = state_with_round(round);
15986 let body = pr_message(&state, 'A').body;
15987 assert!(!body.contains("Open review findings"), "{body}");
15988 assert!(!body.contains("Declined"), "{body}");
15989 }
15990
15991 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15992 let mut state = RunState::new(
15993 PathBuf::from("/repo"),
15994 "main".to_owned(),
15995 "abc1234".to_owned(),
15996 instruction.to_owned(),
15997 Config::default(),
15998 );
15999 state.candidates.push(Candidate {
16000 index: 0,
16001 label: 'A',
16002 agent: "alpha".to_owned(),
16003 branch: "magi/x/A".to_owned(),
16004 worktree: PathBuf::from("/wt"),
16005 summary: summary.to_owned(),
16006 stat: String::new(),
16007 files: 1,
16008 commits: 1,
16009 empty: false,
16010 failed: None,
16011 verified_noop: None,
16012 folded: false,
16013 duration_ms: 0,
16014 });
16015 state
16016 }
16017
16018 fn review_state(subjects: &[&str]) -> RunState {
16019 let mut state = state_with_summary(
16020 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16021 "",
16022 );
16023 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16024 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16025 state
16026 }
16027
16028 #[tokio::test]
16031 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16032 ask_test_home();
16033 let tmp = tempfile::tempdir().unwrap();
16034 let repo = tmp.path().join("repo");
16035 std::fs::create_dir_all(&repo).unwrap();
16036 init_repo(&repo);
16037 let origin = tmp.path().join("origin.git");
16038 let g = |dir: &Path, args: &[&str]| {
16039 let out = std::process::Command::new("git")
16040 .args(args)
16041 .current_dir(dir)
16042 .quiet()
16043 .output()
16044 .expect("spawn git");
16045 assert!(
16046 out.status.success(),
16047 "git {args:?}: {}",
16048 String::from_utf8_lossy(&out.stderr)
16049 );
16050 };
16051 g(
16052 tmp.path(),
16053 &[
16054 "clone",
16055 "--bare",
16056 "-q",
16057 repo.to_str().unwrap(),
16058 origin.to_str().unwrap(),
16059 ],
16060 );
16061 g(
16062 &repo,
16063 &["remote", "add", "origin", origin.to_str().unwrap()],
16064 );
16065 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16066
16067 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16069 g(&repo, &["add", "-A"]);
16070 g(
16071 &repo,
16072 &["commit", "-q", "-m", "chore(deps): update a crate"],
16073 );
16074 g(&repo, &["push", "-q", "origin", "main"]);
16075 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16076 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16077 g(&repo, &["add", "-A"]);
16078 g(
16079 &repo,
16080 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16081 );
16082 g(&repo, &["checkout", "-q", "main"]);
16083
16084 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16085 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16086 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16088 assert_eq!(fallback, c1);
16089
16090 let mut state = review_state(&[
16091 "chore(deps): update a crate",
16092 "fix(daemon): apply a chosen action",
16093 ]);
16094 state.repo = repo.clone();
16095 state.base_branch = "main".to_owned();
16096 state.base_commit = c1;
16097 refresh_reviewed_commits(&mut state, "feat/own").await;
16098 assert_eq!(
16099 state.reviewed_commits,
16100 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16101 );
16102 assert_eq!(
16103 review_title(&state).as_deref(),
16104 Some("fix(daemon): apply a chosen action")
16105 );
16106 assert_eq!(
16107 leaked_subjects(&state, "feat/own").await,
16108 Some(vec!["chore(deps): update a crate".to_owned()])
16109 );
16110 state.config.merge.remote = "nowhere".to_owned();
16113 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16114 }
16115
16116 struct SummaryFixture {
16120 _tmp: tempfile::TempDir,
16121 repo: PathBuf,
16122 reply: PathBuf,
16123 runner: Runner,
16124 }
16125
16126 fn summary_fixture() -> SummaryFixture {
16127 ask_test_home();
16128 let tmp = tempfile::tempdir().unwrap();
16129 let repo = tmp.path().join("repo");
16130 std::fs::create_dir_all(&repo).unwrap();
16131 init_repo(&repo);
16132 let origin = tmp.path().join("origin.git");
16133 let g = |args: &[&str]| {
16134 let out = std::process::Command::new("git")
16135 .args(args)
16136 .current_dir(&repo)
16137 .quiet()
16138 .output()
16139 .expect("spawn git");
16140 assert!(
16141 out.status.success(),
16142 "git {args:?}: {}",
16143 String::from_utf8_lossy(&out.stderr)
16144 );
16145 };
16146 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16147 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16148 g(&["push", "-q", "origin", "main"]);
16149 g(&["checkout", "-q", "-b", "feat/own"]);
16150 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16151 g(&["add", "-A"]);
16152 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16153 let reply = tmp.path().join("reply.txt");
16154 std::fs::write(
16155 &reply,
16156 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16157 )
16158 .unwrap();
16159 let mut runner = runner_at(RunStatus::Gating);
16160 let mut state = review_state(&["feat(own): add the own file"]);
16161 state.repo = repo.clone();
16162 state.base_branch = "main".to_owned();
16163 state.candidates[0].branch = "feat/own".to_owned();
16164 state.candidates[0].worktree = repo.clone();
16165 let mut author = spec("alpha");
16166 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16167 author
16168 .env
16169 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16170 state.config.agents = vec![author];
16171 runner.state = state;
16172 SummaryFixture {
16173 _tmp: tmp,
16174 repo,
16175 reply,
16176 runner,
16177 }
16178 }
16179
16180 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16181 let winner = f.runner.state.candidates[0].clone();
16182 f.runner
16183 .guarded_pr_message(&winner, None, true)
16184 .await
16185 .unwrap()
16186 .expect("an English description never parks")
16187 }
16188
16189 fn clean_round_with_minor_findings() -> ReviewRound {
16190 let mut round = passing_round(2, None);
16191 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16192 round.reviews = vec![ReviewRecord {
16193 attempts: 0,
16194 reviewer: 1,
16195 agent: "alpha".into(),
16196 summary: "内部レビュー".into(),
16197 findings: vec![
16198 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16199 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16200 ],
16201 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16202 failed: None,
16203 duration_ms: 0,
16204 }];
16205 round
16206 }
16207
16208 #[tokio::test]
16209 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16210 for review_only in [false, true] {
16211 let mut f = summary_fixture();
16212 if !review_only {
16213 f.runner.state.candidates[0].agent = "alpha".into();
16214 }
16215 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16216 assert!(f.runner.state.reviews[0].clean);
16217 assert!(f.runner.state.reviews[0].fix.is_none());
16218 assert!(f.runner.state.open_findings().is_empty());
16219 assert_eq!(
16220 crate::pr_outcome::required_finding_ids(&f.runner.state),
16221 ["R2-1-1", "R2-1-2"]
16222 );
16223 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16224 let evidence = crate::pr_outcome::summary_evidence(
16225 &f.runner.state,
16226 crate::pr_outcome::FinalHead::At(&head),
16227 );
16228 assert!(
16229 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16230 );
16231 let omitted = posted(&mut f).await;
16233 assert!(omitted.body.contains("No description"));
16234 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16235 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16236 assert!(f.runner.state.change_summary.is_none());
16237 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16238 let described = posted(&mut f).await;
16239 assert!(described.body.contains("improve documentation for callers"));
16240 assert!(described.body.contains("remaining edge case"));
16241 assert!(described.body.contains("## Open review findings"));
16242 assert!(!described.body.contains("未修正"));
16243 }
16244 }
16245
16246 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16247 let mut f = summary_fixture();
16248 if !review_only {
16249 f.runner.state.candidates[0].agent = "alpha".into();
16250 }
16251 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16252 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16253 fixed
16254 .fix
16255 .as_mut()
16256 .unwrap()
16257 .rejected
16258 .push(crate::verdict::Rejection {
16259 id: "R1-1-1".into(),
16260 why: "呼び出されないコードのため辞退".into(),
16261 });
16262 let mut clean = clean_round_with_minor_findings();
16263 clean.verified_head = Some(head.clone());
16264 f.runner.state.reviews = vec![fixed, clean];
16265 ok_gate(&mut f.runner.state, Some(&head));
16266 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16267 f.runner.state.config.agents.push(scripted("beta", reply));
16268 f.runner.state.config.roles.language_judge =
16269 Some(crate::config::AgentChoice::One("beta".into()));
16270 f
16271 }
16272
16273 #[tokio::test]
16274 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16275 for review_only in [false, true] {
16276 let mut f = rewrite_fixture_with_findings(review_only,
16277 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16278 let m = posted(&mut f).await;
16279 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16280 let expected =
16281 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16282 assert!(m.body.contains(&expected));
16283 assert!(m.body.contains("## Review rounds"));
16284 assert!(
16285 m.body
16286 .contains("Gate (`verify.gate`): passed on the final commit")
16287 );
16288 assert!(
16289 m.body
16290 .contains("Tests (`verify.e2e`): passed on the final commit")
16291 );
16292 let c = f.runner.state.change_summary.as_ref().unwrap();
16293 assert!(
16294 m.body
16295 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16296 );
16297 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16298 assert!(crate::pr_outcome::covers_required_findings(
16299 &f.runner.state,
16300 &m.body
16301 ));
16302 assert!(m.body.contains("documentation remains incomplete"));
16303 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16304 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16305 assert!(f.runner.state.github_text.is_none());
16306 }
16307 }
16308
16309 #[tokio::test]
16310 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16311 for review_only in [false, true] {
16312 for reply in [
16313 "TITLE: fix: public description\nA short English description.",
16314 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16315 ] {
16316 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16317 let winner = f.runner.state.candidates[0].clone();
16318 let posted = f
16319 .runner
16320 .guarded_pr_message(&winner, None, true)
16321 .await
16322 .unwrap();
16323 assert!(posted.is_none());
16324 assert!(f.runner.state.parked);
16325 let withheld = f.runner.state.github_text.as_ref().unwrap();
16326 assert!(
16327 withheld.rewrite.is_none(),
16328 "an incomplete rewrite must not be selectable"
16329 );
16330 let q = ask::Questions::open()
16331 .get(withheld.question.as_ref().unwrap())
16332 .unwrap();
16333 assert!(
16334 !q.choices
16335 .contains(&crate::github_text::USE_REWRITE.to_owned())
16336 );
16337 assert!(
16338 q.detail.contains("R1-1-1")
16339 && q.detail.contains("R2-1-1")
16340 && q.detail.contains("R2-1-2")
16341 );
16342 }
16343 }
16344 }
16345
16346 #[tokio::test]
16347 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16348 for review_only in [false, true] {
16349 let mut f = summary_fixture();
16350 if !review_only {
16351 f.runner.state.candidates[0].agent = "alpha".into();
16352 }
16353 f.runner.state.instruction = "内部タスクの説明".into();
16354 f.runner.state.candidates[0].summary = "内部の候補報告".into();
16355 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16356 f.runner.state.reviews = vec![
16357 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16358 round_with_notes(2, Some("二回目の内部修正報告です。")),
16359 passing_round(3, Some(&head)),
16360 ];
16361 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16362 ok_gate(&mut f.runner.state, Some(&head));
16363 let captured = f.repo.join("public-prompt.txt");
16364 let author = &mut f.runner.state.config.agents[0];
16365 author.command = vec![
16366 "sh".into(),
16367 "-c".into(),
16368 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16369 ];
16370 author.env.insert(
16371 "PROMPT_FILE".into(),
16372 captured.to_string_lossy().into_owned(),
16373 );
16374 let m = posted(&mut f).await;
16375 assert!(m.body.contains("Adds `own.txt`"));
16376 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16377 assert!(m.body.contains("R1-1-1"));
16378 assert!(
16379 m.body
16380 .contains("Gate (`verify.gate`): passed on the final commit")
16381 );
16382 assert!(
16383 m.body
16384 .contains("Tests (`verify.e2e`): passed on the final commit")
16385 );
16386 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16387 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16388 let prompt = std::fs::read_to_string(captured).unwrap();
16389 assert!(prompt.contains(&format!(
16390 "git diff {} {head}",
16391 f.runner.state.change_summary.as_ref().unwrap().base
16392 )));
16393 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16394 assert_eq!(
16395 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16396 "二回目の内部修正報告です。"
16397 );
16398 }
16399 }
16400
16401 #[tokio::test]
16402 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16403 for review_only in [false, true] {
16404 let mut f = summary_fixture();
16405 if !review_only {
16406 f.runner.state.candidates[0].agent = "alpha".into();
16407 }
16408 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16409 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16410 std::fs::write(&f.reply, "no usable description").unwrap();
16411 let failed = posted(&mut f).await;
16412 assert!(
16413 failed
16414 .body
16415 .contains("No description of what this change does could be generated")
16416 );
16417 assert!(failed.body.is_ascii());
16418 assert!(f.runner.state.github_text.is_none());
16419 assert!(f.runner.state.change_summary.is_none());
16420 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16421 let evidence = crate::pr_outcome::summary_evidence(
16422 &f.runner.state,
16423 crate::pr_outcome::FinalHead::At(&head),
16424 );
16425 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16426 assert!(
16427 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16428 .exists()
16429 );
16430 f.runner.state =
16431 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16432 std::fs::write(
16433 &f.reply,
16434 "TITLE: feat(own): add the own file\nAdds the committed own file.",
16435 )
16436 .unwrap();
16437 let retried = posted(&mut f).await;
16438 assert!(retried.body.contains("Adds the committed own file."));
16439 assert_eq!(
16440 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16441 "日本語の修正報告"
16442 );
16443 }
16444 }
16445
16446 #[tokio::test]
16447 async fn same_head_verification_and_findings_invalidate_public_prose() {
16448 let mut f = summary_fixture();
16449 posted(&mut f).await;
16450 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16451 let original = f
16452 .runner
16453 .state
16454 .change_summary
16455 .as_ref()
16456 .unwrap()
16457 .evidence
16458 .clone();
16459 ok_gate(&mut f.runner.state, Some(&head));
16460 std::fs::write(
16461 &f.reply,
16462 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16463 )
16464 .unwrap();
16465 let verified = posted(&mut f).await;
16466 assert!(verified.body.contains("now verified"));
16467 assert_ne!(
16468 f.runner.state.change_summary.as_ref().unwrap().evidence,
16469 original
16470 );
16471 let mut round = round_with_notes(1, Some("内部報告"));
16472 round.reviews.push(ReviewRecord {
16473 attempts: 0,
16474 reviewer: 1,
16475 agent: "alpha".into(),
16476 summary: "内部レビュー".into(),
16477 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16478 vote: None,
16479 failed: None,
16480 duration_ms: 0,
16481 });
16482 round
16483 .fix
16484 .as_mut()
16485 .unwrap()
16486 .rejected
16487 .push(crate::verdict::Rejection {
16488 id: "R1-1-1".into(),
16489 why: "呼び出し元から到達できない".into(),
16490 });
16491 f.runner.state.reviews = vec![round];
16492 let omitted = posted(&mut f).await;
16494 assert!(omitted.body.contains("No description"));
16495 assert!(omitted.body.contains("R1-1-1"));
16496 assert!(!omitted.body.contains("到達"));
16497 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16498 let described = posted(&mut f).await;
16499 assert!(described.body.contains("unreachable from callers"));
16500 assert!(described.body.contains("R1-1-1"));
16501 assert!(!described.body.contains("到達"));
16502 let evidence = f
16503 .runner
16504 .state
16505 .change_summary
16506 .as_ref()
16507 .unwrap()
16508 .evidence
16509 .clone();
16510 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16511 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16512 assert!(posted(&mut f).await.body.contains("rationale updated"));
16513 assert_ne!(
16514 f.runner.state.change_summary.as_ref().unwrap().evidence,
16515 evidence
16516 );
16517 f.runner
16519 .state
16520 .change_summary
16521 .as_mut()
16522 .unwrap()
16523 .evidence
16524 .clear();
16525 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16526 assert!(posted(&mut f).await.body.contains("current specification"));
16527 }
16528
16529 #[tokio::test]
16530 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16531 let mut f = summary_fixture();
16532 let m = posted(&mut f).await;
16533 assert_eq!(m.title, "feat(own): add the own file");
16534 assert!(
16535 m.body
16536 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16537 "{}",
16538 m.body
16539 );
16540 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16541 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16542 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16543 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16544 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16545 let kept = f.runner.state.change_summary.as_ref().unwrap();
16546 assert_eq!(
16547 (kept.head.as_str(), kept.base.as_str()),
16548 (head.as_str(), base.as_str())
16549 );
16550 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16551 }
16552
16553 #[tokio::test]
16554 async fn review_only_summary_receives_the_configured_github_language() {
16555 let mut f = summary_fixture();
16556 f.runner.state.config.graph.github_language = "ja".into();
16557 let captured = f.repo.join("summary-prompt.txt");
16558 let author = &mut f.runner.state.config.agents[0];
16559 author.command = vec![
16560 "sh".into(),
16561 "-c".into(),
16562 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16563 ];
16564 author.env.insert(
16565 "PROMPT_FILE".into(),
16566 captured.to_string_lossy().into_owned(),
16567 );
16568 std::fs::write(
16569 &f.reply,
16570 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16571 )
16572 .unwrap();
16573 let m = posted(&mut f).await;
16574 let prompt = std::fs::read_to_string(captured).unwrap();
16575 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16576 assert!(!prompt.contains("in English"));
16577 assert_eq!(m.title, "feat(own): ファイルを追加");
16578 assert!(m.body.contains("own.txt を追加します。"));
16579 }
16580
16581 #[tokio::test]
16582 async fn review_fixes_come_after_the_whole_change_summary() {
16583 let mut f = summary_fixture();
16584 f.runner.state.reviews = vec![
16585 round_with_notes(1, Some("handled the PATH case")),
16586 round_with_notes(2, None),
16587 ];
16588 let m = posted(&mut f).await;
16589 let summary = m.body.find("## Summary").expect("summary");
16590 let fixes = m.body.find("## Review rounds").expect("review rounds");
16591 assert!(summary < fixes, "{}", m.body);
16592 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16593 assert!(m.body.contains("## Final verification"), "{}", m.body);
16594 }
16595
16596 #[tokio::test]
16597 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16598 let mut f = summary_fixture();
16599 let first = posted(&mut f).await;
16600 let first_head = f.runner.state.change_summary.clone().unwrap().head;
16601 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16603 let again = posted(&mut f).await;
16604 assert_eq!((again.title, again.body), (first.title, first.body));
16605
16606 let g = |args: &[&str]| {
16607 let out = std::process::Command::new("git")
16608 .args(args)
16609 .current_dir(&f.repo)
16610 .quiet()
16611 .output()
16612 .expect("spawn git");
16613 assert!(out.status.success(), "git {args:?}");
16614 };
16615 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16616 g(&["add", "-A"]);
16617 g(&["commit", "-q", "-m", "fix(own): handle more"]);
16618 let second = posted(&mut f).await;
16619 let kept = f.runner.state.change_summary.clone().unwrap();
16620 assert_ne!(kept.head, first_head);
16621 assert!(second.body.contains("Not read yet."), "{}", second.body);
16622 assert!(!second.body.contains("so the first summary is read"));
16623 assert_eq!(second.title, "feat(own): other");
16624 }
16625
16626 #[tokio::test]
16627 async fn an_unavailable_summary_is_stated_and_never_invented() {
16628 let mut f = summary_fixture();
16629 std::fs::write(&f.reply, "no title line here").unwrap();
16630 let m = posted(&mut f).await;
16631 assert!(
16632 m.body
16633 .contains("No description of what this change does could be generated"),
16634 "{}",
16635 m.body
16636 );
16637 assert!(m.body.contains("internal reports remain in local records"));
16638 assert!(f.runner.state.change_summary.is_none());
16639 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16640 let winner = f.runner.state.candidates[0].clone();
16642 let preview = f
16643 .runner
16644 .guarded_pr_message(&winner, None, false)
16645 .await
16646 .unwrap()
16647 .unwrap();
16648 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16649 }
16650
16651 #[test]
16652 fn pr_message_review_single_commit_uses_its_subject() {
16653 let state = review_state(&["feat(nats): per-role user"]);
16654 let m = pr_message(&state, 'A');
16655 assert_eq!(m.title, "feat(nats): per-role user");
16656 assert!(!m.body.contains("Review the work already"), "{}", m.body);
16657 assert!(m.body.contains("## Commits under review"), "{}", m.body);
16658 }
16659
16660 #[test]
16661 fn pr_message_review_multi_commit_takes_the_oldest() {
16662 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16663 let m = pr_message(&state, 'A');
16664 assert_eq!(m.title, "feat: the change");
16665 for s in ["feat: the change", "fix: typo", "fix: again"] {
16666 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16667 }
16668 }
16669
16670 #[test]
16671 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16672 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16673 let mut state = review_state(&[first, "fix: later fixup"]);
16674 state.candidates[0].branch = "機能/ログイン".to_owned();
16675 let m = pr_message(&state, 'A');
16676 assert!(
16677 m.title.starts_with("chore: land candidate A of run"),
16678 "{}",
16679 m.title
16680 );
16681 }
16682 }
16683
16684 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16685 BranchFacts {
16686 commits: commits
16687 .iter()
16688 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16689 .collect(),
16690 stat: stat.to_owned(),
16691 }
16692 }
16693
16694 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16695 let mut r = review_round(true, 0, 1, 1, true, true);
16696 r.round = round;
16697 r.fix = notes.map(|n| FixRecord {
16698 agent: "fixer".to_owned(),
16699 addressed: Vec::new(),
16700 rejected: Vec::new(),
16701 notes: n.to_owned(),
16702 committed: true,
16703 failed: None,
16704 duration_ms: 0,
16705 continuation: None,
16706 });
16707 r
16708 }
16709
16710 #[test]
16711 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16712 let state = review_state(&["ignored"]);
16713 let f = facts(
16714 &[
16715 (
16716 "fix(login): resolve PATH on macOS",
16717 "Login shells skip rc files.",
16718 ),
16719 ("fix: address review", ""),
16720 ],
16721 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16722 );
16723 let m = pr_message_with(&state, 'A', Some(&f));
16724 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16725 assert!(
16726 m.body
16727 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
16728 "{}",
16729 m.body
16730 );
16731 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16732 assert!(m.body.contains("## Diff stat"), "{}", m.body);
16733 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16734 for banned in [
16735 "Review the work already",
16736 "no task statement",
16737 "Original task",
16738 ] {
16739 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16740 }
16741 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16742 }
16743
16744 #[test]
16745 fn pr_message_review_truncates_a_huge_first_commit_body() {
16746 let state = review_state(&["ignored"]);
16747 let f = facts(
16748 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16749 "s",
16750 );
16751 let m = pr_message_with(&state, 'A', Some(&f));
16752 assert!(m.body.len() < 30_000, "{}", m.body.len());
16753 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16754 assert!(
16755 m.body.contains("- ... 1 more commit(s)"),
16756 "{}",
16757 m.body.len()
16758 );
16759 assert!(m.body.ends_with("magi:candidate-a\n"));
16760 }
16761
16762 #[test]
16763 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16764 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16765 assert_eq!(m.title, "feat: x");
16766 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16767 assert!(!m.body.contains("Diff stat"), "{}", m.body);
16768 assert!(!m.body.contains("no task statement"), "{}", m.body);
16769 }
16770
16771 #[test]
16772 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16773 let mut state = review_state(&["日本語の件名"]);
16774 state.candidates[0].branch = "fix/macos-login-path".to_owned();
16775 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16776 state.candidates[0].branch = "機能/ログイン".to_owned();
16777 assert!(
16778 pr_message(&state, 'A')
16779 .title
16780 .starts_with("chore: land candidate A")
16781 );
16782 }
16783
16784 #[test]
16785 fn pr_message_review_fixes_survive_a_clean_final_round() {
16786 let mut state = review_state(&["feat: x"]);
16787 state.reviews = vec![
16788 round_with_notes(1, Some("handled the PATH case")),
16789 round_with_notes(2, None),
16790 ];
16791 let body = pr_message(&state, 'A').body;
16792 assert!(
16793 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16794 "{body}"
16795 );
16796 assert!(body.contains("### Round 1"), "{body}");
16797
16798 state.reviews = vec![
16799 round_with_notes(1, Some("first fix")),
16800 round_with_notes(2, Some("")),
16801 round_with_notes(3, Some("second fix")),
16802 round_with_notes(4, None),
16803 ];
16804 let body = pr_message(&state, 'A').body;
16805 assert!(
16806 body.contains("### Round 1") && !body.contains("first fix"),
16807 "{body}"
16808 );
16809 assert!(
16810 body.contains("### Round 3") && !body.contains("second fix"),
16811 "{body}"
16812 );
16813 assert!(body.contains("### Round 2"), "{body}");
16814 }
16815
16816 const FINAL: &str = "abcdef1234567890";
16817
16818 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16819 let mut r = round_with_notes(round, None);
16820 r.verified_head = verified.map(str::to_owned);
16821 r
16822 }
16823
16824 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16825 state.gate = vec![CommandOutcome {
16826 command: "gate".to_owned(),
16827 code: Some(0),
16828 output_tail: String::new(),
16829 duration_ms: 0,
16830 resource_blocked: false,
16831 }];
16832 state.gate_ran = true;
16833 state.gate_head = head.map(str::to_owned);
16834 }
16835
16836 fn body_at(state: &RunState) -> String {
16837 pr_message_raw(
16838 state,
16839 'A',
16840 None,
16841 crate::pr_outcome::FinalHead::At(FINAL),
16842 ChangeNote::NotAsked,
16843 )
16844 .body
16845 }
16846
16847 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16848
16849 #[test]
16850 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16851 let mut state = review_state(&["feat: x"]);
16852 state.reviews = vec![
16853 round_with_notes(1, Some(SANDBOXED)),
16854 passing_round(2, Some(FINAL)),
16855 ];
16856 ok_gate(&mut state, Some(FINAL));
16857 let body = body_at(&state);
16858 assert!(!body.contains("Handled the PATH case."), "{body}");
16859 assert!(!body.contains("could not commit"), "{body}");
16860 assert!(!body.contains("could not run"), "{body}");
16861 assert!(
16862 body.contains("The fix is committed on the branch"),
16863 "{body}"
16864 );
16865 assert!(!body.contains("sandbox"), "{body}");
16866 assert!(
16867 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16868 "{body}"
16869 );
16870 assert!(
16871 body.contains("Gate (`verify.gate`): passed on the final commit"),
16872 "{body}"
16873 );
16874 assert!(
16876 state.reviews[0]
16877 .fix
16878 .as_ref()
16879 .unwrap()
16880 .notes
16881 .contains("could not commit")
16882 );
16883 }
16884
16885 #[test]
16886 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16887 let mut state = review_state(&["feat: x"]);
16888 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16889 r1.fix.as_mut().unwrap().committed = false;
16890 let mut r2 = passing_round(2, Some(FINAL));
16891 r2.e2e[0].code = Some(1);
16892 state.reviews = vec![r1, r2];
16893 let body = body_at(&state);
16894 assert!(!body.contains("could not commit"), "{body}");
16895 assert!(!body.contains("could not run"), "{body}");
16896 assert!(body.contains("This round produced no commit"), "{body}");
16897 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16898 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16899 assert!(!body.contains("passed"), "{body}");
16900 }
16901
16902 #[test]
16903 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16904 let mut state = review_state(&["feat: x"]);
16905 state.reviews = vec![
16906 round_with_notes(1, Some(SANDBOXED)),
16907 passing_round(2, Some("0123456789abcdef")),
16908 ];
16909 ok_gate(&mut state, Some("0123456789abcdef"));
16910 let body = body_at(&state);
16911 assert!(!body.contains("HTTP tests could not run"), "{body}");
16912 assert!(!body.contains("could not commit"), "{body}");
16914 assert!(
16915 body.contains("passed on an earlier commit (`0123456`)"),
16916 "{body}"
16917 );
16918 assert!(!body.contains("passed on the final commit"), "{body}");
16919
16920 ok_gate(&mut state, None);
16922 let body = body_at(&state);
16923 assert!(
16924 body.contains("passed on a commit that was not recorded"),
16925 "{body}"
16926 );
16927 }
16928
16929 #[test]
16930 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16931 let state = state_with_summary(
16932 "add retries to the client",
16933 "TITLE: feat: retries\n\nDid it.",
16934 );
16935 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16936 assert_eq!(m.title, "feat: retries");
16937 assert!(
16938 m.body.contains("<summary>Original task</summary>"),
16939 "{}",
16940 m.body
16941 );
16942 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16943 assert!(
16944 m.body
16945 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16946 "{}",
16947 m.body
16948 );
16949 }
16950
16951 #[test]
16952 fn pr_message_review_bounds_a_long_english_subject() {
16953 let long = format!("feat: {}", "word ".repeat(100));
16954 let m = pr_message(&review_state(&[&long]), 'A');
16955 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16956 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16957 }
16958
16959 #[test]
16960 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16961 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16962 let m = pr_message(&state, 'A');
16963 assert_eq!(m.title, "add retries");
16964 assert!(m.body.contains("<summary>Original task</summary>"));
16965 assert!(!m.body.contains("Commits under review"));
16966 assert_eq!(landing_subject_source(&state), state.instruction);
16967 }
16968
16969 #[test]
16970 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16971 let state = review_state(&["feat: the change", "fix: typo"]);
16972 let source = landing_subject_source(&state);
16973 assert_eq!(land::merge_subject("", &source), "feat: the change");
16974 assert_eq!(
16975 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16976 "feat: the change"
16977 );
16978 assert_eq!(
16980 land::merge_subject("feat: renamed by hand", &source),
16981 "feat: renamed by hand"
16982 );
16983 let blank = review_state(&["日本語"]);
16984 assert!(
16985 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16986 );
16987 }
16988
16989 #[test]
16990 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16991 let state = review_state(&["feat: the change"]);
16992 let source = landing_subject_source(&state);
16993 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16994 assert_eq!(
16995 land::merge_subject(landing_title(&state, old), &source),
16996 "feat: the change"
16997 );
16998 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16999 let task = state_with_summary("add retries", "");
17000 assert_eq!(landing_title(&task, old), old);
17001 }
17002
17003 #[test]
17004 fn pr_message_describes_the_change_not_the_task() {
17005 let state = state_with_summary(
17006 "今回やってほしいこと: results projector を直す",
17007 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17008 );
17009 let m = pr_message(&state, 'A');
17010 assert_eq!(m.title, "fix(web): batch the runs list reads");
17011 assert!(
17012 m.body.starts_with("## Summary\n\n- reads run.json once"),
17013 "{}",
17014 m.body
17015 );
17016 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17017 let task_at = m.body.find("今回やってほしいこと").unwrap();
17018 let details_at = m.body.find("<details>").unwrap();
17019 assert!(
17020 details_at < task_at,
17021 "the task lives inside <details>: {}",
17022 m.body
17023 );
17024 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17025 assert!(m.body.contains("magi:candidate-a"));
17026 }
17027
17028 #[test]
17029 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17030 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17031 let m = pr_message(&state, 'A');
17032 assert_eq!(m.title, "add retries");
17033 assert!(
17034 m.body.contains("## Summary\n\n- did some things"),
17035 "{}",
17036 m.body
17037 );
17038
17039 let none = RunState::new(
17040 PathBuf::from("/repo"),
17041 "main".to_owned(),
17042 "abc1234".to_owned(),
17043 "add retries".to_owned(),
17044 Config::default(),
17045 );
17046 let m = pr_message(&none, 'A');
17047 assert_eq!(m.title, "add retries");
17048 assert!(!m.body.contains("## Summary"), "{}", m.body);
17049 }
17050
17051 #[test]
17052 fn pr_message_refuses_the_candidate_commit_subject() {
17053 for bad in [
17054 "TITLE: magi: candidate A (uncommitted work)",
17055 "TITLE: chore: stuff (uncommitted work)",
17056 "TITLE: ",
17057 ] {
17058 let state = state_with_summary("add retries", bad);
17059 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17060 }
17061 }
17062
17063 #[test]
17064 fn pr_message_bounds_a_very_long_task_and_title() {
17065 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17066 let state = state_with_summary(&long, "- nothing");
17067 let m = pr_message(&state, 'A');
17068 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17069 assert!(!m.title.contains('\n'));
17070
17071 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17072 let m = pr_message(&state, 'A');
17073 assert!(m.title.starts_with("feat: "));
17074 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17075 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17076 }
17077
17078 fn long_title_of(instruction: &str) -> String {
17079 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17080 }
17081
17082 #[test]
17083 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17084 let first = "Make the landing path keep a readable title for long tasks";
17085 let line = format!(
17086 "{first}. {}",
17087 "And then keep going with more words ".repeat(20)
17088 );
17089 let t = long_title_of(&line);
17090 assert_eq!(t, first);
17091 assert!(!t.starts_with("chore: land"));
17092 }
17093
17094 #[test]
17095 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17096 let line = "word ".repeat(200);
17097 let t = long_title_of(&line);
17098 assert!(t.ends_with("word..."), "{t}");
17099 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17100 }
17101
17102 #[test]
17103 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17104 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17105 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17106 }
17107 }
17108
17109 #[test]
17110 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17111 let mut state = state_with_summary("再試行を追加する", "- nothing");
17112 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17113 state.config.graph.github_language = "ja".into();
17114 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17115 }
17116
17117 #[test]
17118 fn pr_message_title_limit_is_exact() {
17119 let at = "a".repeat(PR_TITLE_MAX);
17120 assert_eq!(long_title_of(&at), at);
17121 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17122 assert!(over.ends_with("..."), "{over}");
17123 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17124 }
17125
17126 #[test]
17127 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17128 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17129 let t = long_title_of(&line);
17130 assert!(t.ends_with("..."), "{t}");
17131 assert!(t.is_ascii(), "{t}");
17132 }
17133
17134 #[test]
17135 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17136 let line = format!(
17137 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17138 "plus more filler words ".repeat(20)
17139 );
17140 let t = long_title_of(&line);
17141 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17142 }
17143
17144 #[test]
17145 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17146 let line = format!("\n\n# {}", "title words ".repeat(40));
17147 let state = state_with_summary(&line, "- nothing");
17148 let m = pr_message(&state, 'A');
17149 assert!(m.title.starts_with("title words"), "{}", m.title);
17150 assert_eq!(
17151 land::merge_subject(&m.title, &landing_subject_source(&state)),
17152 m.title
17153 );
17154 assert_eq!(
17156 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17157 "feat: renamed by hand"
17158 );
17159 }
17160
17161 #[tokio::test]
17162 async fn github_text_rewrite_is_bounded_and_falls_back() {
17163 crate::run::pin_test_home();
17164 for (reply, accepted) in [
17165 (
17166 "TITLE: fix: retries\nAdd retries for failed requests.",
17167 true,
17168 ),
17169 (
17170 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17171 false,
17172 ),
17173 (
17174 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17175 true,
17176 ),
17177 (
17178 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17179 true,
17180 ),
17181 (
17182 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17183 true,
17184 ),
17185 (
17186 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17187 false,
17188 ),
17189 ("TITLE: fix: retries\n", false),
17190 ] {
17191 let dir = tempfile::tempdir().unwrap();
17192 let mut runner = runner_at(RunStatus::Gating);
17193 runner.state = state_with_summary(
17194 "add retries",
17195 "TITLE: fix: retries\n日本語の説明を書きます。",
17196 );
17197 runner.state.repo = dir.path().to_owned();
17198 runner.state.candidates[0].worktree = dir.path().to_owned();
17199 let mut author = spec("alpha");
17200 author.command = vec![
17201 "sh".into(),
17202 "-c".into(),
17203 "printf '%s' \"$REWRITE_REPLY\"".into(),
17204 ];
17205 author.env.insert("REWRITE_REPLY".into(), reply.into());
17206 runner.state.config.agents = vec![author];
17207 seed_public_summary(&mut runner);
17208 let winner = runner.state.candidates[0].clone();
17209 let first = runner
17210 .guarded_pr_message(&winner, None, true)
17211 .await
17212 .unwrap();
17213 let message = if accepted {
17214 first.expect("a passing rewrite never asks")
17215 } else {
17216 assert!(first.is_none());
17218 assert!(runner.state.parked);
17219 let store = ask::Questions::open();
17220 let asked: Vec<_> = store
17221 .list()
17222 .into_iter()
17223 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17224 .collect();
17225 assert_eq!(asked.len(), 1);
17226 assert!(!asked[0].detail.contains("secret"));
17227 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17228 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17229 assert_eq!(
17230 asked[0]
17231 .choices
17232 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17233 rewrite.is_some()
17234 );
17235 store
17238 .update(&asked[0].id, |q| {
17239 q.abandon("test");
17240 Ok(())
17241 })
17242 .unwrap();
17243 let again = runner
17244 .guarded_pr_message(&winner, None, true)
17245 .await
17246 .unwrap()
17247 .expect("an abandoned question falls back");
17248 let n = store
17249 .list()
17250 .into_iter()
17251 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17252 .count();
17253 assert_eq!(n, 1, "asked once per rejected text");
17254 again
17255 };
17256 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17257 assert!(!message.body.contains("secret"));
17258 assert!(!message.body.contains("prefix"));
17259 assert!(!message.title.contains("secret"));
17260 assert_eq!(
17261 message.body.contains("Add retries for failed requests."),
17262 accepted
17263 );
17264 assert_eq!(
17265 runner.state.seats["github-text-rewrite"].turns, 1,
17266 "only one rewrite invocation"
17267 );
17268 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17269 if accepted {
17270 assert!(runner.state.github_text.is_none());
17271 assert!(!runner.state.parked);
17272 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17273 assert!(message.body.contains("## Final verification"));
17274 assert!(message.body.contains("Evidence: final diff"));
17275 assert!(
17276 message
17277 .body
17278 .contains(&format!("magi:run/{}", runner.state.id))
17279 );
17280 }
17281 if !accepted {
17282 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17283 assert!(
17284 message
17285 .body
17286 .contains(&format!("magi:run/{}", runner.state.id))
17287 );
17288 }
17289 }
17290 }
17291
17292 fn scripted(id: &str, reply: &str) -> AgentSpec {
17293 let mut a = spec(id);
17294 a.command = vec![
17295 "sh".into(),
17296 "-c".into(),
17297 "printf '%s' \"$REWRITE_REPLY\"".into(),
17298 ];
17299 a.env.insert("REWRITE_REPLY".into(), reply.into());
17300 a
17301 }
17302
17303 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17304 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17305
17306 fn seed_public_summary(runner: &mut Runner) {
17308 let repo = &runner.state.repo;
17309 let g = |args: &[&str]| {
17310 let out = std::process::Command::new("git")
17311 .args(args)
17312 .current_dir(repo)
17313 .quiet()
17314 .output()
17315 .unwrap();
17316 assert!(out.status.success(), "git {args:?}");
17317 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17318 };
17319 g(&["init", "-q", "-b", "main"]);
17320 g(&[
17321 "-c",
17322 "user.name=Test",
17323 "-c",
17324 "user.email=test@example.invalid",
17325 "commit",
17326 "-q",
17327 "--allow-empty",
17328 "-m",
17329 "feat: fixture",
17330 ]);
17331 let head = g(&["rev-parse", "HEAD"]);
17332 g(&["update-ref", "refs/remotes/origin/main", &head]);
17333 let evidence = crate::pr_outcome::summary_evidence(
17334 &runner.state,
17335 crate::pr_outcome::FinalHead::At(&head),
17336 );
17337 let generated = &runner.state.candidates[0].summary;
17338 runner.state.change_summary = Some(crate::run::ChangeSummary {
17339 base: head.clone(),
17340 head,
17341 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17342 title: summary_title(generated).unwrap(),
17343 body: summary_without_title(generated),
17344 });
17345 }
17346
17347 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17348 let mut runner = runner_at(RunStatus::Gating);
17349 runner.state = state_with_summary("add retries", JA_SUMMARY);
17350 if review_only {
17351 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17352 }
17353 runner.state.repo = dir.to_owned();
17354 runner.state.candidates[0].worktree = dir.to_owned();
17355 runner.state.config.agents = agents;
17356 seed_public_summary(&mut runner);
17357 runner
17358 }
17359
17360 fn gate_questions(runner: &Runner) -> usize {
17361 ask::Questions::open()
17362 .list()
17363 .into_iter()
17364 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17365 .count()
17366 }
17367
17368 #[tokio::test]
17369 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17370 crate::run::pin_test_home();
17371 let dir = tempfile::tempdir().unwrap();
17372 let mut runner = rewrite_runner(
17373 dir.path(),
17374 true,
17375 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17376 );
17377 runner.state.config.roles.language_judge =
17378 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17379 let winner = runner.state.candidates[0].clone();
17380 let msg = runner
17381 .guarded_pr_message(&winner, None, true)
17382 .await
17383 .unwrap()
17384 .expect("an English rewrite posts without asking");
17385 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17386 assert!(msg.body.contains("<details>kept</details>"));
17387 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17388 assert_eq!(gate_questions(&runner), 0);
17389 assert!(runner.state.github_text.is_none());
17390 assert!(!runner.state.seats.contains_key("impl-A"));
17391 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17392 }
17393
17394 #[tokio::test]
17395 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17396 crate::run::pin_test_home();
17397 let dir = tempfile::tempdir().unwrap();
17398 let mut runner = rewrite_runner(
17399 dir.path(),
17400 false,
17401 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17402 );
17403 runner.state.config.roles.language_judge =
17404 Some(crate::config::AgentChoice::One("beta".into()));
17405 runner.state.seats.insert(
17406 "impl-A".into(),
17407 SeatState::new("impl-A", "alpha", runner.state.seed),
17408 );
17409 let winner = runner.state.candidates[0].clone();
17410 let msg = runner
17411 .guarded_pr_message(&winner, None, true)
17412 .await
17413 .unwrap()
17414 .expect("rewritten");
17415 assert!(msg.body.contains("<details>kept</details>"));
17416 assert_eq!(runner.state.seats["impl-A"].turns, 0);
17417 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17418 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17419 }
17420
17421 #[tokio::test]
17422 async fn rewrite_reuses_a_resumable_implementer_session() {
17423 crate::run::pin_test_home();
17424 let dir = tempfile::tempdir().unwrap();
17425 let mut runner = rewrite_runner(
17426 dir.path(),
17427 false,
17428 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17429 );
17430 runner.state.config.roles.language_judge =
17431 Some(crate::config::AgentChoice::One("beta".into()));
17432 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17433 seat.turns = 1;
17434 runner.state.seats.insert("impl-A".into(), seat);
17435 let winner = runner.state.candidates[0].clone();
17436 runner
17437 .guarded_pr_message(&winner, None, true)
17438 .await
17439 .unwrap()
17440 .expect("rewritten");
17441 assert_eq!(runner.state.seats["impl-A"].turns, 2);
17442 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17443 }
17444
17445 #[tokio::test]
17446 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17447 crate::run::pin_test_home();
17448 let dir = tempfile::tempdir().unwrap();
17449 let mut broken = spec("bad");
17450 broken.command = vec!["false".into()];
17451 let mut runner = rewrite_runner(
17452 dir.path(),
17453 true,
17454 vec![
17455 broken,
17456 scripted("noformat", "Add retries."),
17457 scripted("good", EN_REWRITE),
17458 ],
17459 );
17460 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17461 "bad".into(),
17462 "noformat".into(),
17463 "good".into(),
17464 ]));
17465 let winner = runner.state.candidates[0].clone();
17466 let msg = runner
17467 .guarded_pr_message(&winner, None, true)
17468 .await
17469 .unwrap()
17470 .expect("the third agent rewrites");
17471 assert!(msg.body.contains("`src/net.rs`"));
17472 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17473 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17474 }
17475
17476 #[tokio::test]
17477 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17478 crate::run::pin_test_home();
17479 for (agents, role, saved) in [
17481 (
17482 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17483 Some("beta"),
17484 true,
17485 ),
17486 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17487 (Vec::new(), None, false),
17488 ] {
17489 let dir = tempfile::tempdir().unwrap();
17490 let mut runner = rewrite_runner(dir.path(), true, agents);
17491 runner.state.config.roles.language_judge =
17492 role.map(|r| crate::config::AgentChoice::One(r.into()));
17493 let winner = runner.state.candidates[0].clone();
17494 let first = runner
17495 .guarded_pr_message(&winner, None, true)
17496 .await
17497 .unwrap();
17498 assert!(first.is_none(), "withheld text parks the run");
17499 assert!(runner.state.parked);
17500 assert_eq!(gate_questions(&runner), 1);
17501 let g = runner.state.github_text.as_ref().unwrap();
17502 assert_eq!(g.rewrite.is_some(), saved);
17503 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17504 }
17505 }
17506
17507 #[tokio::test]
17508 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17509 crate::run::pin_test_home();
17510 let dir = tempfile::tempdir().unwrap();
17511 let mut runner = runner_at(RunStatus::Gating);
17512 runner.state =
17513 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17514 runner.state.repo = dir.path().to_owned();
17515 runner.state.candidates[0].worktree = dir.path().to_owned();
17516 let mut author = spec("alpha");
17517 author.command = vec![
17518 "sh".into(),
17519 "-c".into(),
17520 "printf '%s' \"$REWRITE_REPLY\"".into(),
17521 ];
17522 author.env.insert(
17523 "REWRITE_REPLY".into(),
17524 format!(
17525 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17526 "English sentence. ".repeat(600)
17527 ),
17528 );
17529 runner.state.config.agents = vec![author];
17530 seed_public_summary(&mut runner);
17531 let winner = runner.state.candidates[0].clone();
17532 assert!(
17533 runner
17534 .guarded_pr_message(&winner, None, true)
17535 .await
17536 .unwrap()
17537 .is_none()
17538 );
17539 let record = runner.state.github_text.clone().unwrap();
17540 let rewrite = record.rewrite.clone().unwrap();
17541 assert_eq!(rewrite.failed, ["body-language"]);
17542 assert!(
17543 rewrite
17544 .body
17545 .ends_with(&format!("magi:run/{}", runner.state.id))
17546 );
17547 let store = ask::Questions::open();
17548 let q = store.get(record.question.as_ref().unwrap()).unwrap();
17549 assert!(
17550 q.choices
17551 .contains(&crate::github_text::USE_REWRITE.to_owned())
17552 );
17553 for label in [
17554 "ORIGINAL",
17555 "FALLBACK",
17556 "REWRITE",
17557 "Rules still failed: body-language",
17558 "truncated",
17559 ] {
17560 assert!(q.detail.contains(label), "{label}");
17561 }
17562 let artifact = std::fs::read_to_string(crate::run::artifact_path(
17563 &runner.state,
17564 &crate::github_text::artifact_name(&record.fingerprint),
17565 ))
17566 .unwrap();
17567 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17568 runner.state =
17570 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17571 store
17572 .update(&q.id, |q| {
17573 q.say("まだ日本語")?;
17574 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17575 })
17576 .unwrap();
17577 assert!(
17578 runner
17579 .guarded_pr_message(&winner, None, true)
17580 .await
17581 .unwrap()
17582 .is_none()
17583 );
17584 let retry = runner.state.github_text.as_ref().unwrap();
17585 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17586 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17587 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17588 assert!(
17589 q.posting_comparison
17590 .as_ref()
17591 .unwrap()
17592 .matches_detail(&q.detail)
17593 );
17594 store
17595 .update(&q.id, |q| {
17596 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17597 })
17598 .unwrap();
17599 let message = runner
17600 .guarded_pr_message(&winner, None, true)
17601 .await
17602 .unwrap()
17603 .unwrap();
17604 assert_eq!(message.title, rewrite.title);
17605 assert_eq!(message.body, rewrite.body);
17606 assert_eq!(
17607 runner.state.github_text.as_ref().unwrap().choice,
17608 Some(GithubTextChoice::Rewrite)
17609 );
17610 let again = runner
17611 .guarded_pr_message(&winner, None, true)
17612 .await
17613 .unwrap()
17614 .unwrap();
17615 assert_eq!(again.title, message.title);
17616 assert_eq!(again.body, message.body);
17617 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17618 assert!(
17619 !runner
17620 .state
17621 .events
17622 .iter()
17623 .any(|e| e.message.contains(&rewrite.title)
17624 || e.message.contains("English sentence."))
17625 );
17626 }
17627
17628 #[tokio::test]
17629 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17630 crate::run::pin_test_home();
17631 let dir = tempfile::tempdir().unwrap();
17632 let mut runner = runner_at(RunStatus::Gating);
17633 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17634 runner.state.repo = dir.path().to_owned();
17635 runner.state.candidates[0].worktree = dir.path().to_owned();
17636 seed_public_summary(&mut runner);
17637 let winner = runner.state.candidates[0].clone();
17638 assert!(
17639 runner
17640 .guarded_pr_message(&winner, None, true)
17641 .await
17642 .unwrap()
17643 .is_none()
17644 );
17645 let record = runner.state.github_text.as_ref().unwrap();
17646 assert!(record.rewrite.is_none());
17647 let mut old = serde_json::to_value(record).unwrap();
17649 old.as_object_mut().unwrap().remove("rewrite");
17650 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17651 assert!(old.rewrite.is_none());
17652 let id = record.question.clone().unwrap();
17653 ask::Questions::open()
17654 .update(&id, |q| {
17655 q.choices.push(crate::github_text::USE_REWRITE.into());
17656 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17657 })
17658 .unwrap();
17659 let message = runner
17660 .guarded_pr_message(&winner, None, true)
17661 .await
17662 .unwrap()
17663 .unwrap();
17664 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17665 assert_eq!(
17666 runner.state.github_text.as_ref().unwrap().choice,
17667 Some(GithubTextChoice::Fallback)
17668 );
17669 }
17670
17671 #[tokio::test]
17672 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17673 crate::run::pin_test_home();
17674 let dir = tempfile::tempdir().unwrap();
17675 let mut runner = runner_at(RunStatus::Gating);
17676 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17677 runner.state.repo = dir.path().to_owned();
17678 runner.state.candidates[0].worktree = dir.path().to_owned();
17679 seed_public_summary(&mut runner);
17680 let winner = runner.state.candidates[0].clone();
17681 assert!(
17682 runner
17683 .guarded_pr_message(&winner, None, true)
17684 .await
17685 .unwrap()
17686 .is_none()
17687 );
17688 let store = ask::Questions::open();
17689 let find = |run: &str| {
17690 store
17691 .list()
17692 .into_iter()
17693 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17694 .unwrap()
17695 };
17696 let run = runner.state.id.clone();
17697 let q = find(&run);
17698 let fp = runner
17699 .state
17700 .github_text
17701 .as_ref()
17702 .unwrap()
17703 .fingerprint
17704 .clone();
17705 let name = crate::github_text::artifact_name(&fp);
17706 let path = crate::run::artifact_path(&runner.state, &name);
17707 assert!(q.detail.contains(&path.display().to_string()));
17708 assert!(q.detail.contains("title-language: title"));
17709 let saved = std::fs::read_to_string(&path).expect("artifact written");
17710 assert!(saved.contains("日本語のタイトル"));
17711 assert!(q.cwd.is_none());
17712 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17713 let say = |id: &str, text: &str, choice: &str| {
17715 store
17716 .update(id, |q| {
17717 q.thread.push(ask::Turn {
17718 who: ask::Who::Operator,
17719 body: text.to_owned(),
17720 at: jiff::Timestamp::now(),
17721 note: None,
17722 });
17723 q.answer(ask::Answer::Choice(choice.to_owned()))
17724 })
17725 .unwrap();
17726 };
17727 say(&q.id, "まだ日本語", "use my text");
17728 assert!(
17729 runner
17730 .guarded_pr_message(&winner, None, true)
17731 .await
17732 .unwrap()
17733 .is_none()
17734 );
17735 let second = find(&run);
17736 assert_ne!(second.id, q.id);
17737 assert!(second.detail.contains("did not pass"));
17738 assert!(second.detail.contains("日本語のタイトル"));
17741 assert!(!second.detail.contains("まだ日本語"));
17742 assert!(second.detail.contains(&path.display().to_string()));
17743 say(&second.id, "fix: retry failed requests", "use my text");
17744 let message = runner
17745 .guarded_pr_message(&winner, None, true)
17746 .await
17747 .unwrap()
17748 .expect("a vetted replacement is posted");
17749 assert_eq!(message.title, "fix: retry failed requests");
17750 assert_eq!(
17751 runner
17752 .state
17753 .github_text
17754 .as_ref()
17755 .unwrap()
17756 .chosen_title
17757 .as_deref(),
17758 Some("fix: retry failed requests")
17759 );
17760 assert!(runner.state.github_text.as_ref().unwrap().resolved);
17761 let again = runner
17763 .guarded_pr_message(&winner, None, true)
17764 .await
17765 .unwrap()
17766 .unwrap();
17767 assert_eq!(again.title, message.title);
17768 assert_eq!(
17769 store
17770 .list()
17771 .into_iter()
17772 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17773 .count(),
17774 2
17775 );
17776 }
17777
17778 #[test]
17779 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17780 let mut state = state_with_summary(
17784 "add retries",
17785 "TITLE: fix(web): batch reads\n- reads run.json once",
17786 );
17787 state.config.graph.language = "ja".to_owned();
17788 let m = pr_message(&state, 'A');
17789 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17790 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17791
17792 let task = "今回やってほしいこと: results projector を直す";
17795 let mut state = state_with_summary(task, "- no title line");
17796 state.config.graph.language = "ja".to_owned();
17797 let m = pr_message(&state, 'A');
17798 assert_eq!(
17799 m.title,
17800 format!("chore: land candidate A of run {}", state.id)
17801 );
17802 assert!(
17803 m.body.contains(&format!(
17804 "<summary>Original task</summary>\n\n{task}\n\n</details>"
17805 )),
17806 "{}",
17807 m.body
17808 );
17809 }
17810
17811 #[test]
17812 fn pr_message_scrubs_home_paths_and_addresses() {
17813 let state = state_with_summary(
17814 "fix it in /Users/someone/src/x",
17815 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17816 );
17817 let m = pr_message(&state, 'A');
17818 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17819 assert!(!m.body.contains(leak), "{}", m.body);
17820 }
17821 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17822 }
17823
17824 #[test]
17825 fn pr_message_survives_a_task_that_closes_details() {
17826 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17827 let m = pr_message(&state, 'A');
17828 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17829 }
17830
17831 #[test]
17832 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17833 let cmd = manual_merge_command(
17834 MergeStyle::Squash,
17835 Path::new("/repo"),
17836 "b",
17837 "fix: \"quoted\" $(x) `y`\n\nbody",
17838 );
17839 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17840 }
17841
17842 #[test]
17843 fn manual_merge_command_matches_the_configured_style() {
17844 let repo = Path::new("/repo");
17845 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17846
17847 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17848 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17849
17850 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17851 assert_eq!(
17852 squash,
17853 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17854 \"Merge magi run 0832 (candidate A)\""
17855 );
17856
17857 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17858 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17859 }
17860
17861 #[test]
17862 fn a_nudge_gets_a_quarter_of_the_budget() {
17863 assert_eq!(retry_budget(secs(1200), true), secs(300));
17865 assert_eq!(retry_budget(secs(3600), true), secs(900));
17866 }
17867
17868 #[test]
17869 fn a_resent_prompt_keeps_the_whole_budget() {
17870 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17873 assert_eq!(retry_budget(secs(60), false), secs(60));
17874 }
17875
17876 #[test]
17877 fn the_floor_never_exceeds_the_original_budget() {
17878 assert_eq!(retry_budget(secs(60), true), secs(60));
17882 assert_eq!(retry_budget(secs(480), true), secs(120));
17883 assert_eq!(retry_budget(secs(0), true), secs(0));
17884 }
17885
17886 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17887 agent::CommandEvidence {
17888 id: "item1".to_owned(),
17889 description: "cargo test".to_owned(),
17890 exit_code,
17891 result_summary: String::new(),
17892 source: "codex".to_owned(),
17893 }
17894 }
17895
17896 #[test]
17897 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17898 assert!(!has_unconfirmed_command(&[]));
17902 }
17903
17904 #[test]
17905 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17906 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17910 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17911 assert!(!has_unconfirmed_command(&[
17912 evidence(Some(0)),
17913 evidence(Some(101))
17914 ]));
17915 }
17916
17917 #[test]
17918 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17919 assert!(has_unconfirmed_command(&[
17920 evidence(Some(0)),
17921 evidence(None)
17922 ]));
17923 }
17924
17925 #[test]
17926 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17927 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17928 assert_eq!(
17929 verified_noop_claim(true, &[], text).as_deref(),
17930 Some("already fixed by b32cfc4, on main.")
17931 );
17932 }
17933
17934 #[test]
17935 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17936 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17939 assert!(verified_noop_claim(false, &[], text).is_none());
17940 }
17941
17942 #[test]
17943 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17944 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17945 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17946 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17948 }
17949
17950 #[test]
17951 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17952 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17953 }
17954
17955 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17958 runner.state.candidates = shape
17959 .iter()
17960 .enumerate()
17961 .map(|(i, &(empty, verified))| Candidate {
17962 index: i,
17963 label: (b'A' + i as u8) as char,
17964 agent: "sonnet".to_owned(),
17965 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17966 worktree: PathBuf::from(format!("/wt/{i}")),
17967 summary: String::new(),
17968 stat: String::new(),
17969 files: 0,
17970 commits: 0,
17971 empty,
17972 failed: None,
17973 verified_noop: verified.map(str::to_owned),
17974 duration_ms: 0,
17975 folded: false,
17976 })
17977 .collect();
17978 }
17979
17980 #[test]
17981 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17982 ask_test_home();
17983 let mut runner = runner_at(RunStatus::Implementing);
17984 set_candidates(
17985 &mut runner,
17986 &[
17987 (true, Some("already on main at b32cfc4")),
17988 (true, Some("same fix, see the existing test")),
17989 ],
17990 );
17991
17992 runner
17993 .after_implement()
17994 .expect("a verified no-op is not an error");
17995
17996 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17997 }
17998
17999 #[test]
18000 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18001 ask_test_home();
18002 let mut runner = runner_at(RunStatus::Implementing);
18003 set_candidates(
18007 &mut runner,
18008 &[(true, Some("already on main at b32cfc4")), (true, None)],
18009 );
18010
18011 let err = runner
18012 .after_implement()
18013 .expect_err("an unverified empty candidate must still fail the run");
18014
18015 assert!(
18016 err.to_string().contains("no candidate produced a change"),
18017 "{err}"
18018 );
18019 assert_eq!(runner.state.status, RunStatus::Failed);
18020 }
18021
18022 #[test]
18023 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18024 ask_test_home();
18025 let mut runner = runner_at(RunStatus::Implementing);
18026 set_candidates(&mut runner, &[(true, None), (true, None)]);
18027
18028 let err = runner
18029 .after_implement()
18030 .expect_err("no candidate declared anything; this is an ordinary failure");
18031
18032 assert!(
18033 err.to_string().contains("no candidate produced a change"),
18034 "{err}"
18035 );
18036 assert_eq!(runner.state.status, RunStatus::Failed);
18037 }
18038}
18039
18040#[derive(Debug)]
18043struct GraftRestoreFailed(String);
18044
18045impl std::fmt::Display for GraftRestoreFailed {
18046 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18047 write!(
18048 f,
18049 "could not restore the winner to its pre-graft commit: {}",
18050 self.0
18051 )
18052 }
18053}
18054
18055impl std::error::Error for GraftRestoreFailed {}
18056
18057#[cfg(test)]
18061mod resource_wait_tests {
18062 use super::*;
18063 use std::collections::BTreeMap;
18064
18065 fn sh(id: &str, script: &str) -> AgentSpec {
18066 AgentSpec {
18067 id: id.to_owned(),
18068 kind: crate::config::AgentKind::Command,
18069 model: None,
18070 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18071 extra_args: Vec::new(),
18072 env: BTreeMap::new(),
18073 prompt_delivery: None,
18074 }
18075 }
18076
18077 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18078 SeatJob {
18079 seat: SeatState::new("impl-A", &spec.id, 1),
18080 spec,
18081 cwd: dir.to_path_buf(),
18082 prompt: "do it".to_owned(),
18083 timeout,
18084 allow_write: true,
18085 sessions: false,
18086 artifacts: dir.join("art"),
18087 stem: "impl-A".to_owned(),
18088 handover: None,
18089 }
18090 }
18091
18092 fn state(resource_wait_secs: u64) -> RunState {
18093 let mut config = Config::default();
18094 config.graph.timeout_resource_wait = resource_wait_secs;
18095 RunState::new(
18096 PathBuf::from("/repo"),
18097 "main".to_owned(),
18098 "abc1234".to_owned(),
18099 "task".to_owned(),
18100 config,
18101 )
18102 }
18103
18104 fn hold(cache: &Path) -> crate::cache::Guard {
18105 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18106 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18107 crate::cache::AcquireOutcome::Acquired(g) => g,
18108 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18109 }
18110 }
18111
18112 async fn run(
18113 jobs: Vec<SeatJob>,
18114 sem: Arc<Semaphore>,
18115 cache: Option<&Path>,
18116 st: &mut RunState,
18117 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18118 let prompts = Prompts::default();
18119 let ctx = WaveCtx {
18120 run: "r",
18121 node: "implement",
18122 prompts: &prompts,
18123 cache,
18124 round: None,
18125 carry_seats: false,
18126 pauses: None,
18127 };
18128 wave(jobs, sem, &ctx, st, 0).await
18129 }
18130
18131 #[tokio::test]
18132 async fn verification_cache_wait_preserves_the_command_budget() {
18133 crate::run::pin_test_home();
18134 let tmp = tempfile::tempdir().unwrap();
18135 let cache = tmp.path().join("target");
18136 let guard = hold(&cache);
18137 let release = tokio::spawn(async move {
18138 tokio::time::sleep(Duration::from_millis(100)).await;
18139 guard.release();
18140 });
18141 let mut st = state(10);
18142 let budget = Duration::from_millis(50);
18143 let wait = repository_wait(&st, None);
18144 let (outcomes, _) = with_cache_lease(
18145 &mut st,
18146 Some(&cache),
18147 "gate",
18148 "gate",
18149 tmp.path(),
18150 "abc1234",
18151 budget,
18152 "test gate",
18153 &wait,
18154 |_, available| async move {
18155 assert_eq!(
18156 available, budget,
18157 "contention must not shorten verification"
18158 );
18159 (Vec::new(), false, Vec::new())
18160 },
18161 )
18162 .await
18163 .unwrap();
18164 release.await.unwrap();
18165 assert!(outcomes.is_empty(), "{outcomes:?}");
18166 assert!(
18167 st.events
18168 .iter()
18169 .any(|event| event.message.contains("waiting for"))
18170 );
18171 assert!(st.resource_wait.is_none());
18172 hold(&cache).release();
18174 }
18175
18176 #[tokio::test]
18177 async fn verification_cache_wait_expires_without_starting_commands() {
18178 crate::run::pin_test_home();
18179 let tmp = tempfile::tempdir().unwrap();
18180 let cache = tmp.path().join("target");
18181 let _held = hold(&cache);
18182 let mut st = state(0);
18183 let wait = repository_wait(&st, None);
18184 let (outcomes, _) = with_cache_lease(
18185 &mut st,
18186 Some(&cache),
18187 "gate",
18188 "gate",
18189 tmp.path(),
18190 "abc1234",
18191 Duration::from_secs(60),
18192 "test gate",
18193 &wait,
18194 |_, _| async { panic!("commands must not start without the cache lease") },
18195 )
18196 .await
18197 .unwrap();
18198 assert_eq!(outcomes.len(), 1);
18199 assert!(outcomes[0].resource_blocked);
18200 assert!(st.resource_wait.is_none());
18201 }
18202
18203 #[tokio::test]
18204 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18205 crate::run::pin_test_home();
18206 let tmp = tempfile::tempdir().unwrap();
18207 let cache = tmp.path().join("target-a");
18208 let guard = hold(&cache);
18209 tokio::spawn(async move {
18211 tokio::time::sleep(Duration::from_millis(1500)).await;
18212 guard.release();
18213 });
18214 let mut st = state(60);
18215 let results = run(
18216 vec![job(
18217 sh("alpha", "echo ok"),
18218 tmp.path(),
18219 Duration::from_secs(1),
18220 )],
18221 Arc::new(Semaphore::new(1)),
18222 Some(&cache),
18223 &mut st,
18224 )
18225 .await;
18226 assert!(
18227 matches!(results[0].2, AgentOutcome::Ok(_)),
18228 "the wait must not be charged to the agent"
18229 );
18230 assert!(st.resource_block.is_none());
18231 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18232 assert!(st.resource_wait.is_none() && st.active.is_empty());
18233 }
18234
18235 #[tokio::test]
18236 #[cfg(unix)]
18237 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18238 use std::os::fd::AsRawFd as _;
18239 crate::run::pin_test_home();
18240 let tmp = tempfile::tempdir().unwrap();
18241 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18242 let common = tmp.path().join(".git");
18243 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18244 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18245 assert_eq!(
18246 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18247 0
18248 );
18249 let mut st = state(1);
18250 st.config.coordination = crate::config::test_coordination();
18251 let mut reader = job(
18252 sh("reader", "echo must-not-run"),
18253 tmp.path(),
18254 Duration::from_secs(30),
18255 );
18256 reader.allow_write = false;
18257 let results = run(
18258 vec![reader.clone()],
18259 Arc::new(Semaphore::new(1)),
18260 None,
18261 &mut st,
18262 )
18263 .await;
18264 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18265 assert_eq!(results[0].1.turns, 0);
18266 assert!(FailClass::of(&results[0].2).is_none());
18267 assert_eq!(
18268 st.resource_block.as_ref().unwrap().kind,
18269 "repository coordination"
18270 );
18271 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18272 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18273 drop(external);
18274 let editor = crate::repo_lock::SeatLease::for_checkout(
18275 tmp.path(),
18276 true,
18277 crate::config::test_coordination().as_ref(),
18278 )
18279 .await
18280 .unwrap();
18281 let pause = Pause::new();
18282 let interrupt = Pause::new();
18283 let prompts = Prompts::default();
18284 let ctx = WaveCtx {
18285 run: "r",
18286 node: "review",
18287 prompts: &prompts,
18288 cache: None,
18289 round: None,
18290 carry_seats: false,
18291 pauses: Some((&pause, &interrupt)),
18292 };
18293 let (results, ()) = tokio::join!(
18294 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18295 async {
18296 tokio::time::sleep(Duration::from_millis(50)).await;
18297 pause.park();
18298 }
18299 );
18300 assert!(
18301 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18302 );
18303 assert_eq!(results[0].1.turns, 0);
18304 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18305 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18306 drop(editor);
18307 }
18308
18309 #[tokio::test]
18310 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18311 crate::run::pin_test_home();
18312 let tmp = tempfile::tempdir().unwrap();
18313 let cache = tmp.path().join("target-b");
18314 let _held = hold(&cache);
18315 let mut st = state(1);
18316 let results = run(
18317 vec![job(
18318 sh("alpha", "echo ok"),
18319 tmp.path(),
18320 Duration::from_secs(1),
18321 )],
18322 Arc::new(Semaphore::new(1)),
18323 Some(&cache),
18324 &mut st,
18325 )
18326 .await;
18327 let (_, seat, out) = &results[0];
18328 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18329 assert!(wave_blocked(&results));
18330 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18331 assert_eq!(seat.turns, 0, "nothing was asked");
18332 assert!(
18333 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18334 "no agent was spawned"
18335 );
18336 assert!(FailClass::of(out).is_none(), "not a failure class");
18337 let block = st.resource_block.as_ref().expect("recorded for the resume");
18338 assert_eq!(block.kind, "shared build cache");
18339 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18340 assert!(st.active.is_empty());
18341 }
18342
18343 #[tokio::test]
18344 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18345 crate::run::pin_test_home();
18346 let tmp = tempfile::tempdir().unwrap();
18347 let mut st = state(1);
18348 let results = run(
18349 vec![job(
18350 sh("alpha", "echo ok"),
18351 tmp.path(),
18352 Duration::from_secs(1),
18353 )],
18354 Arc::new(Semaphore::new(0)),
18355 None,
18356 &mut st,
18357 )
18358 .await;
18359 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18360 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18361 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18362 }
18363
18364 #[tokio::test]
18365 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18366 crate::run::pin_test_home();
18367 let tmp = tempfile::tempdir().unwrap();
18368 let mut st = state(60);
18369 let results = run(
18370 vec![job(
18371 sh("alpha", "sleep 30"),
18372 tmp.path(),
18373 Duration::from_secs(1),
18374 )],
18375 Arc::new(Semaphore::new(1)),
18376 None,
18377 &mut st,
18378 )
18379 .await;
18380 let out = &results[0].2;
18381 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18382 let class = FailClass::of(out).expect("a failure");
18383 assert_eq!(class, FailClass::Timeout);
18384 assert!(should_hand_over(None, &class), "existing handover policy");
18385 assert!(st.resource_block.is_none());
18386 }
18387}