1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(
488 repo: &Path,
489 base_branch: &str,
490 remote: &str,
491 coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493 let _lease = crate::repo_lock::SeatLease::operation_with(
494 repo,
495 coordination,
496 &crate::repo_lock::WaitPolicy::current(),
497 )
498 .await?;
499 let tracking = format!("{remote}/{base_branch}");
500 let fetched = git::fetch(repo, remote, base_branch).await;
501 if let Ok(out) = &fetched
502 && out.ok()
503 && git::rev_exists(repo, &tracking).await
504 {
505 return Ok(crate::run::FetchedBase {
506 remote: remote.to_owned(),
507 branch: base_branch.to_owned(),
508 commit: git::rev_parse(repo, &tracking).await?,
509 fetched_at: jiff::Timestamp::now(),
510 });
511 }
512 let why = match &fetched {
513 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514 Ok(_) => format!("{remote} has no {base_branch}"),
515 Err(e) => e.to_string(),
516 };
517 bail!(
520 "cannot read {tracking} ({why}); refusing to branch off the local \
521 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522 base / remote in magi.toml. This is launcher preparation: check the \
523 launcher's Git administration access, network and repository coordination \
524 before retrying; an unattended seat cannot approve those permissions"
525 )
526}
527
528struct FixClaim {
545 path: PathBuf,
546}
547
548impl FixClaim {
549 fn acquire(dir: &Path) -> Result<Self> {
550 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551 let path = dir.join("fix.lock");
552 match Self::create(&path) {
553 Ok(claim) => Ok(claim),
554 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555 if Self::reclaim_if_dead(&path) {
556 Self::create(&path).with_context(|| format!("lock {}", path.display()))
557 } else {
558 bail!(
559 "another repair or `magi fix` is already running for this run ({} exists)",
560 path.display()
561 )
562 }
563 }
564 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565 }
566 }
567
568 fn create(path: &Path) -> std::io::Result<Self> {
569 let mut f = std::fs::OpenOptions::new()
570 .write(true)
571 .create_new(true)
572 .open(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
627 repo: &Path,
628 instruction: String,
629 also_scan: &str,
630 config: Config,
631 origin: Origin,
632 ) -> Result<Self> {
633 let repo = git::toplevel(repo).await?;
634 let missing = agent::missing_programs(&config.agents);
635 if !missing.is_empty() {
636 bail!(
637 "these agent programs are not on PATH: {}. Fix the roster in \
638 magi.toml or install them.",
639 missing.join(", ")
640 );
641 }
642 let base_branch =
643 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644 .await?;
645 let fetched_base = resolve_base(
646 &repo,
647 &base_branch,
648 &config.merge.remote,
649 config.coordination.as_ref(),
650 )
651 .await?;
652 let base_commit = fetched_base.commit.clone();
653 let roles = config.resolve_roles()?;
654 let max_parallel = config.graph.max_parallel.max(1);
655 let seeds = refs::resolve(
658 &repo,
659 &base_commit,
660 &config.merge.remote,
661 &format!("{also_scan}\n{instruction}"),
662 )
663 .await;
664 refs::plan(&repo, &seeds).await?;
665 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666 state.fetched_base = Some(fetched_base);
670 state.origin = Some(origin);
671 for seed in &seeds {
672 state.event(
673 "seed",
674 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675 );
676 }
677 state.seeds = seeds;
678 state.event("start", format!("run {} created", state.id));
679 state.save()?;
680 Ok(Self {
681 state,
682 roles,
683 sem: Arc::new(Semaphore::new(max_parallel)),
684 pause: Pause::new(),
685 interrupt: Pause::new(),
686 })
687 }
688
689 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703 Self::review_taking_over(repo, branch, config, None, origin).await
704 }
705
706 pub async fn review_taking_over(
713 repo: &Path,
714 branch: &str,
715 config: Config,
716 takeover: Option<crate::handover::Takeover>,
717 origin: Origin,
718 ) -> Result<Self> {
719 let repo = git::toplevel(repo).await?;
720 let missing = agent::missing_programs(&config.agents);
721 if !missing.is_empty() {
722 bail!(
723 "these agent programs are not on PATH: {}. Fix the roster in \
724 magi.toml or install them.",
725 missing.join(", ")
726 );
727 }
728 let base_branch =
729 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730 .await?;
731 if base_branch == branch {
732 bail!("`{branch}` is the base branch; there is nothing to review against");
733 }
734 let fetched_base = resolve_base(
735 &repo,
736 &base_branch,
737 &config.merge.remote,
738 config.coordination.as_ref(),
739 )
740 .await?;
741 let base_commit = fetched_base.commit.clone();
742
743 let roles = config.resolve_roles()?;
744 let max_parallel = config.graph.max_parallel.max(1);
745 let mut state = RunState::new(
746 repo.clone(),
747 base_branch,
748 base_commit.clone(),
749 String::new(),
750 config,
751 );
752 state.fetched_base = Some(fetched_base);
753 state.origin = Some(origin);
754
755 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762 earlier: Vec::new(),
763 home: crate::run::home(),
764 choice: None,
765 task_instruction: None,
766 });
767 let wait = repository_wait(&state, None);
768 let coordination_config = state.config.coordination.clone();
769 let coordination =
770 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771 .await?;
772 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773 if let Some(released) = &released {
774 state.event(
775 "release",
776 format!(
777 "took `{branch}` over from run {}: its worktree was released: {}",
778 crate::run::short_of(&released.old_id),
779 released.audit
780 ),
781 );
782 }
783 if let Some(choice) = takeover.choice.as_ref()
787 && let Err(e) =
788 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789 .await
790 {
791 if let Some(released) = &released {
792 released.restore(&repo, branch).await;
793 }
794 return Err(e.context("applying the owner's answer about the diverged branch"));
795 }
796 for id in takeover.earlier.iter().rev() {
802 if let Ok(old) = RunState::load_under(id, &takeover.home)
803 && let Some(head) = old.pr_head.clone()
804 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805 {
806 state.pr_head = Some(head);
807 break;
808 }
809 }
810 drop(coordination);
811 let opened = Self::open_review(
812 &repo,
813 branch,
814 state,
815 roles,
816 max_parallel,
817 base_commit,
818 &takeover,
819 )
820 .await;
821 if opened.is_err()
822 && let Some(released) = &released
823 {
824 let _coordination = crate::repo_lock::SeatLease::operation_with(
825 &repo,
826 coordination_config.as_ref(),
827 &wait,
828 )
829 .await?;
830 released.restore(&repo, branch).await;
831 }
832 opened
833 }
834
835 pub async fn recover_pr(
837 repo: &Path,
838 task: &str,
839 selection: crate::pr_recovery::Selection,
840 mut config: Config,
841 ) -> Result<Self> {
842 selection.configure(&mut config)?;
843 selection.check_task(task, &selection.instruction)?;
844 let base_branch =
845 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
846 .await?;
847 let fetched_base = resolve_base(
848 repo,
849 &base_branch,
850 &config.merge.remote,
851 config.coordination.as_ref(),
852 )
853 .await?;
854 let base_commit = fetched_base.commit.clone();
855 selection
856 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
857 .await?;
858 selection.verify_source(repo).await?;
859 let roles = config.resolve_roles()?;
860 let max_parallel = config.graph.max_parallel.max(1);
861 let mut state = RunState::new(
862 repo.to_path_buf(),
863 base_branch,
864 base_commit.clone(),
865 selection.instruction.clone(),
866 config,
867 );
868 state.fetched_base = Some(fetched_base);
869 state.origin = Some(crate::run::Origin::queue(task));
870 state.pr_head = Some(selection.branch.clone());
871 state.pr_recovery = Some(selection.clone());
872 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
873 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
874 }
875
876 async fn open_selected(
877 repo: &Path,
878 state: RunState,
879 roles: ResolvedRoles,
880 max_parallel: usize,
881 base_commit: String,
882 selection: crate::pr_recovery::Selection,
883 ) -> Result<Self> {
884 let mut state = state;
885 let branch = state.branch_for('A');
886 let wait = repository_wait(&state, None);
887 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
888 git::git(repo, &["branch", &branch, selection.source()]).await?;
889 drop(coordination);
890 state.event(
891 "recovery",
892 format!(
893 "selected implementation {} for {} (public lease {})",
894 selection.source(),
895 selection.url,
896 selection.head
897 ),
898 );
899 let takeover = crate::handover::Takeover {
900 earlier: Vec::new(),
901 home: crate::run::home(),
902 choice: None,
903 task_instruction: Some(selection.instruction.clone()),
904 };
905 state.save()?;
906 let id = state.id.clone();
907 let opened = Self::open_review(
908 repo,
909 &branch,
910 state,
911 roles,
912 max_parallel,
913 base_commit,
914 &takeover,
915 )
916 .await;
917 match opened {
918 Ok(runner) => Ok(runner),
919 Err(e) => {
920 if let Ok(mut partial) = RunState::load(&id) {
921 partial.status = RunStatus::Blocked;
922 partial.driver_exited = true;
923 partial.event(
924 "recovery",
925 format!("startup failed; retained source and candidate {branch}: {e:#}"),
926 );
927 partial.save()?;
928 }
929 Err(crate::pr_recovery::StartupFailed {
930 run: id,
931 cause: format!("{e:#}"),
932 }
933 .into())
934 }
935 }
936 }
937
938 async fn inherit(
943 repo: &Path,
944 branch: &str,
945 base_commit: &str,
946 takeover: &crate::handover::Takeover,
947 ) -> Option<crate::recovery::RecoveryRecord> {
948 use crate::recovery::{Relation, assess};
949 let newest = takeover.earlier.last()?;
950 let mut chosen = None;
951 for id in takeover.earlier.iter().rev() {
952 if let Ok(old) = RunState::load_under(id, &takeover.home)
953 && !old.reviews.is_empty()
954 && (old.winner().is_some_and(|w| w.branch == branch)
955 || old.pr_head.as_deref() == Some(branch))
956 {
957 chosen = Some(old);
958 break;
959 }
960 }
961 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
962 .await
963 .unwrap_or_default();
964 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
965 Some(last) if last.head == tip => Relation::Same,
966 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
967 _ => Relation::Diverged,
968 };
969 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
970 Some(assess(from, chosen.as_ref(), base_commit, relation))
971 }
972
973 async fn open_review(
976 repo: &Path,
977 branch: &str,
978 mut state: RunState,
979 roles: ResolvedRoles,
980 max_parallel: usize,
981 base_commit: String,
982 takeover: &crate::handover::Takeover,
983 ) -> Result<Self> {
984 let wait = repository_wait(&state, None);
985 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
986 if state.pr_recovery.is_none() {
987 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
988 }
989 let start = review_base(
992 repo,
993 &state.config.merge.remote,
994 &state.base_branch,
995 &base_commit,
996 branch,
997 )
998 .await;
999 let log = git::log_oneline(repo, &start, branch)
1000 .await
1001 .unwrap_or_default();
1002 let log_text = if log.trim().is_empty() {
1003 "(no commit messages)"
1004 } else {
1005 log.trim()
1006 };
1007 let mut instruction = format!(
1008 "Review the work already on branch `{branch}`. There is no task \
1009 statement: what the change claims to do is whatever its commits \
1010 say.\n\n{log_text}"
1011 );
1012 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1016 if let Some(task) = takeover
1017 .task_instruction
1018 .as_deref()
1019 .filter(|t| !t.trim().is_empty())
1020 && rec.reviewed_head.is_some()
1021 {
1022 instruction = format!(
1023 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1024 task.trim()
1025 );
1026 rec.criteria_inherited = true;
1027 }
1028 state.event("recovery", rec.summary());
1029 state.recovery = Some(rec);
1030 }
1031 if let Some(selected) = &state.pr_recovery {
1032 instruction = selected.instruction.clone();
1033 }
1034 state.task_instruction = takeover.task_instruction.clone();
1035 state.instruction = instruction;
1036 state.reviewed_commits = Some(
1037 git::subjects(repo, &start, branch)
1038 .await
1039 .unwrap_or_default(),
1040 );
1041
1042 let worktree = state.worktree_root().join("under-review");
1045 if let Some(parent) = worktree.parent() {
1046 tokio::fs::create_dir_all(parent).await.ok();
1047 }
1048 let path = worktree.to_string_lossy().to_string();
1049 git::git(repo, &["worktree", "add", &path, branch])
1050 .await
1051 .with_context(|| {
1052 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1053 })?;
1054
1055 let setup = match ensure_setup_config(&mut state, repo).await {
1056 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1057 Err(e) => Err(e),
1058 };
1059 if let Err(e) = setup {
1060 git::worktree_remove_clean(repo, &worktree).await.ok();
1061 return Err(e);
1062 }
1063 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1064 .await
1065 .unwrap_or(0);
1066 if commits == 0 {
1067 git::worktree_remove_clean(repo, &worktree).await.ok();
1068 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1069 }
1070 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1071 .await
1072 .map(|f| f.len())
1073 .unwrap_or(0);
1074 if files == 0
1075 && let (Ok(head_tree), Ok(base_tree)) = (
1076 git::tree_of(&worktree, "HEAD").await,
1077 git::tree_of(&worktree, &base_commit).await,
1078 )
1079 && head_tree == base_tree
1080 {
1081 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!(
1084 "`{branch}` at {} has a tree identical to base {}; this usually means \
1085 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1086 against `{}/{branch}`) rather than an empty change",
1087 short(&head),
1088 short(&base_commit),
1089 state.config.merge.remote
1090 );
1091 }
1092 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1093 .await
1094 .unwrap_or_default();
1095
1096 state.candidates.push(Candidate {
1097 index: 0,
1098 label: 'A',
1099 agent: EXISTING_BRANCH.to_owned(),
1102 branch: branch.to_owned(),
1103 worktree,
1104 summary: String::new(),
1105 stat,
1106 files,
1107 commits,
1108 empty: false,
1109 failed: None,
1110 verified_noop: None,
1111 duration_ms: 0,
1112 folded: false,
1113 });
1114 state.tally = Some(Tally {
1115 first_choice: BTreeMap::from([('A', 0)]),
1116 borda: BTreeMap::new(),
1117 winner: 'A',
1118 rankings: 0,
1119 unanimous_initial: false,
1120 deliberated: false,
1121 changed_votes: 0,
1122 unanimous_final: false,
1123 tie_break: None,
1124 judges: 0,
1128 present: 0,
1129 quorum: 0,
1130 met_quorum: true,
1131 uncontested: Some("review-only run: nothing competed".to_owned()),
1132 });
1133 state.status = RunStatus::Reviewing;
1134 state.prep_complete = Some(true);
1135 state.event(
1136 "start",
1137 format!(
1138 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1139 state.id
1140 ),
1141 );
1142 state.save()?;
1143 Ok(Self {
1144 state,
1145 roles,
1146 sem: Arc::new(Semaphore::new(max_parallel)),
1147 pause: Pause::new(),
1148 interrupt: Pause::new(),
1149 })
1150 }
1151
1152 pub fn resume(id: &str) -> Result<Self> {
1154 let state = RunState::load(id)?;
1155 crate::pr_head::ensure_owner(&state)?;
1156 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1157 bail!(
1158 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1159 );
1160 }
1161 if let Some(to) = &state.released_to {
1162 bail!(
1163 "run {} cannot be resumed: its worktree was released to run {}",
1164 state.short(),
1165 crate::run::short_of(to)
1166 );
1167 }
1168 let roles = state.config.resolve_roles()?;
1169 if let Some(id) = &state.config.graph.selected_implementer
1170 && state.candidates.iter().any(|c| c.agent != *id)
1171 {
1172 bail!(
1173 "replacement implementer pin does not match the saved candidates; task must stay held"
1174 );
1175 }
1176 let max_parallel = state.config.graph.max_parallel.max(1);
1177 Ok(Self {
1178 state,
1179 roles,
1180 sem: Arc::new(Semaphore::new(max_parallel)),
1181 pause: Pause::new(),
1182 interrupt: Pause::new(),
1183 })
1184 }
1185
1186 pub async fn execute(&mut self) -> Result<()> {
1193 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1196 || self.state.land_approval.is_some()
1197 || self.state.merge_decision.is_some()
1198 || !self.state.merge_repairs.is_empty()
1199 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1200 {
1201 let claim = FixClaim::acquire(&self.state.dir())?;
1202 let disk = RunState::load(&self.state.id)?;
1203 let checkpoint = |s: &RunState| {
1204 (
1205 s.merge_repairs.len(),
1206 s.merge_repairs
1207 .last()
1208 .map(|r| (r.question.clone(), r.phase)),
1209 s.land_approval
1210 .as_ref()
1211 .map(|a| (a.question.clone(), a.head.clone())),
1212 )
1213 };
1214 if checkpoint(&disk) != checkpoint(&self.state) {
1215 bail!(
1216 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1217 );
1218 }
1219 Some(claim)
1220 } else {
1221 None
1222 };
1223 let mut execution = self.claim_driver()?;
1224 let runner = &mut *execution.runner;
1225 let wait = runner.repository_wait();
1226 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1229 let result = match result {
1230 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1231 record_repository_block(&mut runner.state, &e);
1232 runner.park_here()?;
1233 Ok(())
1234 }
1235 other => other,
1236 };
1237 execution.driver.finish(&mut runner.state)?;
1238 let ended = if result.is_err() {
1239 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1240 } else {
1241 crate::notices::run_ended(&runner.state)
1242 };
1243 if let Some(notice) = ended {
1244 crate::notices::raise(notice);
1245 }
1246 result
1247 }
1248
1249 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1250 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1251 }
1252
1253 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1254 let wait = self.repository_wait();
1255 coordinate_operation(&mut self.state, cwd, &wait).await
1256 }
1257
1258 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1260 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1261 let disk = RunState::load(&self.state.id)?;
1262 crate::pr_head::ensure_owner(&disk)?;
1263 if disk.liveness(false) == crate::run::Liveness::Live {
1264 bail!("run already has a live driver; wait for its execution to end");
1265 }
1266 if let Some(to) = &disk.released_to {
1267 bail!(
1268 "run {} cannot continue: its worktree was released to run {}",
1269 self.state.short(),
1270 crate::run::short_of(to)
1271 );
1272 }
1273 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1274 let mut execution = DriverExecution {
1275 runner: self,
1276 driver,
1277 };
1278 let _owner = owner;
1281 let state = &mut execution.runner.state;
1282 state.parked = false;
1287 state.resource_wait = None;
1288 state.resource_block = None;
1289 state.clear_active();
1296 let pid = std::process::id();
1311 state.driver_pid = Some(pid);
1312 state.driver_started_at = crate::proc::process_started_at(pid);
1313 state.driver_exited = false;
1314 state.begin_pass();
1315 execution.driver.bind(state);
1316 state.save()?;
1317 execution.driver.record(state)?;
1318 Ok(execution)
1319 }
1320
1321 async fn execute_graph(&mut self) -> Result<()> {
1322 self.state.configure_recovery_resume()?;
1323 self.persist_candidates().await?;
1326 if self
1327 .state
1328 .merge_decision
1329 .as_ref()
1330 .is_some_and(|decision| !decision.closed)
1331 && self.state.status != RunStatus::Landing
1332 {
1333 if crate::merge_decision::resumable_hold(&self.state) {
1334 self.state.merge = Some(MergeOutcome {
1335 mode: MergeMode::Pr,
1336 ok: false,
1337 detail: self
1338 .state
1339 .merge_decision
1340 .as_ref()
1341 .unwrap()
1342 .target
1343 .pr
1344 .clone(),
1345 empty: false,
1346 });
1347 self.run_land().await?;
1348 self.settle_questions();
1349 return Ok(());
1350 }
1351 bail!(
1352 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1353 );
1354 }
1355 if self.state.config.graph.selected_implementer.is_some()
1356 && self.state.status == RunStatus::Blocked
1357 && self.state.tally.is_none()
1358 && self.state.candidate_preservation.is_empty()
1359 {
1360 bail!(
1361 "selected implementation is held; explicitly release the owning task to retry it"
1362 );
1363 }
1364
1365 if self.state.merge_repairs.last().is_some_and(|r| {
1366 matches!(
1367 r.phase,
1368 crate::run::MergeRepairPhase::Requested
1369 | crate::run::MergeRepairPhase::Fixing
1370 | crate::run::MergeRepairPhase::Reviewing
1371 )
1372 }) {
1373 if self.repair_merge_review().await? {
1374 self.run_land().await?;
1375 }
1376 self.settle_questions();
1377 return Ok(());
1378 }
1379 if self.state.pr_publication_pending {
1382 self.merge().await?;
1383 if self.resource_halt()? {
1384 return Ok(());
1385 }
1386 self.state.save()?;
1387 self.settle_questions();
1388 return Ok(());
1389 }
1390 if self.state.status == RunStatus::Stalled {
1403 if self.recover_stall().await? {
1404 self.finish_after_tally().await?;
1405 } else {
1406 self.state.save()?;
1408 }
1409 return Ok(());
1410 }
1411 if self.state.status == RunStatus::Landing {
1421 self.run_land().await?;
1422 self.settle_questions();
1427 return Ok(());
1428 }
1429 if self.state.github_text.is_some()
1434 && self.state.merge.is_none()
1435 && self.state.status == RunStatus::Gating
1436 {
1437 self.merge().await?;
1438 if self.resource_halt()? {
1439 return Ok(());
1440 }
1441 self.state.save()?;
1442 return Ok(());
1443 }
1444 if let Err(e) = self.prep().await {
1445 if e.is::<crate::repo_lock::ResourceBlocked>() {
1446 return Err(e);
1447 }
1448 self.state.status = RunStatus::Blocked;
1449 self.state
1450 .prep_failure
1451 .get_or_insert_with(|| format!("{e:#}"));
1452 self.state
1453 .event("prep", format!("preparation stopped: {e:#}"));
1454 self.state.save()?;
1455 return Err(e);
1456 }
1457 if self.park_here()? {
1458 return Ok(());
1459 }
1460 self.advise().await?;
1461 if self.park_here()? {
1462 return Ok(());
1463 }
1464 self.implement().await?;
1465 if self.park_here()? {
1466 return Ok(());
1467 }
1468 if self.state.status == RunStatus::VerifiedNoop {
1472 return Ok(());
1473 }
1474 self.judge().await?;
1475 if self.park_here()? {
1476 return Ok(());
1477 }
1478 self.deliberate().await?;
1479 if self.park_here()? {
1480 return Ok(());
1481 }
1482 self.vote().await?;
1483 if self.park_here()? {
1484 return Ok(());
1485 }
1486 self.tally()?;
1487 if self.state.status == RunStatus::Stalled {
1492 self.state.save()?;
1496 return Ok(());
1497 }
1498 self.finish_after_tally().await?;
1499 Ok(())
1500 }
1501
1502 fn stop_for_resource(&mut self) -> Result<()> {
1507 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1509 {
1510 self.state.status = RunStatus::Blocked;
1511 }
1512 self.state.save()
1513 }
1514
1515 fn resource_halt(&mut self) -> Result<bool> {
1517 if self.state.resource_block.is_none() {
1518 return Ok(false);
1519 }
1520 self.stop_for_resource()?;
1521 Ok(true)
1522 }
1523
1524 fn park_here(&mut self) -> Result<bool> {
1531 let parking = self.pause.parked() || self.interrupt.parked();
1532 if !parking && self.state.resource_block.is_some() {
1533 self.stop_for_resource()?;
1534 return Ok(true);
1535 }
1536 if !self.pause.parked() && !self.interrupt.parked() {
1541 return Ok(false);
1542 }
1543 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1544 let why = match cause.as_ref().map(ParkCause::label) {
1545 Some(reason) => format!(
1546 "parked after `{}` ({reason}) — resume to carry on from here",
1547 self.state.status.as_str()
1548 ),
1549 None => format!(
1550 "parked after `{}` — resume to carry on from here",
1551 self.state.status.as_str()
1552 ),
1553 };
1554 self.state.event("park", why);
1555 self.state
1556 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1557 self.state.parked = true;
1558 self.state.save()?;
1559 Ok(true)
1560 }
1561
1562 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1564 if self.state.prep_failure.is_none() {
1565 return Ok(());
1566 }
1567 let failed_at = self
1568 .state
1569 .events
1570 .iter()
1571 .rev()
1572 .find(|event| event.node == "prep")
1573 .map_or(self.state.updated_at, |event| event.at);
1574 anyhow::ensure!(
1575 approval_at > failed_at,
1576 "preparation retry requires a new explicit release after the failure"
1577 );
1578 self.state.prep_failure = None;
1579 self.state.prep_complete = Some(false);
1580 self.state
1581 .event("prep-retry", "operator approved retry of preparation");
1582 Ok(())
1583 }
1584
1585 pub fn on_pause(&mut self, pause: Pause) {
1587 self.pause = pause;
1588 }
1589
1590 pub fn watch_interrupt(&mut self, pause: Pause) {
1596 self.interrupt = pause;
1597 }
1598
1599 fn settle_questions(&mut self) {
1619 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1620 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1621 }
1622 }
1623
1624 async fn finish_after_tally(&mut self) -> Result<()> {
1627 self.graft().await?;
1630 if self.resource_halt()? {
1631 return Ok(());
1632 }
1633 self.fold_losers().await?;
1634 self.sync_to_base().await?;
1639 if self.state.status == RunStatus::AlreadyInBase {
1640 return Ok(());
1641 }
1642 self.review_loop().await?;
1643 if self.resource_halt()? {
1644 return Ok(());
1645 }
1646 self.sync_to_base().await?;
1647 if self.state.status == RunStatus::AlreadyInBase {
1648 return Ok(());
1649 }
1650 self.gate().await?;
1651 self.merge().await?;
1652 if self.resource_halt()? {
1653 return Ok(());
1654 }
1655 self.state.save()?;
1656 Ok(())
1657 }
1658
1659 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1666 let result = match ensure_setup_config(&mut self.state, repo).await {
1667 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1668 Err(e) => Err(e),
1669 };
1670 if let Err(e) = result {
1671 git::worktree_remove_clean(repo, wt).await.ok();
1672 if in_prep {
1673 self.rollback_prep(repo).await;
1674 }
1675 self.state.status = RunStatus::Blocked;
1676 self.state.event(
1677 "setup",
1678 format!("worktree setup failed in {}: {e:#}", wt.display()),
1679 );
1680 self.state.save()?;
1681 return Err(e);
1682 }
1683 Ok(())
1684 }
1685
1686 async fn rollback_prep(&mut self, repo: &Path) {
1688 let root = self.state.worktree_root();
1689 for c in std::mem::take(&mut self.state.candidates) {
1690 if matches!(
1691 git::worktree_remove_clean(repo, &c.worktree).await,
1692 Ok(true)
1693 ) {
1694 git::branch_delete(repo, &c.branch).await.ok();
1695 } else {
1696 self.state.candidates.push(c);
1697 }
1698 }
1699 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1700 for j in 1..=seats {
1701 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1702 .await
1703 .ok();
1704 }
1705 for k in 1..=self.state.config.graph.advisors {
1706 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1707 .await
1708 .ok();
1709 }
1710 }
1711
1712 async fn prep(&mut self) -> Result<()> {
1713 if let Some(reason) = &self.state.prep_failure {
1714 bail!(
1715 "candidate preparation previously failed: {reason}; correct the source before retrying"
1716 );
1717 }
1718 if self
1719 .state
1720 .prep_complete
1721 .unwrap_or(!self.state.candidates.is_empty())
1722 {
1723 return Ok(());
1724 }
1725 let repo = self.state.repo.clone();
1726 let _coordination = self.coordinate(&repo).await?;
1729 self.rollback_prep(&repo).await;
1730 if !self.state.candidates.is_empty() {
1731 bail!(
1732 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1733 );
1734 }
1735 self.state.prep_complete = Some(false);
1736 self.state.status = RunStatus::Prep;
1737 self.state.save()?;
1738 let repo = self.state.repo.clone();
1739 let base = self.state.base_commit.clone();
1740 let plan = refs::plan(&repo, &self.state.seeds).await?;
1741 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1742 let root = self.state.worktree_root();
1743 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1744
1745 let hooks_dir = self.state.dir().join("hooks");
1748 if self.state.config.blind.commit_msg_hook {
1749 std::fs::create_dir_all(&hooks_dir)
1750 .with_context(|| format!("create {}", hooks_dir.display()))?;
1751 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1752 let path = hooks_dir.join("commit-msg");
1753 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1754 make_executable(&path)?;
1755 git::acquire_worktree_config(&repo).await?;
1763 self.state.enabled_worktree_config = true;
1764 }
1765
1766 ensure_setup_config(&mut self.state, &repo).await?;
1769
1770 for (index, (spec, label)) in self
1771 .roles
1772 .implementers
1773 .clone()
1774 .into_iter()
1775 .zip(labels)
1776 .enumerate()
1777 {
1778 let branch = self.state.branch_for(label);
1779 let worktree = root.join(format!("cand-{label}"));
1780 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1781 if self.state.config.blind.commit_msg_hook {
1782 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1783 }
1784 git::local_exclude(&worktree, "/.magi/").await?;
1785 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1786 git::worktree_remove_clean(&repo, &worktree).await.ok();
1789 git::branch_delete(&repo, &branch).await.ok();
1790 self.rollback_prep(&repo).await;
1791 self.state.status = RunStatus::Blocked;
1792 self.state
1793 .event("prep", format!("worktree setup failed: {e:#}"));
1794 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1795 self.state.save()?;
1796 return Err(e);
1797 }
1798 self.state.candidates.push(Candidate {
1799 index,
1800 label,
1801 agent: spec.id.clone(),
1802 branch,
1803 worktree,
1804 summary: String::new(),
1805 stat: String::new(),
1806 files: 0,
1807 commits: 0,
1808 empty: false,
1809 failed: None,
1810 verified_noop: None,
1811 duration_ms: 0,
1812 folded: false,
1813 });
1814 self.state.save()?;
1815 for pick in &plan.picks {
1816 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1817 {
1818 self.state.status = RunStatus::Blocked;
1819 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1820 self.state
1821 .event("prep", format!("cannot apply referenced commit: {e}"));
1822 self.state.save()?;
1823 return Err(e);
1824 }
1825 }
1826 }
1827
1828 for j in 1..=self.roles.judges.len() {
1829 let wt = root.join(format!("judge-{j}"));
1830 if !wt.exists() {
1831 git::worktree_add_detached(&repo, &wt, &base).await?;
1832 self.setup_seat_worktree(&repo, &wt, true).await?;
1833 }
1834 }
1835
1836 if self.state.config.graph.advise {
1844 for k in 1..=self.state.config.graph.advisors {
1845 let wt = root.join(format!("advisor-{k}"));
1846 if !wt.exists() {
1847 git::worktree_add_detached(&repo, &wt, &base).await?;
1848 self.setup_seat_worktree(&repo, &wt, true).await?;
1849 }
1850 }
1851 }
1852
1853 let authors: Vec<&str> = self
1858 .roles
1859 .implementers
1860 .iter()
1861 .map(|a| a.id.as_str())
1862 .collect();
1863 let overlap: Vec<String> = self
1864 .roles
1865 .judges
1866 .iter()
1867 .enumerate()
1868 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1869 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1870 .collect();
1871 if !overlap.is_empty() {
1872 let note = format!(
1873 "{} also authored a candidate; blind, but the panel is less \
1874 independent than {} distinct agents would be",
1875 overlap.join(", "),
1876 self.roles.judges.len()
1877 );
1878 self.state.event("prep", note);
1879 }
1880
1881 self.state.event(
1882 "prep",
1883 format!(
1884 "{} candidates, {} judges, base {} ({})",
1885 self.state.candidates.len(),
1886 self.roles.judges.len(),
1887 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1888 self.state.base_branch
1889 ),
1890 );
1891 self.state.prep_complete = Some(true);
1892 self.state.status = RunStatus::Implementing;
1893 self.state.save()?;
1894 Ok(())
1895 }
1896
1897 async fn advise(&mut self) -> Result<()> {
1930 let implement_untouched = self
1931 .state
1932 .candidates
1933 .iter()
1934 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1935 if !self.state.config.graph.advise || self.state.advise_attempted {
1936 return Ok(());
1937 }
1938 if !implement_untouched {
1939 self.state.event(
1940 "advise",
1941 "skipping the design-deliberation stage: at least one \
1942 candidate already shows implementation progress, so this \
1943 run is past the point the stage exists to run before"
1944 .to_owned(),
1945 );
1946 self.state.advise_attempted = true;
1947 self.state.save()?;
1948 return Ok(());
1949 }
1950 let run_id = self.state.id.clone();
1951 let prompts = self.state.config.prompts.clone();
1952 let instruction = self.state.instruction.clone();
1953 let language = self.state.config.graph.language.clone();
1954 let root = self.state.worktree_root();
1955 let n = self.state.config.graph.advisors;
1956 let where_recorded = self.state.dir().join("run.json");
1957
1958 let seats = match self.state.config.advisors() {
1959 Ok(seats) if !seats.is_empty() => seats,
1960 Ok(_) => {
1961 self.state.event(
1962 "advise",
1963 format!(
1964 "[graph] advisors is 0; skipping the design-deliberation \
1965 stage and continuing without a synthesis brief (see {})",
1966 where_recorded.display()
1967 ),
1968 );
1969 self.state.advise_attempted = true;
1970 self.state.save()?;
1971 return Ok(());
1972 }
1973 Err(e) => {
1974 self.state.event(
1975 "advise",
1976 format!(
1977 "could not resolve advisor seats ({e:#}); continuing \
1978 without a design-deliberation brief (see {})",
1979 where_recorded.display()
1980 ),
1981 );
1982 self.state.advise_attempted = true;
1983 self.state.save()?;
1984 return Ok(());
1985 }
1986 };
1987
1988 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1989 let artifacts = agent::artifacts_dir(&self.state.dir());
1990 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1991
1992 let mut jobs = Vec::new();
1993 for (i, spec) in seats.iter().cloned().enumerate() {
1994 let seat_key = format!("advisor-{}", i + 1);
1995 let seat = self.seat(&seat_key, &spec.id);
1996 jobs.push(SeatJob {
1997 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1998 spec,
1999 seat,
2000 cwd: worktrees[i % worktrees.len()].clone(),
2001 timeout,
2002 allow_write: false,
2003 sessions: false,
2004 artifacts: artifacts.clone(),
2005 stem: seat_key,
2006 handover: None,
2007 });
2008 }
2009
2010 self.state.event(
2011 "advise",
2012 format!(
2013 "{} advisor seat(s) sketching a design in parallel",
2014 jobs.len()
2015 ),
2016 );
2017 let mut quota_losses = Vec::new();
2018 let cache = self.state.config.cache_dir();
2019 let ctx = WaveCtx {
2020 carry_seats: false,
2021 pauses: Some((&self.pause, &self.interrupt)),
2022 run: &run_id,
2023 node: "advise",
2024 prompts: &prompts,
2025 cache: cache.as_deref(),
2026 round: None,
2027 };
2028 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2029 let results = ask_json_wave::<Proposal>(
2030 jobs,
2031 Arc::clone(&self.sem),
2032 self.state.config.graph.retries,
2033 &advisor_roster,
2034 &ctx,
2035 &mut quota_losses,
2036 &mut self.state,
2037 &|p: &Proposal| p.validate(),
2038 )
2039 .await;
2040 if self.state.resource_block.is_some() {
2041 return self.stop_for_resource();
2042 }
2043 self.state.quota.extend(quota_losses);
2044
2045 let mut records = Vec::with_capacity(results.len());
2046 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2047 let agent_id = seat.agent.clone();
2048 self.state.seats.insert(seat.key.clone(), seat);
2049 match res {
2050 Ok((proposal, out)) => {
2051 self.state
2052 .event("advise", format!("advisor-{} proposed a design", i + 1));
2053 records.push(advise::AdvisorRecord::proposed(
2054 i + 1,
2055 agent_id,
2056 proposal,
2057 out.duration_ms,
2058 ));
2059 }
2060 Err(e) => {
2061 self.state.event(
2062 "advise",
2063 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2064 );
2065 records.push(advise::AdvisorRecord::failed(
2066 i + 1,
2067 agent_id,
2068 e.to_string(),
2069 ));
2070 }
2071 }
2072 }
2073
2074 let mut advice = advise::Advice {
2075 records,
2076 synthesis: None,
2077 };
2078 if advice.proposals().is_empty() {
2079 self.state.event(
2080 "advise",
2081 "no advisor produced a usable proposal; continuing without a \
2082 synthesis brief"
2083 .to_owned(),
2084 );
2085 } else {
2086 match self
2087 .synthesize_brief(
2088 &advice,
2089 &instruction,
2090 &language,
2091 &worktrees[0],
2092 &artifacts,
2093 &run_id,
2094 &prompts,
2095 cache.as_deref(),
2096 )
2097 .await
2098 {
2099 Ok(Some(text)) => {
2100 self.state.event(
2101 "advise",
2102 "synthesized a design brief for the implementer".to_owned(),
2103 );
2104 advice.synthesis = Some(text);
2105 }
2106 Ok(None) => {
2107 self.state.event(
2108 "advise",
2109 "the synthesis seat produced nothing usable; continuing \
2110 without a design brief"
2111 .to_owned(),
2112 );
2113 }
2114 Err(e) => {
2115 if e.is::<crate::repo_lock::ResourceBlocked>() {
2116 return Err(e);
2117 }
2118 self.state.event(
2119 "advise",
2120 format!("could not synthesize a design brief: {e:#}"),
2121 );
2122 }
2123 }
2124 }
2125 advise::apply_reflection(&mut advice);
2126
2127 self.state.advice = Some(advice);
2128 self.state.advise_attempted = true;
2129 self.state.save()?;
2130 Ok(())
2131 }
2132
2133 #[allow(clippy::too_many_arguments)]
2145 async fn synthesize_brief(
2146 &mut self,
2147 advice: &advise::Advice,
2148 instruction: &str,
2149 language: &str,
2150 cwd: &Path,
2151 artifacts: &Path,
2152 run_id: &str,
2153 prompts: &Prompts,
2154 cache: Option<&Path>,
2155 ) -> Result<Option<String>> {
2156 let chain = agent::pick_chain(
2157 &self.state.config.agents,
2158 self.state.config.roles.synthesizer.as_ref(),
2159 &agent::installed,
2160 "synthesizer",
2161 )?;
2162 let proposals = advice.proposals();
2163 let mut prompt = prompt::with_overlay(
2164 prompt::synthesize_brief(instruction, &proposals, language),
2165 prompts.overlay("advise"),
2166 );
2167 if cache.is_some() {
2168 prompt.push('\n');
2173 prompt.push_str(&prompt::build_cache_note("advise", false));
2174 }
2175 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2176 let mut last = None;
2181 for (n, spec) in chain.iter().enumerate() {
2182 if n > 0 {
2183 self.state
2184 .event("advise", format!("synthesis falling back to {}", spec.id));
2185 }
2186 let mut seat = self.seat("advise-synthesis", &spec.id);
2187 let wait = self.repository_wait();
2188 let outcome = wait
2189 .scope(agent::invoke(
2190 spec,
2191 &mut seat,
2192 &Invocation {
2193 scope: crate::agent::InvocationScope::Repository,
2194 coordination: self.state.config.coordination.as_ref(),
2195 fetched_base: None,
2196 cwd,
2197 prompt: &prompt,
2198 timeout,
2199 allow_write: false,
2200 unsandboxed: false,
2201 sessions: false,
2202 artifacts,
2203 stem: &if n == 0 {
2204 "advise-synthesis".to_owned()
2205 } else {
2206 format!("advise-synthesis-{}", spec.id)
2207 },
2208 run: run_id,
2209 node: "advise",
2210 cache_dir: None,
2211 attachments: &[],
2212 writable: &[],
2213 },
2214 ))
2215 .await;
2216 if outcome.is_ok() {
2217 self.state.seats.insert(seat.key.clone(), seat);
2218 }
2219 let advance = agent::chain_advances(&outcome);
2220 last = Some(outcome);
2221 if !advance {
2222 break;
2223 }
2224 }
2225 let out = last.expect("a chain holds at least one agent")?;
2228 if !out.usable() {
2229 return Ok(None);
2230 }
2231 let text =
2232 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2233 Ok((!text.trim().is_empty()).then_some(text))
2234 }
2235
2236 async fn implement(&mut self) -> Result<()> {
2239 self.persist_candidates().await?;
2240 let run_id = self.state.id.clone();
2245 let prompts = self.state.config.prompts.clone();
2246 let todo: Vec<usize> = self
2247 .state
2248 .candidates
2249 .iter()
2250 .enumerate()
2251 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2252 .map(|(i, _)| i)
2253 .collect();
2254 if todo.is_empty() {
2255 return self.after_implement();
2256 }
2257 self.state.status = RunStatus::Implementing;
2258
2259 let language = self.state.config.graph.language.clone();
2260 let github_language = self.state.config.graph.github_language.clone();
2261 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2262 let sessions = self.state.config.graph.sessions;
2263 let artifacts = agent::artifacts_dir(&self.state.dir());
2264 let brief = self
2268 .state
2269 .advice
2270 .as_ref()
2271 .and_then(|a| a.synthesis.as_deref())
2272 .map(str::to_owned);
2273 let attachments = self.state.attachments.clone();
2274
2275 let mut jobs = Vec::new();
2276 for &i in &todo {
2277 let (index, label, worktree) = {
2278 let c = &self.state.candidates[i];
2279 (c.index, c.label, c.worktree.clone())
2280 };
2281 let spec = self.roles.implementers[index].clone();
2282 let seat_key = format!("impl-{label}");
2283 let seat = self.seat(&seat_key, &spec.id);
2284 let instruction = seeded_instruction(&self.state);
2285 jobs.push(SeatJob {
2286 spec,
2287 seat,
2288 prompt: prompt::implement_in(
2289 &instruction,
2290 &worktree.to_string_lossy(),
2291 &language,
2292 &github_language,
2293 brief.as_deref(),
2294 &attachments,
2295 ),
2296 cwd: worktree,
2297 timeout,
2298 allow_write: true,
2299 sessions,
2300 artifacts: artifacts.clone(),
2301 stem: format!("impl-{label}"),
2302 handover: None,
2303 });
2304 }
2305
2306 self.state.event(
2307 "implement",
2308 format!("{} candidates in parallel", jobs.len()),
2309 );
2310 let mut sent = jobs.clone();
2316 let cache = self.state.config.cache_dir();
2317 let ctx = WaveCtx {
2318 carry_seats: false,
2319 pauses: Some((&self.pause, &self.interrupt)),
2320 run: &run_id,
2321 node: "implement",
2322 prompts: &prompts,
2323 cache: cache.as_deref(),
2324 round: None,
2325 };
2326 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2327 if !wave_blocked(&results) {
2328 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2329 .await;
2330 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2331 .await?;
2332 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2333 .await;
2334 }
2335 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2336
2337 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2338 && results
2339 .iter()
2340 .any(|(_, _, out)| FailClass::of(out).is_some());
2341 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2342 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2343 continue;
2344 }
2345 let seat_key = seat.key.clone();
2346 let agent = seat.agent.clone();
2355 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2356 self.state.seats.insert(seat.key.clone(), seat);
2357 let label = self.state.candidates[i].label;
2358
2359 let (summary, duration, failed, verified_claim) = match out {
2360 AgentOutcome::Ok(o) => {
2361 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2362 let failed = (!o.usable()).then(|| {
2363 if o.timed_out {
2364 "agent timed out".to_owned()
2365 } else {
2366 format!(
2367 "agent exited with {}",
2368 crate::notices::exit_status(o.exit_code)
2369 )
2370 }
2371 });
2372 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2373 (text, o.duration_ms, failed, verified_claim)
2374 }
2375 AgentOutcome::Dropped(o) => {
2381 let why = o
2382 .dropped
2383 .as_ref()
2384 .map(|d| d.why.as_str())
2385 .unwrap_or("the CLI ended the stream without delivering its answer");
2386 (
2387 String::new(),
2388 o.duration_ms,
2389 Some(format!("the CLI dropped the stream ({why})")),
2390 None,
2391 )
2392 }
2393 AgentOutcome::Quota(o) => {
2394 self.state.quota.push(QuotaLoss {
2395 seat: seat_key,
2396 node: "implement".to_owned(),
2397 at: Timestamp::now(),
2398 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2399 });
2400 (
2401 String::new(),
2402 o.duration_ms,
2403 Some("rate limited (quota); produced no change".to_owned()),
2404 None,
2405 )
2406 }
2407 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2408 (String::new(), 0, Some(e), None)
2409 }
2410 };
2411
2412 let c = &mut self.state.candidates[i];
2415 if !exhausted_the_fallback_chain {
2416 c.agent = agent;
2417 }
2418 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2419 c.duration_ms = duration;
2420 c.failed = failed;
2421 c.verified_noop = verified_claim;
2422 self.state
2423 .candidate_preservation
2424 .insert(label, "awaiting persistence".to_owned());
2425 }
2426 self.state.save()?;
2427 self.persist_candidates().await?;
2428
2429 if resource_blocked {
2430 return self.stop_for_resource();
2431 }
2432 if selected_failed {
2433 self.state.status = RunStatus::Blocked;
2434 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2435 self.state.save()?;
2436 bail!(
2437 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2438 );
2439 }
2440 self.after_implement()
2441 }
2442
2443 async fn persist_candidates(&mut self) -> Result<()> {
2445 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2446 for label in labels {
2447 let i = self
2448 .state
2449 .candidates
2450 .iter()
2451 .position(|c| c.label == label)
2452 .context("pending candidate is missing")?;
2453 let c = self.state.candidates[i].clone();
2454 let result = async {
2455 let wait = crate::repo_lock::WaitPolicy::new(
2456 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2457 || false,
2458 );
2459 let _coordination =
2460 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2461 if let Err(e) = git::snapshot_uncommitted(
2464 &c.worktree,
2465 &self
2466 .state
2467 .dir()
2468 .join("artifacts")
2469 .join(format!("candidate-{label}-recovery")),
2470 )
2471 .await
2472 {
2473 self.state.event(
2474 "preserve",
2475 format!("candidate {label} snapshot incomplete: {e:#}"),
2476 );
2477 }
2478 let rescued = git::rescue_commit(
2479 &c.worktree,
2480 &format!("magi: candidate {label} (uncommitted work)"),
2481 )
2482 .await?;
2483 self.state.note_withheld("implement", &rescued.withheld);
2484 let base = &self.state.base_commit;
2485 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2486 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2487 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2488 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2489 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2490 if c.empty && commits > 0 && !patch.trim().is_empty()
2491 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2492 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2493 || self.state.tally.is_some())
2494 {
2495 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2499 }
2500 let c = &mut self.state.candidates[i];
2501 c.commits = commits;
2502 c.stat = stat;
2503 c.files = files;
2504 c.empty = commits == 0 || patch.trim().is_empty();
2505 if !c.empty {
2506 c.failed = None;
2507 c.verified_noop = None;
2508 }
2509 self.state.event(
2510 "preserve",
2511 format!("candidate {label}: {files} files, {commits} commits"),
2512 );
2513 Ok::<_, anyhow::Error>(())
2514 }
2515 .await;
2516 match result {
2517 Ok(()) => {
2518 self.state.candidate_preservation.remove(&label);
2519 }
2520 Err(e) => {
2521 let reason = format!("candidate {label} preservation failed: {e:#}");
2522 self.state.event("preserve", &reason);
2523 self.state.candidate_preservation.insert(label, reason);
2524 }
2525 }
2526 if self.state.candidate_preservation.is_empty()
2527 && self.state.status == RunStatus::Blocked
2528 && self.state.candidates.iter().all(|c| c.failed.is_none())
2529 {
2530 self.state.status = RunStatus::Implementing;
2534 self.state.event(
2535 "preserve",
2536 "candidate persistence recovered; implementation can continue",
2537 );
2538 }
2539 self.state.save()?;
2540 }
2541 if !self.state.candidate_preservation.is_empty() {
2542 self.state.status = RunStatus::Blocked;
2543 self.state.save()?;
2544 bail!(
2545 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2546 );
2547 }
2548 Ok(())
2549 }
2550
2551 async fn resume_undelivered(
2579 &mut self,
2580 results: &mut [(usize, SeatState, AgentOutcome)],
2581 sent: &[SeatJob],
2582 prompts: &Prompts,
2583 run_id: &str,
2584 ) {
2585 for (wi, seat, out) in results.iter_mut() {
2586 let Some(dropped) = (match &*out {
2587 AgentOutcome::Dropped(o) => o.dropped.clone(),
2588 _ => None,
2589 }) else {
2590 continue;
2591 };
2592 let Some(job) = sent.get(*wi) else { continue };
2593 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2595 self.state.event(
2596 "implement",
2597 format!(
2598 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2599 work is in the tree",
2600 seat.key, dropped.output_tokens, dropped.why
2601 ),
2602 );
2603 continue;
2604 }
2605 if !has_context(&job.spec, seat, job.sessions) {
2613 self.state.event(
2614 "implement",
2615 format!(
2616 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2617 is no session left to resume",
2618 seat.key, dropped.output_tokens, dropped.why
2619 ),
2620 );
2621 continue;
2622 }
2623 self.state.event(
2624 "implement",
2625 format!(
2626 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2627 conversation",
2628 seat.key, dropped.output_tokens, dropped.why
2629 ),
2630 );
2631 let mut retry = job.clone();
2632 retry.seat = seat.clone();
2633 retry.prompt = prompt::resume_after_drop(&dropped.why);
2634 retry.timeout = retry_budget(job.timeout, true);
2635 retry.stem = format!("{}-resume", job.stem);
2636 let cache = self.state.config.cache_dir();
2637 let ctx = WaveCtx {
2638 carry_seats: false,
2639 pauses: Some((&self.pause, &self.interrupt)),
2640 run: run_id,
2641 node: "implement",
2642 prompts,
2643 cache: cache.as_deref(),
2644 round: None,
2645 };
2646 let (resumed_seat, resumed) =
2647 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2648 *seat = resumed_seat;
2649 *out = resumed;
2650 }
2651 }
2652
2653 async fn resume_seat_handovers(
2718 &mut self,
2719 results: &mut [(usize, SeatState, AgentOutcome)],
2720 sent: &mut [SeatJob],
2721 prompts: &Prompts,
2722 run_id: &str,
2723 ) -> Result<()> {
2724 let instruction = seeded_instruction(&self.state);
2725 let language = self.state.config.graph.language.clone();
2726 let github_language = self.state.config.graph.github_language.clone();
2727 let brief = self
2728 .state
2729 .advice
2730 .as_ref()
2731 .and_then(|a| a.synthesis.as_deref())
2732 .map(str::to_owned);
2733 let attachments = self.state.attachments.clone();
2734 for (wi, seat, out) in results.iter_mut() {
2735 let others: BTreeSet<String> = sent
2738 .iter()
2739 .enumerate()
2740 .filter(|(j, _)| j != wi)
2741 .map(|(_, j)| j.spec.id.clone())
2742 .collect();
2743 let Some(job) = sent.get_mut(*wi) else {
2744 continue;
2745 };
2746 let start = self
2751 .roles
2752 .implementer_roster
2753 .iter()
2754 .position(|s| s.id == job.spec.id)
2755 .unwrap_or(0);
2756 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2757 let mut fallback_attempt = 0usize;
2758 let mut prev: Option<FailClass> = None;
2759 while let Some(cur) = FailClass::of(&*out) {
2760 if !should_hand_over(prev.as_ref(), &cur) {
2761 break;
2762 }
2763 let Some(next) =
2764 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2765 .cloned()
2766 else {
2767 break;
2768 };
2769 tried.insert(next.id.clone());
2770 fallback_attempt += 1;
2771
2772 let coordination = self.coordinate(&job.cwd).await?;
2773 if let Ok(r) = git::rescue_commit(
2774 &job.cwd,
2775 &format!(
2776 "magi: candidate {} (uncommitted work before {} fallback)",
2777 seat.key,
2778 if cur == FailClass::Quota {
2779 "quota"
2780 } else {
2781 "handover"
2782 }
2783 ),
2784 )
2785 .await
2786 {
2787 self.state.note_withheld("implement", &r.withheld);
2788 }
2789 drop(coordination);
2790
2791 record_handover(
2792 &mut self.state,
2793 "implement",
2794 &seat.key,
2795 &seat.agent,
2796 &next.id,
2797 &cur,
2798 &fail_reason(&*out),
2799 );
2800 prev = Some(cur.clone());
2801
2802 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2803 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2804 job.spec = next.clone();
2812 let mut retry = job.clone();
2813 retry.seat = new_seat;
2814 retry.prompt = prompt::implement_in(
2815 &instruction,
2816 &job.cwd.to_string_lossy(),
2817 &language,
2818 &github_language,
2819 brief.as_deref(),
2820 &attachments,
2821 );
2822 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2823 let cache = self.state.config.cache_dir();
2824 let ctx = WaveCtx {
2825 carry_seats: false,
2826 pauses: Some((&self.pause, &self.interrupt)),
2827 run: run_id,
2828 node: "implement",
2829 prompts,
2830 cache: cache.as_deref(),
2831 round: None,
2832 };
2833 let (fallback_seat, fallback_out) = run_one(
2834 retry,
2835 Arc::clone(&self.sem),
2836 &ctx,
2837 &mut self.state,
2838 fallback_attempt,
2839 )
2840 .await;
2841 *seat = fallback_seat;
2842 *out = fallback_out;
2843 }
2844 }
2845 Ok(())
2846 }
2847
2848 async fn resume_unconfirmed_commands(
2872 &mut self,
2873 results: &mut [(usize, SeatState, AgentOutcome)],
2874 sent: &[SeatJob],
2875 prompts: &Prompts,
2876 run_id: &str,
2877 ) {
2878 for (wi, seat, out) in results.iter_mut() {
2879 let AgentOutcome::Ok(o) = &*out else {
2880 continue;
2881 };
2882 if !has_unconfirmed_command(&o.commands) {
2883 continue;
2884 }
2885 let Some(job) = sent.get(*wi) else { continue };
2886 if !has_context(&job.spec, seat, job.sessions) {
2887 self.state.event(
2888 "implement",
2889 format!(
2890 "{}: the reply named a command whose own CLI never confirmed the exit \
2891 status of, but there is no session left to resume",
2892 seat.key
2893 ),
2894 );
2895 continue;
2896 }
2897 self.state.event(
2898 "implement",
2899 format!(
2900 "{}: the reply named a command whose own CLI never confirmed the exit \
2901 status of; resuming the conversation",
2902 seat.key
2903 ),
2904 );
2905 let mut retry = job.clone();
2906 retry.seat = seat.clone();
2907 retry.prompt = prompt::resume_incomplete(
2908 "a command in your last reply had no confirmed exit status",
2909 );
2910 retry.timeout = retry_budget(job.timeout, true);
2911 retry.stem = format!("{}-confirm", job.stem);
2912 let cache = self.state.config.cache_dir();
2913 let ctx = WaveCtx {
2914 carry_seats: false,
2915 pauses: Some((&self.pause, &self.interrupt)),
2916 run: run_id,
2917 node: "implement",
2918 prompts,
2919 cache: cache.as_deref(),
2920 round: None,
2921 };
2922 let (resumed_seat, resumed) =
2923 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2924 *seat = resumed_seat;
2925 *out = resumed;
2926 }
2927 }
2928
2929 async fn continue_fix_report(
2950 &mut self,
2951 mut seat: SeatState,
2952 parse_err: String,
2953 job: &SeatJob,
2954 prompts: &Prompts,
2955 run_id: &str,
2956 round: usize,
2957 ) -> (
2958 SeatState,
2959 Option<FixReport>,
2960 Option<String>,
2961 ContinuationRecord,
2962 ) {
2963 let mut last_err = parse_err;
2964 let mut cumulative_wait_ms = 0u64;
2965 let mut attempts = 0usize;
2966 loop {
2967 if !has_context(&job.spec, &seat, job.sessions) {
2968 self.state.event(
2969 "fix",
2970 format!(
2971 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2972 session left to resume into"
2973 ),
2974 );
2975 let outcome = if attempts == 0 {
2976 ContinuationOutcome::NoSession
2977 } else {
2978 ContinuationOutcome::Exhausted
2979 };
2980 return (
2981 seat,
2982 None,
2983 Some(format!("unparsable fix report: {last_err}")),
2984 ContinuationRecord {
2985 attempts,
2986 cumulative_wait_ms,
2987 outcome,
2988 },
2989 );
2990 }
2991 if attempts >= MAX_FIX_CONTINUATIONS {
2992 self.state.event(
2993 "fix",
2994 format!(
2995 "round {round}: fixer's reply still had no adoption report after \
2996 {attempts} continuation(s) ({last_err}); giving up"
2997 ),
2998 );
2999 return (
3000 seat,
3001 None,
3002 Some(format!(
3003 "unparsable fix report after {attempts} continuation(s): {last_err}"
3004 )),
3005 ContinuationRecord {
3006 attempts,
3007 cumulative_wait_ms,
3008 outcome: ContinuationOutcome::Exhausted,
3009 },
3010 );
3011 }
3012 attempts += 1;
3013 self.state.event(
3014 "fix",
3015 format!(
3016 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3017 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3018 ),
3019 );
3020 let mut retry = job.clone();
3021 retry.seat = seat.clone();
3022 retry.prompt = prompt::resume_incomplete(&last_err);
3023 retry.timeout = retry_budget(job.timeout, true);
3024 retry.stem = format!("{}-continue{attempts}", job.stem);
3025 let cache = self.state.config.cache_dir();
3026 let ctx = WaveCtx {
3027 carry_seats: false,
3028 pauses: Some((&self.pause, &self.interrupt)),
3029 run: run_id,
3030 node: "fix",
3031 prompts,
3032 cache: cache.as_deref(),
3033 round: Some(round),
3034 };
3035 let (resumed_seat, resumed_out) = run_one(
3036 retry,
3037 Arc::clone(&self.sem),
3038 &ctx,
3039 &mut self.state,
3040 attempts,
3041 )
3042 .await;
3043 seat = resumed_seat;
3044 match resumed_out {
3045 AgentOutcome::Ok(o) => {
3046 cumulative_wait_ms += o.duration_ms;
3047 match verdict::extract_json::<FixReport>(&o.text) {
3048 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3049 self.state.event(
3050 "fix",
3051 format!(
3052 "round {round}: fixer's adoption report recovered after \
3053 {attempts} continuation(s)"
3054 ),
3055 );
3056 return (
3057 seat,
3058 Some(report),
3059 None,
3060 ContinuationRecord {
3061 attempts,
3062 cumulative_wait_ms,
3063 outcome: ContinuationOutcome::Resumed,
3064 },
3065 );
3066 }
3067 Ok(_) => {
3075 last_err = "the reply parsed, but it reported a command whose own CLI \
3076 never confirmed an exit status"
3077 .to_owned();
3078 }
3079 Err(e) => last_err = e.to_string(),
3080 }
3081 }
3082 AgentOutcome::Quota(o) => {
3083 cumulative_wait_ms += o.duration_ms;
3084 self.state.quota.push(QuotaLoss {
3085 seat: seat.key.clone(),
3086 node: "fix".to_owned(),
3087 at: Timestamp::now(),
3088 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3089 });
3090 self.state.event(
3091 "fix",
3092 format!(
3093 "round {round}: continuation rate limited (quota); not retrying now"
3094 ),
3095 );
3096 return (
3097 seat,
3098 None,
3099 Some("rate limited (quota) while recovering the fix report".to_owned()),
3100 ContinuationRecord {
3101 attempts,
3102 cumulative_wait_ms,
3103 outcome: ContinuationOutcome::QuotaLost,
3104 },
3105 );
3106 }
3107 AgentOutcome::Dropped(o) => {
3108 cumulative_wait_ms += o.duration_ms;
3109 let why = o
3110 .dropped
3111 .as_ref()
3112 .map(|d| d.why.as_str())
3113 .unwrap_or("the CLI ended the stream without delivering its answer");
3114 last_err = format!("the CLI dropped the stream ({why})");
3115 }
3116 AgentOutcome::Failed(e) => last_err = e,
3117 AgentOutcome::ResourceBlocked(e) => {
3120 return (
3121 seat,
3122 None,
3123 Some(format!("the continuation was not started: {e}")),
3124 ContinuationRecord {
3125 attempts: attempts.saturating_sub(1),
3126 cumulative_wait_ms,
3127 outcome: ContinuationOutcome::NoSession,
3128 },
3129 );
3130 }
3131 }
3132 }
3133 }
3134
3135 fn after_implement(&mut self) -> Result<()> {
3136 if !self.state.candidate_preservation.is_empty() {
3137 bail!("candidate preservation incomplete; judging is unsafe");
3138 }
3139 if self.state.leaks.is_empty() {
3141 let cfg = self.state.config.blind.clone();
3142 let mut leaks = Vec::new();
3143 for c in &self.state.candidates {
3144 let Some(patch) =
3145 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3146 else {
3147 continue;
3148 };
3149 leaks.extend(blind::scan(
3150 &format!("candidate {} patch", c.label),
3151 &patch,
3152 &cfg.vendor_tokens,
3153 ));
3154 }
3155 if !leaks.is_empty() {
3156 let summary = leaks
3157 .iter()
3158 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3159 .collect::<Vec<_>>()
3160 .join(", ");
3161 match cfg.on_leak {
3162 LeakPolicy::Fail => {
3163 self.state.status = RunStatus::Failed;
3164 self.state
3165 .event("blind", format!("vendor text in a patch: {summary}"));
3166 self.state.leaks = leaks;
3167 self.state.save()?;
3168 self.settle_questions();
3169 bail!(
3170 "blind.on_leak = \"fail\" and vendor text reached a \
3171 judged patch: {summary}"
3172 );
3173 }
3174 LeakPolicy::Redact => self.state.event(
3175 "blind",
3176 format!("redacting vendor text for judging: {summary}"),
3177 ),
3178 LeakPolicy::Warn => self.state.event(
3179 "blind",
3180 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3181 ),
3182 }
3183 self.state.leaks = leaks;
3184 }
3185 }
3186
3187 if self.state.viable().is_empty() {
3188 if self.state.all_candidates_verified_noop() {
3189 self.state.status = RunStatus::VerifiedNoop;
3200 self.state.save()?;
3201 self.settle_questions();
3202 return Ok(());
3203 }
3204 self.state.status = RunStatus::Failed;
3205 self.state.save()?;
3206 self.settle_questions();
3207 bail!("no candidate produced a change; nothing to judge");
3208 }
3209 self.state.status = RunStatus::Judging;
3210 self.state.save()?;
3211 Ok(())
3212 }
3213
3214 async fn judge(&mut self) -> Result<()> {
3217 let run_id = self.state.id.clone();
3222 let prompts = self.state.config.prompts.clone();
3223 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3224 return Ok(());
3225 }
3226 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3227 if viable.len() == 1 {
3228 self.state.judge_skipped = true;
3235 self.state.event(
3236 "judge",
3237 format!(
3238 "only candidate {} produced a change; judging skipped",
3239 viable[0].label
3240 ),
3241 );
3242 self.state.save()?;
3243 return Ok(());
3244 }
3245 self.state.status = RunStatus::Judging;
3246
3247 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3248 let language = self.state.config.graph.language.clone();
3249 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3250 let sessions = self.state.config.graph.sessions;
3251 let artifacts = agent::artifacts_dir(&self.state.dir());
3252 let root = self.state.worktree_root();
3253 let base_short = short(&self.state.base_commit);
3254
3255 let mut jobs = Vec::new();
3256 let mut orders = Vec::new();
3257 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3258 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3259 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3260 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3261 let seat_key = format!("judge-{}", j + 1);
3262 let seat = self.seat(&seat_key, &spec.id);
3263 jobs.push(SeatJob {
3264 prompt: prompt::judge(
3265 &self.state.instruction,
3266 &views,
3267 self.roles.judges.len(),
3268 &base_short,
3269 &language,
3270 ),
3271 spec,
3272 seat,
3273 cwd: root.join(format!("judge-{}", j + 1)),
3274 timeout,
3275 allow_write: false,
3276 sessions,
3277 artifacts: artifacts.clone(),
3278 stem: format!("judge-{}", j + 1),
3279 handover: None,
3280 });
3281 }
3282
3283 self.state.event(
3284 "judge",
3285 format!(
3286 "{} judges ranking {} candidates blind",
3287 jobs.len(),
3288 viable.len()
3289 ),
3290 );
3291 let labels_for_check = labels.clone();
3292 let mut quota_losses = Vec::new();
3293 let cache = self.state.config.cache_dir();
3294 let ctx = WaveCtx {
3295 carry_seats: false,
3296 pauses: Some((&self.pause, &self.interrupt)),
3297 run: &run_id,
3298 node: "judge",
3299 prompts: &prompts,
3300 cache: cache.as_deref(),
3301 round: None,
3302 };
3303 let results = ask_json_wave::<Ranking>(
3304 jobs,
3305 Arc::clone(&self.sem),
3306 self.state.config.graph.retries,
3307 &self.roles.judge_roster,
3308 &ctx,
3309 &mut quota_losses,
3310 &mut self.state,
3311 &move |r: &Ranking| r.validate(&labels_for_check),
3312 )
3313 .await;
3314 if self.state.resource_block.is_some() {
3315 return self.stop_for_resource();
3316 }
3317 self.state.quota.extend(quota_losses);
3318
3319 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3320 let agent_id = seat.agent.clone();
3321 self.state.seats.insert(seat.key.clone(), seat);
3322 let mut record = Judgement {
3323 judge: j + 1,
3324 seat: format!("judge-{}", j + 1),
3325 agent: agent_id,
3326 ranking: Vec::new(),
3327 reasons: BTreeMap::new(),
3328 confidence: None,
3329 strengths: BTreeMap::new(),
3330 order: orders[j].clone(),
3331 failed: None,
3332 duration_ms: 0,
3333 };
3334 match res {
3335 Ok((ranking, out)) => {
3336 record.ranking = ranking.normalized();
3337 record.reasons = ranking.reasons;
3338 record.strengths = ranking.strengths;
3339 record.confidence = ranking.confidence;
3340 record.duration_ms = out.duration_ms;
3341 self.state.event(
3342 "judge",
3343 format!(
3344 "judge {} ranked {}",
3345 j + 1,
3346 record.ranking.iter().collect::<String>()
3347 ),
3348 );
3349 }
3350 Err(e) => {
3351 record.failed = Some(e.to_string());
3352 self.state
3353 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3354 }
3355 }
3356 self.state.judgements.push(record);
3357 self.state.save()?;
3358 }
3359 Ok(())
3360 }
3361
3362 async fn deliberate(&mut self) -> Result<()> {
3365 let run_id = self.state.id.clone();
3370 let prompts = self.state.config.prompts.clone();
3371 if !self.state.deliberation.is_empty() {
3372 return Ok(());
3373 }
3374 let tops: Vec<char> = self
3375 .state
3376 .judgements
3377 .iter()
3378 .filter_map(|j| j.ranking.first().copied())
3379 .collect();
3380 let rounds = self.state.config.graph.deliberate_rounds;
3381 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3382 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3383 self.state.event(
3384 "deliberate",
3385 format!("judges agreed on {} outright; no deliberation", tops[0]),
3386 );
3387 }
3388 self.state.status = RunStatus::Voting;
3389 self.state.save()?;
3390 return Ok(());
3391 }
3392
3393 self.state.status = RunStatus::Deliberating;
3394 self.state.event(
3395 "deliberate",
3396 format!(
3397 "split: first choices were {} — opening {rounds} round(s)",
3398 tops.iter().collect::<String>()
3399 ),
3400 );
3401
3402 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3403 let language = self.state.config.graph.language.clone();
3404 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3405 let sessions = self.state.config.graph.sessions;
3406 let artifacts = agent::artifacts_dir(&self.state.dir());
3407 let root = self.state.worktree_root();
3408 let base_short = short(&self.state.base_commit);
3409
3410 for round in 1..=rounds {
3414 let mut turns: Vec<DeliberationTurn> = Vec::new();
3415 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3416 if self.state.judgements[j].failed.is_some() {
3417 continue;
3418 }
3419 let seat_key = format!("judge-{}", j + 1);
3420 let spec = self.occupant(&seat_key, spec);
3421 let mut seat = self.seat(&seat_key, &spec.id);
3422 let transcript = self.transcript(&turns, j);
3423 let build = |context: Option<&str>| {
3424 prompt::deliberate(
3425 &self.state.instruction,
3426 context,
3427 &transcript,
3428 round,
3429 rounds,
3430 &language,
3431 )
3432 };
3433 let block = self.candidate_block(&viable, &base_short);
3434 let full = build(Some(&block));
3435 let text = if has_context(&spec, &seat, sessions) {
3436 build(None)
3437 } else {
3438 full.clone()
3439 };
3440 let job = SeatJob {
3441 spec,
3442 seat: seat.clone(),
3443 prompt: text,
3444 cwd: root.join(format!("judge-{}", j + 1)),
3445 timeout,
3446 allow_write: false,
3447 sessions,
3448 artifacts: artifacts.clone(),
3449 stem: format!("delib-{round}-judge-{}", j + 1),
3450 handover: Some(full),
3451 };
3452 let cache = self.state.config.cache_dir();
3453 let ctx = WaveCtx {
3454 carry_seats: false,
3455 pauses: Some((&self.pause, &self.interrupt)),
3456 run: &run_id,
3457 node: "deliberate",
3458 prompts: &prompts,
3459 cache: cache.as_deref(),
3460 round: None,
3461 };
3462 let mut losses = Vec::new();
3466 let mut results = ask_wave_with::<String>(
3467 vec![job],
3468 Arc::clone(&self.sem),
3469 0,
3470 &self.roles.judge_roster,
3471 &ctx,
3472 &mut losses,
3473 &mut self.state,
3474 &|text: &str| {
3475 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3476 },
3477 )
3478 .await;
3479 self.state.quota.extend(losses);
3480 let (updated, res, _) = results.pop().expect("one job in, one result out");
3481 seat = updated;
3482 let agent_id = seat.agent.clone();
3483 self.state.seats.insert(seat.key.clone(), seat);
3484 let body = match res {
3485 Ok((body, _)) => body,
3486 Err(e) => {
3489 self.state
3490 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3491 continue;
3492 }
3493 };
3494 let tentative = verdict::extract_json::<Position>(&body)
3495 .ok()
3496 .and_then(|p| p.tentative)
3497 .and_then(|s| s.trim().chars().next())
3498 .map(|c| c.to_ascii_uppercase());
3499 self.state.event(
3500 "deliberate",
3501 format!(
3502 "round {round}: judge {} now favours {}",
3503 j + 1,
3504 tentative.map_or("—".to_owned(), |c| c.to_string())
3505 ),
3506 );
3507 turns.push(DeliberationTurn {
3508 judge: j + 1,
3509 agent: agent_id,
3510 body: blind::sanitize_prose(&body, &self.state.config.blind),
3511 tentative,
3512 });
3513 }
3514 self.state
3515 .deliberation
3516 .push(DeliberationRound { round, turns });
3517 self.state.save()?;
3518 }
3519
3520 self.state.status = RunStatus::Voting;
3521 self.state.save()?;
3522 Ok(())
3523 }
3524
3525 async fn vote(&mut self) -> Result<()> {
3528 let run_id = self.state.id.clone();
3533 let prompts = self.state.config.prompts.clone();
3534 if !self.state.votes.is_empty() {
3535 return Ok(());
3536 }
3537 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3538 if viable.len() == 1 {
3539 return Ok(());
3540 }
3541 self.state.status = RunStatus::Voting;
3542
3543 let language = self.state.config.graph.language.clone();
3544 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3545 let sessions = self.state.config.graph.sessions;
3546 let artifacts = agent::artifacts_dir(&self.state.dir());
3547 let root = self.state.worktree_root();
3548 let base_short = short(&self.state.base_commit);
3549 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3550
3551 let mut jobs = Vec::new();
3552 let mut seats_at = Vec::new();
3553 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3554 if self
3555 .state
3556 .judgements
3557 .get(j)
3558 .is_some_and(|r| r.failed.is_some())
3559 {
3560 continue;
3561 }
3562 let seat_key = format!("judge-{}", j + 1);
3563 let spec = self.occupant(&seat_key, spec);
3564 let seat = self.seat(&seat_key, &spec.id);
3565 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3566 let text = if has_context(&spec, &seat, sessions) {
3567 prompt::final_vote(&viable, &language)
3568 } else {
3569 full.clone()
3570 };
3571 jobs.push(SeatJob {
3572 spec,
3573 seat,
3574 prompt: text,
3575 cwd: root.join(format!("judge-{}", j + 1)),
3576 timeout,
3577 allow_write: false,
3578 sessions,
3579 artifacts: artifacts.clone(),
3580 stem: format!("vote-judge-{}", j + 1),
3581 handover: Some(full),
3582 });
3583 seats_at.push(j);
3584 }
3585
3586 self.state.event(
3587 "vote",
3588 format!(
3589 "collecting {} final votes one by one, privately",
3590 jobs.len()
3591 ),
3592 );
3593 let allowed = viable.clone();
3594 let mut quota_losses = Vec::new();
3595 let cache = self.state.config.cache_dir();
3596 let ctx = WaveCtx {
3597 carry_seats: false,
3598 pauses: Some((&self.pause, &self.interrupt)),
3599 run: &run_id,
3600 node: "vote",
3601 prompts: &prompts,
3602 cache: cache.as_deref(),
3603 round: None,
3604 };
3605 let results = ask_json_wave::<FinalVote>(
3606 jobs,
3607 Arc::clone(&self.sem),
3608 self.state.config.graph.retries,
3609 &self.roles.judge_roster,
3610 &ctx,
3611 &mut quota_losses,
3612 &mut self.state,
3613 &move |v: &FinalVote| match v.label() {
3614 Some(c) if allowed.contains(&c) => Ok(()),
3615 other => bail!("vote {other:?} is not one of {allowed:?}"),
3616 },
3617 )
3618 .await;
3619 if self.state.resource_block.is_some() {
3620 return self.stop_for_resource();
3621 }
3622 self.state.quota.extend(quota_losses);
3623
3624 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3625 let agent_id = seat.agent.clone();
3626 self.state.seats.insert(seat.key.clone(), seat);
3627 let initial = self
3628 .state
3629 .judgements
3630 .get(j)
3631 .and_then(|r| r.ranking.first().copied());
3632 let mut record = VoteRecord {
3633 judge: j + 1,
3634 agent: agent_id,
3635 vote: None,
3636 reason: String::new(),
3637 changed: false,
3638 };
3639 match res {
3640 Ok((v, _)) => {
3641 record.vote = v.label();
3642 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3643 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3644 self.state.event(
3645 "vote",
3646 format!(
3647 "judge {} voted {}{}",
3648 j + 1,
3649 record.vote.unwrap_or('?'),
3650 if record.changed { " (changed)" } else { "" }
3651 ),
3652 );
3653 }
3654 Err(e) => {
3655 self.state
3656 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3657 }
3658 }
3659 self.state.votes.push(record);
3660 self.state.save()?;
3661 }
3662 Ok(())
3663 }
3664
3665 fn tally(&mut self) -> Result<()> {
3668 if self.state.tally.is_some() {
3669 return Ok(());
3670 }
3671 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3672 let tops: Vec<char> = self
3673 .state
3674 .judgements
3675 .iter()
3676 .filter_map(|j| j.ranking.first().copied())
3677 .collect();
3678 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3679
3680 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3683 let mut cast: Vec<char> = Vec::new();
3684 for (i, j) in self.state.judgements.iter().enumerate() {
3685 let vote = self
3686 .state
3687 .votes
3688 .iter()
3689 .find(|v| v.judge == i + 1)
3690 .and_then(|v| v.vote)
3691 .or_else(|| j.ranking.first().copied());
3692 if let Some(v) = vote {
3693 *first_choice.entry(v).or_insert(0) += 1;
3694 cast.push(v);
3695 }
3696 }
3697
3698 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3699 for j in &self.state.judgements {
3700 let n = j.ranking.len();
3701 for (pos, label) in j.ranking.iter().enumerate() {
3702 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3703 }
3704 }
3705
3706 let best = first_choice.values().copied().max().unwrap_or(0);
3707 let mut leaders: Vec<char> = first_choice
3708 .iter()
3709 .filter(|(_, v)| **v == best)
3710 .map(|(k, _)| *k)
3711 .collect();
3712 let mut tie_break = None;
3713 if leaders.len() > 1 {
3714 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3715 let borda_leaders: Vec<char> = leaders
3716 .iter()
3717 .copied()
3718 .filter(|l| borda[l] == top_borda)
3719 .collect();
3720 tie_break = Some(if borda_leaders.len() == 1 {
3721 format!(
3722 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3723 leaders.len()
3724 )
3725 } else {
3726 format!(
3727 "{} way tie on both first-choice votes and Borda points, broken by label order",
3728 leaders.len()
3729 )
3730 });
3731 leaders = borda_leaders;
3732 leaders.sort_unstable();
3733 }
3734 let winner = *leaders
3735 .first()
3736 .or(viable.first())
3737 .context("no candidate to declare a winner from")?;
3738
3739 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3740 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3741 let deliberated = !self.state.deliberation.is_empty();
3742
3743 let quota_seats: std::collections::BTreeSet<&str> =
3747 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3748 let mut present = 0usize;
3749 for (i, j) in self.state.judgements.iter().enumerate() {
3750 if quota_seats.contains(j.seat.as_str()) {
3751 continue;
3752 }
3753 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3754 let voted = self
3755 .state
3756 .votes
3757 .iter()
3758 .any(|v| v.judge == i + 1 && v.vote.is_some());
3759 if ranked || voted {
3760 present += 1;
3761 }
3762 }
3763 let needs_quorum = viable.len() > 1;
3769 let judges_total = if needs_quorum {
3770 self.roles.judges.len()
3771 } else {
3772 0
3773 };
3774 let quorum = if needs_quorum {
3775 judges_total / 2 + 1
3776 } else {
3777 0
3778 };
3779 let met_quorum = !needs_quorum || present >= quorum;
3780 let uncontested = (!needs_quorum).then(|| {
3781 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3782 });
3783
3784 self.state.event(
3785 "tally",
3786 match &uncontested {
3787 Some(reason) => format!("winner {winner} — {reason}"),
3788 None => format!(
3789 "winner {winner} — votes {} | initial {} | {} changed | \
3790 {present}/{judges_total} judges{}",
3791 first_choice
3792 .iter()
3793 .map(|(k, v)| format!("{k}:{v}"))
3794 .collect::<Vec<_>>()
3795 .join(" "),
3796 if unanimous_initial {
3797 "unanimous"
3798 } else {
3799 "split"
3800 },
3801 changed_votes,
3802 if met_quorum {
3803 String::new()
3804 } else {
3805 format!(" — below quorum ({quorum} required)")
3806 },
3807 ),
3808 },
3809 );
3810 if !met_quorum {
3811 self.state.event(
3812 "stall",
3813 format!(
3814 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3815 the run stops here, resumable"
3816 ),
3817 );
3818 }
3819 self.state.tally = Some(Tally {
3820 first_choice,
3821 borda,
3822 winner,
3823 rankings: tops.len(),
3824 unanimous_initial,
3825 deliberated,
3826 changed_votes,
3827 unanimous_final,
3828 tie_break,
3829 judges: judges_total,
3830 present,
3831 quorum,
3832 met_quorum,
3833 uncontested,
3834 });
3835 self.state.status = if met_quorum {
3836 RunStatus::Reviewing
3837 } else {
3838 RunStatus::Stalled
3839 };
3840 self.state.save()?;
3841 Ok(())
3842 }
3843
3844 #[allow(clippy::too_many_lines)]
3865 async fn recover_stall(&mut self) -> Result<bool> {
3866 let run_id = self.state.id.clone();
3871 let prompts = self.state.config.prompts.clone();
3872 let quota_seats: BTreeSet<&str> =
3877 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3878 let absent: Vec<String> = self
3879 .state
3880 .judgements
3881 .iter()
3882 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3883 .map(|j| j.seat.clone())
3884 .collect();
3885 if absent.is_empty() {
3886 return Ok(false);
3887 }
3888 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3889 if viable.len() <= 1 {
3890 return Ok(false);
3891 }
3892 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3893 let language = self.state.config.graph.language.clone();
3894 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3895 let sessions = self.state.config.graph.sessions;
3896 let artifacts = agent::artifacts_dir(&self.state.dir());
3897 let root = self.state.worktree_root();
3898 let base_short = short(&self.state.base_commit);
3899 let candidates: Vec<Candidate> = viable.clone();
3900
3901 let mut positions: Vec<usize> = absent
3903 .iter()
3904 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3905 .collect();
3906 if positions.is_empty() {
3907 return Ok(false);
3908 }
3909 positions.sort_unstable();
3910 positions.dedup();
3911
3912 let mut judge_jobs = Vec::new();
3914 for &j in &positions {
3915 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3916 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3917 let seat_key = format!("judge-{}", j + 1);
3918 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3919 let seat = self.seat(&seat_key, &spec.id);
3920 judge_jobs.push(SeatJob {
3921 spec,
3922 seat,
3923 prompt: prompt::judge(
3924 &self.state.instruction,
3925 &views,
3926 self.roles.judges.len(),
3927 &base_short,
3928 &language,
3929 ),
3930 cwd: root.join(seat_key),
3931 timeout,
3932 allow_write: false,
3933 sessions,
3934 artifacts: artifacts.clone(),
3935 stem: format!("judge-{}-recover", j + 1),
3936 handover: None,
3937 });
3938 }
3939
3940 let labels_for_check = labels.clone();
3941 let mut judge_losses = Vec::new();
3942 let retries = self.state.config.graph.retries;
3943 let cache = self.state.config.cache_dir();
3944 let ctx = WaveCtx {
3945 carry_seats: false,
3946 pauses: Some((&self.pause, &self.interrupt)),
3947 run: &run_id,
3948 node: "judge",
3949 prompts: &prompts,
3950 cache: cache.as_deref(),
3951 round: None,
3952 };
3953 let results = ask_json_wave::<Ranking>(
3954 judge_jobs,
3955 Arc::clone(&self.sem),
3956 retries,
3957 &self.roles.judge_roster,
3958 &ctx,
3959 &mut judge_losses,
3960 &mut self.state,
3961 &move |r: &Ranking| r.validate(&labels_for_check),
3962 )
3963 .await;
3964 if self.state.resource_block.is_some() {
3965 self.stop_for_resource()?;
3966 return Ok(false);
3967 }
3968
3969 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3971 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3972 let agent_id = seat.agent.clone();
3973 self.state.seats.insert(seat.key.clone(), seat);
3974 let record = &mut self.state.judgements[j];
3975 match res {
3976 Ok((ranking, out)) => {
3977 record.agent = agent_id;
3978 record.ranking = ranking.normalized();
3979 record.reasons = ranking.reasons;
3980 record.strengths = ranking.strengths;
3981 record.confidence = ranking.confidence;
3982 record.failed = None;
3983 record.duration_ms = out.duration_ms;
3984 recovered.insert(j);
3985 self.state.event(
3986 "recover",
3987 format!("judge {} ranked again after the limit", j + 1),
3988 );
3989 }
3990 Err(e) => {
3991 self.state
3992 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3993 }
3994 }
3995 }
3996
3997 let mut vote_jobs = Vec::new();
3999 let mut vote_pos: Vec<usize> = Vec::new();
4000 for &j in &recovered {
4001 let seat_key = format!("judge-{}", j + 1);
4002 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003 let seat = self.seat(&seat_key, &spec.id);
4004 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4005 let text = if has_context(&spec, &seat, sessions) {
4006 prompt::final_vote(&labels, &language)
4007 } else {
4008 full.clone()
4009 };
4010 vote_jobs.push(SeatJob {
4011 spec,
4012 seat,
4013 prompt: text,
4014 cwd: root.join(seat_key),
4015 timeout,
4016 allow_write: false,
4017 sessions,
4018 artifacts: artifacts.clone(),
4019 stem: format!("vote-judge-{}-recover", j + 1),
4020 handover: Some(full),
4021 });
4022 vote_pos.push(j);
4023 }
4024 let allowed = labels.clone();
4025 let mut vote_losses = Vec::new();
4026 let vote_retries = self.state.config.graph.retries;
4027 let vote_cache = self.state.config.cache_dir();
4028 let ctx = WaveCtx {
4029 carry_seats: false,
4030 pauses: Some((&self.pause, &self.interrupt)),
4031 run: &run_id,
4032 node: "vote",
4033 prompts: &prompts,
4034 cache: vote_cache.as_deref(),
4035 round: None,
4036 };
4037 let votes = ask_json_wave::<FinalVote>(
4038 vote_jobs,
4039 Arc::clone(&self.sem),
4040 vote_retries,
4041 &self.roles.judge_roster,
4042 &ctx,
4043 &mut vote_losses,
4044 &mut self.state,
4045 &move |v: &FinalVote| match v.label() {
4046 Some(c) if allowed.contains(&c) => Ok(()),
4047 other => bail!("vote {other:?} is not one of {allowed:?}"),
4048 },
4049 )
4050 .await;
4051 if self.state.resource_block.is_some() {
4052 self.stop_for_resource()?;
4053 return Ok(false);
4054 }
4055 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4056 let agent_id = seat.agent.clone();
4057 self.state.seats.insert(seat.key.clone(), seat);
4058 match res {
4059 Ok((v, _)) => {
4060 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4061 rec.vote = v.label();
4062 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4063 } else {
4064 self.state.votes.push(VoteRecord {
4065 judge: j + 1,
4066 agent: agent_id,
4067 vote: v.label(),
4068 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4069 changed: false,
4070 });
4071 }
4072 self.state.event(
4073 "recover",
4074 format!("judge {} voted again after the limit", j + 1),
4075 );
4076 }
4077 Err(e) => {
4078 self.state
4079 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4080 }
4081 }
4082 }
4083
4084 let recovered_keys: BTreeSet<String> = recovered
4088 .iter()
4089 .map(|&j| format!("judge-{}", j + 1))
4090 .collect();
4091 self.state
4092 .quota
4093 .retain(|q| !recovered_keys.contains(&q.seat));
4094 for loss in judge_losses.into_iter().chain(vote_losses) {
4098 if recovered_keys.contains(&loss.seat) {
4099 continue;
4100 }
4101 self.state.quota.retain(|q| q.seat != loss.seat);
4102 self.state.quota.push(loss);
4103 }
4104
4105 self.state.tally = None;
4107 self.tally()?;
4108 Ok(self
4109 .state
4110 .tally
4111 .as_ref()
4112 .map(|t| t.met_quorum)
4113 .unwrap_or(false))
4114 }
4115
4116 async fn graft(&mut self) -> Result<()> {
4132 let Err(e) = self.graft_inner().await else {
4133 return Ok(());
4134 };
4135 if e.is::<crate::repo_lock::ResourceBlocked>() {
4136 return Err(e);
4137 }
4138 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4139 self.state.status = RunStatus::Blocked;
4140 self.state.event("graft", format!("blocked: {e:#}"));
4141 self.state.save()?;
4142 return Err(e);
4143 }
4144 if let Some(rec) = self.state.graft.clone()
4149 && !rec.outcome.settled()
4150 {
4151 if let Some(winner) = self.state.winner().cloned() {
4152 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4153 if re.is::<crate::repo_lock::ResourceBlocked>() {
4154 return Err(re);
4155 }
4156 self.state.status = RunStatus::Blocked;
4157 self.state.event("graft", format!("blocked: {re:#}"));
4158 self.state.save()?;
4159 return Err(re);
4160 }
4161 }
4162 if let Some(g) = self.state.graft.as_mut() {
4163 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4164 }
4165 }
4166 self.state.event("graft", format!("skipped: {e:#}"));
4167 if let Err(e) = self.state.save() {
4168 tracing::warn!("save after graft: {e:#}");
4169 }
4170 Ok(())
4171 }
4172
4173 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4176 let _coordination = self.coordinate(wt).await?;
4177 crate::recovery::restore_repair(wt, pre)
4178 .await
4179 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4180 }
4181
4182 fn graft_skipped(&mut self, why: &str) {
4184 let message = format!("skipped: {why}");
4185 if !self
4186 .state
4187 .events
4188 .iter()
4189 .any(|e| e.node == "graft" && e.message == message)
4190 {
4191 self.state.event("graft", message);
4192 }
4193 }
4194
4195 async fn graft_inner(&mut self) -> Result<()> {
4196 if let Some(rec) = self.state.graft.clone()
4199 && !rec.outcome.settled()
4200 {
4201 if self.state.reviews.is_empty()
4205 && let Some(winner) = self.state.winner().cloned()
4206 {
4207 self.graft_restore(&winner.worktree, &rec.pre).await?;
4208 }
4209 if rec.outcome == GraftOutcome::WaitingForRestore {
4210 self.state.graft = None;
4211 self.state.save()?;
4212 } else {
4213 if let Some(g) = self.state.graft.as_mut() {
4214 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4215 }
4216 self.state.event(
4217 "graft",
4218 "rolled back: the previous round was interrupted; the winner is untouched",
4219 );
4220 self.state.save()?;
4221 return Ok(());
4222 }
4223 }
4224 let plan = match graft::plan(&self.state) {
4225 Ok(plan) => plan,
4226 Err(skip) => {
4227 if skip != graft::Skip::AlreadyDone {
4228 self.graft_skipped(skip.reason());
4229 self.state.save()?;
4230 }
4231 return Ok(());
4232 }
4233 };
4234 let Some(winner) = self.state.winner().cloned() else {
4235 return Ok(());
4236 };
4237 let wt = winner.worktree.clone();
4238 if !git::is_clean(&wt).await? {
4239 self.graft_skipped("the winner's worktree has uncommitted changes");
4240 self.state.save()?;
4241 return Ok(());
4242 }
4243 let pre = git::rev_parse(&wt, "HEAD").await?;
4244 self.state.graft = Some(GraftRecord {
4247 pre: pre.clone(),
4248 post: None,
4249 lines: None,
4250 limit: None,
4251 agent: None,
4252 outcome: GraftOutcome::Running,
4253 requested: graft::requested(&plan),
4254 adoption: None,
4255 adoption_error: None,
4256 });
4257 self.state.event(
4258 "graft",
4259 format!(
4260 "porting strengths of candidate(s) {} onto {}",
4261 plan.strengths
4262 .iter()
4263 .map(|(l, _, _)| l.to_string())
4264 .collect::<Vec<_>>()
4265 .join(", "),
4266 plan.winner
4267 ),
4268 );
4269 self.state.save()?;
4270
4271 let fix_prompt = prompt::graft(
4272 &self.state.instruction,
4273 plan.winner,
4274 &plan.strengths,
4275 &self.state.config.graph.language,
4276 );
4277 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4278 let sessions = self.state.config.graph.sessions;
4279 let artifacts = agent::artifacts_dir(&self.state.dir());
4280 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4281 let (_, seat, out) = self
4282 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4283 SeatJob {
4284 prompt: fix_prompt.clone(),
4285 spec,
4286 seat,
4287 cwd: wt.clone(),
4288 timeout,
4289 allow_write: true,
4290 sessions,
4291 artifacts: artifacts.clone(),
4292 stem: "graft".to_owned(),
4293 handover: None,
4294 }
4295 })
4296 .await;
4297 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4298 self.state.graft.as_mut().expect("graft record").outcome =
4301 GraftOutcome::WaitingForRestore;
4302 return self.stop_for_resource();
4303 }
4304 let mut why: Option<String> = None;
4305 match &out {
4306 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4307 why = Some("the fixer produced nothing usable".to_owned());
4308 }
4309 AgentOutcome::Ok(_) => {}
4310 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4311 AgentOutcome::Quota(o) => {
4312 self.state.quota.push(QuotaLoss {
4313 seat: seat.key.clone(),
4314 node: "graft".to_owned(),
4315 at: Timestamp::now(),
4316 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4317 });
4318 why = Some("rate limited (quota)".to_owned());
4319 }
4320 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4321 why = Some(format!(
4322 "the fixer failed: {}",
4323 e.lines()
4324 .next()
4325 .unwrap_or("")
4326 .chars()
4327 .take(160)
4328 .collect::<String>()
4329 ));
4330 }
4331 }
4332 let agent_id = seat.agent.clone();
4333 self.state.seats.insert(seat.key.clone(), seat);
4334
4335 let mut post = None;
4336 let mut lines = None;
4337 let _coordination = self.coordinate(&wt).await?;
4338 if why.is_none() {
4339 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4340 if let Ok(r) = &rescued {
4341 self.state.note_withheld("graft", &r.withheld);
4342 }
4343 let head = git::rev_parse(&wt, "HEAD").await?;
4344 let dirty = match &rescued {
4348 Ok(r) => git::status_porcelain(&wt)
4349 .await
4350 .map(|st| {
4351 st.lines().any(|l| {
4352 !r.withheld
4353 .iter()
4354 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4355 })
4356 })
4357 .unwrap_or(true),
4358 Err(_) => true,
4359 };
4360 if dirty {
4361 why = Some("the fixer left changes that could not be committed".to_owned());
4362 } else if head == pre {
4363 why = Some("the fixer changed nothing".to_owned());
4364 } else {
4365 lines = git::changed_lines(&wt, &pre, &head).await;
4366 post = Some(head);
4367 }
4368 }
4369 let (adoption, adoption_error) = match &out {
4370 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4371 _ => (None, None),
4372 };
4373 let requested = graft::requested(&plan);
4374 let outcome = match why {
4375 None => {
4376 self.state.event(
4377 "graft",
4378 format!(
4379 "applied: {}",
4380 lines.map_or_else(
4381 || "changed lines unavailable".to_owned(),
4382 |n| format!("{n} changed lines")
4383 )
4384 ),
4385 );
4386 GraftOutcome::Applied
4387 }
4388 Some(why) => {
4389 crate::recovery::restore_repair(&wt, &pre)
4392 .await
4393 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4394 self.state.event(
4395 "graft",
4396 format!("rolled back: {why}; the winner is untouched"),
4397 );
4398 GraftOutcome::RolledBack(why)
4399 }
4400 };
4401 self.state.graft = Some(GraftRecord {
4402 pre,
4403 post,
4404 lines,
4405 limit: None,
4406 agent: Some(agent_id),
4407 outcome,
4408 requested,
4409 adoption,
4410 adoption_error,
4411 });
4412 self.state.save()?;
4413 Ok(())
4414 }
4415
4416 async fn fold_losers(&mut self) -> Result<()> {
4419 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4420 return Ok(());
4421 };
4422 if !self.state.candidate_preservation.is_empty() {
4423 bail!("candidate preservation incomplete; refusing cleanup");
4424 }
4425 let repo = self.state.repo.clone();
4426 let _coordination = self.coordinate(&repo).await?;
4427 let mut folded = Vec::new();
4428 for i in 0..self.state.candidates.len() {
4429 let c = &self.state.candidates[i];
4430 if c.label == winner || c.folded {
4431 continue;
4432 }
4433 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4434 remove_preserved_worktree(&repo, &wt).await?;
4435 if git::branch_exists(&repo, &branch).await?
4436 && !git::branch_delete(&repo, &branch).await?
4437 {
4438 bail!("candidate branch removal refused");
4439 }
4440 self.state.candidates[i].folded = true;
4441 self.state.save()?;
4442 folded.push(label.to_string());
4443 }
4444 let root = self.state.worktree_root();
4446 for j in 1..=self.roles.judges.len() {
4447 let wt = root.join(format!("judge-{j}"));
4448 if wt.exists() {
4449 remove_preserved_worktree(&repo, &wt).await?;
4450 }
4451 }
4452 if self.state.config.graph.advise {
4455 for k in 1..=self.state.config.graph.advisors {
4456 let wt = root.join(format!("advisor-{k}"));
4457 if wt.exists() {
4458 remove_preserved_worktree(&repo, &wt).await?;
4459 }
4460 }
4461 }
4462 if !folded.is_empty() {
4463 self.state
4464 .event("fold", format!("folded candidates {}", folded.join(", ")));
4465 self.state.save()?;
4466 }
4467 Ok(())
4468 }
4469
4470 async fn sync_to_base(&mut self) -> Result<()> {
4510 if self.state.status == RunStatus::AlreadyInBase {
4511 return Ok(());
4512 }
4513 let conflicted = self
4514 .state
4515 .base_sync
4516 .as_ref()
4517 .is_some_and(|s| s.conflict.is_some());
4518 let Some(winner) = self.state.winner().cloned() else {
4519 return Ok(());
4520 };
4521
4522 let repo = self.state.repo.clone();
4523 let remote = self.state.config.merge.remote.clone();
4524 let base_branch = self.state.base_branch.clone();
4525 let tracking = format!("{remote}/{base_branch}");
4526
4527 git::fetch(&repo, &remote, &base_branch).await.ok();
4528 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4533 return Ok(());
4534 };
4535
4536 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4537 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4538 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4539
4540 if (behind > 0 || conflicted || head == tip)
4549 && self
4550 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4551 .await?
4552 {
4553 return Ok(());
4554 }
4555 if conflicted {
4556 return Ok(());
4557 }
4558
4559 if behind == 0 {
4560 if let Some(from) = self
4569 .state
4570 .rebase_fixes
4571 .iter()
4572 .rev()
4573 .find_map(|r| r.from.clone())
4574 && from != head
4575 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4576 .await
4577 .is_ok_and(|o| o.ok())
4578 {
4579 git::sync_to_head(&winner.worktree).await?;
4580 }
4581 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4587 if let Some(why) = &conflict {
4588 self.state.status = RunStatus::Blocked;
4589 self.state.event("land", why.clone());
4590 }
4591 self.state.base_sync = Some(BaseSync {
4592 tip,
4593 behind: 0,
4594 attempts,
4595 conflict,
4596 already_in: None,
4597 });
4598 self.state.save()?;
4599 return Ok(());
4600 }
4601
4602 if attempts >= BASE_SYNC_ROUNDS {
4603 let why = format!(
4604 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4605 rebase(s); rebasing again would only race it",
4606 winner.branch
4607 );
4608 self.state.status = RunStatus::Blocked;
4609 self.state.base_sync = Some(BaseSync {
4610 tip,
4611 behind,
4612 attempts,
4613 conflict: Some(why.clone()),
4614 already_in: None,
4615 });
4616 self.state.event("land", why);
4617 self.state.save()?;
4618 return Ok(());
4619 }
4620
4621 self.state.event(
4622 "land",
4623 format!(
4624 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4625 winner.branch
4626 ),
4627 );
4628 self.state.save()?;
4629
4630 let branch_tracking = format!("{remote}/{}", winner.branch);
4636 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4637 let remote_tip = if self.state.task_pr_branch().is_none()
4638 && matches!(&fetched_branch, Ok(o) if o.ok())
4639 {
4640 git::rev_parse(&repo, &branch_tracking).await.ok()
4641 } else {
4642 None
4643 };
4644 if let Some(theirs) = &remote_tip
4648 && !git::is_ancestor(&repo, theirs, &head).await
4649 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4650 .await
4651 .is_ok_and(|missing| missing.is_empty())
4652 {
4653 let why = format!(
4654 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4655 them",
4656 short(theirs),
4657 winner.branch
4658 );
4659 self.state.status = RunStatus::Blocked;
4660 self.state.base_sync = Some(BaseSync {
4661 tip,
4662 behind,
4663 attempts,
4664 conflict: Some(why.clone()),
4665 already_in: None,
4666 });
4667 self.state.event("land", why);
4668 self.state.save()?;
4669 return Ok(());
4670 }
4671
4672 let scratch = self.state.dir().join("base-sync");
4673 let rebased = match crate::rebase::rebase_with_fixer(
4674 &mut self.state,
4675 &scratch,
4676 &winner.branch,
4677 &tracking,
4678 )
4679 .await
4680 {
4681 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4682 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4683 Err(e) => Err(e),
4684 };
4685 let attempts = attempts + 1;
4686 match rebased {
4687 Ok(None) => {
4688 git::sync_to_head(&winner.worktree).await?;
4692 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4693 let mut conflict = None;
4694 if let Some(pinned) = &remote_tip {
4695 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4696 match pushed {
4697 Ok(o) if o.ok() => self.state.event(
4698 "land",
4699 format!("pushed rebased {} to {remote}", winner.branch),
4700 ),
4701 Ok(o) => {
4702 conflict = Some(format!(
4703 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4704 winner.branch,
4705 short(pinned),
4706 o.stderr.chars().take(600).collect::<String>()
4707 ));
4708 }
4709 Err(e) => {
4710 conflict = Some(format!(
4711 "rebased {} locally but could not push it: {e:#}",
4712 winner.branch
4713 ));
4714 }
4715 }
4716 }
4717 if let Some(why) = &conflict {
4718 self.state.status = RunStatus::Blocked;
4719 self.state.event("land", why.clone());
4720 }
4721 self.state.base_sync = Some(BaseSync {
4722 tip: tip.clone(),
4723 behind: 0,
4724 attempts,
4725 conflict,
4726 already_in: None,
4727 });
4728 self.state
4729 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4730 }
4731 Ok(Some(conflict)) => {
4732 let why = format!(
4733 "{} conflicts with {tracking} and did not rebase: {}",
4734 winner.branch,
4735 conflict.chars().take(600).collect::<String>()
4736 );
4737 self.state.status = RunStatus::Blocked;
4738 self.state.base_sync = Some(BaseSync {
4739 tip,
4740 behind,
4741 attempts,
4742 conflict: Some(why.clone()),
4743 already_in: None,
4744 });
4745 self.state.event("land", why);
4746 }
4747 Err(e) => {
4748 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4749 self.state.status = RunStatus::Blocked;
4750 self.state.base_sync = Some(BaseSync {
4751 tip,
4752 behind,
4753 attempts,
4754 conflict: Some(why.clone()),
4755 already_in: None,
4756 });
4757 self.state.event("land", why);
4758 }
4759 }
4760 self.state.save()?;
4761 Ok(())
4762 }
4763
4764 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4770 if self.state.task_pr_branch().is_some() {
4773 return None;
4774 }
4775 let repo = self.state.repo.clone();
4776 let remote = self.state.config.merge.remote.clone();
4777 let fetched = git::fetch(&repo, &remote, branch).await;
4778 if !matches!(&fetched, Ok(o) if o.ok()) {
4779 return None;
4780 }
4781 let branch_tracking = format!("{remote}/{branch}");
4782 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4783 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4784 return None;
4785 }
4786 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4787 .await
4788 .is_ok_and(|missing| missing.is_empty())
4789 {
4790 return Some(format!(
4791 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4792 them",
4793 short(&theirs)
4794 ));
4795 }
4796 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4797 Ok(o) if o.ok() => {
4798 self.state
4799 .event("land", format!("pushed rebased {branch} to {remote}"));
4800 None
4801 }
4802 Ok(o) => Some(format!(
4803 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4804 someone may have pushed): {}",
4805 short(&theirs),
4806 o.stderr.chars().take(600).collect::<String>()
4807 )),
4808 Err(e) => Some(format!(
4809 "{branch} is rebased locally but could not be pushed: {e:#}"
4810 )),
4811 }
4812 }
4813
4814 async fn settle_already_in(
4831 &mut self,
4832 branch: &str,
4833 tip: &str,
4834 head: &str,
4835 attempts: usize,
4836 behind: usize,
4837 ) -> Result<bool> {
4838 let repo = self.state.repo.clone();
4839 let remote = self.state.config.merge.remote.clone();
4840 let start = self.state.base_commit.clone();
4841 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4842 Ok(Some(e)) => e,
4843 Ok(None) => return Ok(false),
4844 Err(e) => {
4845 tracing::warn!("already-in-base check for {branch}: {e:#}");
4846 return Ok(false);
4847 }
4848 };
4849 let mut verified = vec![head.to_owned()];
4850 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4851 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4852 if matches!(&fetched, Ok(o) if o.ok())
4853 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4854 && theirs != head
4855 {
4856 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4857 Ok(Some(_)) => verified.push(theirs),
4858 _ => return Ok(false),
4859 }
4860 }
4861 let base_branch = self.state.base_branch.clone();
4862 let message = format!(
4863 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4864 land",
4865 evidence.names(),
4866 evidence.proof.as_str()
4867 );
4868 let closed =
4869 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4870 .await;
4871 match closed {
4872 Ok(Ok(url)) => self
4873 .state
4874 .event("land", format!("closed {url}: superseded on {base_branch}")),
4875 Ok(Err(why)) => self
4876 .state
4877 .event("land", format!("did not close a pull request: {why}")),
4878 Err(e) => {
4879 let why = format!(
4880 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4881 not be closed ({e:#}); resume to retry"
4882 );
4883 self.state.status = RunStatus::Blocked;
4884 self.state.base_sync = Some(BaseSync {
4885 tip: tip.to_owned(),
4886 behind,
4887 attempts,
4888 conflict: Some(why.clone()),
4889 already_in: None,
4890 });
4891 self.state.event("land", why);
4892 self.state.save()?;
4893 return Ok(true);
4894 }
4895 }
4896 self.state.status = RunStatus::AlreadyInBase;
4897 self.state.base_sync = Some(BaseSync {
4898 tip: tip.to_owned(),
4899 behind,
4900 attempts,
4901 conflict: None,
4902 already_in: Some(evidence),
4903 });
4904 self.state.event("land", message);
4905 self.state.save()?;
4906 self.settle_questions();
4907 Ok(true)
4908 }
4909
4910 fn landing_base(&self) -> String {
4920 self.state
4921 .base_sync
4922 .as_ref()
4923 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4924 }
4925
4926 pub async fn fix_selected(
4959 &mut self,
4960 ids: &[String],
4961 reason: &str,
4962 allow_stale: bool,
4963 ) -> Result<()> {
4964 let reason = reason.trim();
4965 if reason.is_empty() {
4966 bail!("a fix request needs a reason — that is the operator's own record of why");
4967 }
4968 if ids.is_empty() {
4969 bail!("no finding id given");
4970 }
4971 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4972 bail!(
4973 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4974 has already concluded — can be given a targeted fix. A run still \
4975 in progress should simply be resumed; a `merged` run's branch has \
4976 already landed, so its answer is a fresh `magi review <branch>`, \
4977 not reopening this run's own record",
4978 self.state.id,
4979 self.state.status.as_str()
4980 );
4981 }
4982 let Some(winner) = self.state.winner().cloned() else {
4983 bail!("run {} has no winning candidate to fix", self.state.id);
4984 };
4985 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4986 bail!(
4987 "branch `{}` no longer exists; this run cannot be extended",
4988 winner.branch
4989 );
4990 }
4991 let home = crate::run::home();
4992 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
4993 bail!(
4994 "run {} is currently being worked on by another magi process",
4995 self.state.id
4996 );
4997 }
4998 let _claim = FixClaim::acquire(&self.state.dir())?;
5004
5005 let mut seen = BTreeSet::new();
5009 let mut findings = Vec::new();
5010 let mut missing = Vec::new();
5011 for id in ids {
5012 if !seen.insert(id.clone()) {
5013 continue;
5014 }
5015 match self.state.finding(id) {
5016 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5017 id: f.id.clone(),
5018 severity: f.severity,
5019 reviewer_vote: rec.vote,
5020 round: round.round,
5021 round_head: round.head.clone(),
5022 reviewer: rec.reviewer,
5023 agent: rec.agent.clone(),
5024 file: f.file.clone(),
5025 line: f.line,
5026 title: f.title.clone(),
5027 detail: f.detail.clone(),
5028 outcome: OperatorFixOutcome::Pending,
5029 }),
5030 None => missing.push(id.clone()),
5031 }
5032 }
5033 if !missing.is_empty() {
5034 bail!(
5035 "unknown finding id(s): {}; nothing was changed",
5036 missing.join(", ")
5037 );
5038 }
5039
5040 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5041 let stale_details: Vec<(String, String)> = findings
5042 .iter()
5043 .filter(|f| f.round_head != head_at_request)
5044 .map(|f| (f.id.clone(), f.round_head.clone()))
5045 .collect();
5046 let stale = !stale_details.is_empty();
5047 if stale && !allow_stale {
5048 bail!(
5049 "the branch has moved since some finding(s) were raised — {} — now \
5050 at {}; pass --allow-stale to fix anyway, or re-run review first",
5051 stale_details
5052 .iter()
5053 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5054 .collect::<Vec<_>>()
5055 .join(", "),
5056 short(&head_at_request)
5057 );
5058 }
5059
5060 let request = OperatorFixRequest {
5061 requested_at: Timestamp::now(),
5062 reason: reason.to_owned(),
5063 findings,
5064 head_at_request: head_at_request.clone(),
5065 allow_stale,
5066 stale,
5067 fix: None,
5068 result_head: None,
5069 follow_up_review_run: None,
5070 };
5071 self.state.event(
5072 "fix",
5073 format!(
5074 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5075 request.findings.len(),
5076 request
5077 .findings
5078 .iter()
5079 .map(|f| f.id.as_str())
5080 .collect::<Vec<_>>()
5081 .join(", "),
5082 ),
5083 );
5084 self.state.operator_fixes.push(request);
5091 self.state.save()?;
5092 let request_index = self.state.operator_fixes.len() - 1;
5093
5094 let repo = self.state.repo.clone();
5103 let coordination = self.coordinate(&repo).await?;
5104 if winner.worktree.exists() {
5105 let dirty = git::git(
5108 &winner.worktree,
5109 &["status", "--porcelain", "--untracked-files=all"],
5110 )
5111 .await?;
5112 let only_withheld = dirty.lines().all(|l| {
5113 l.strip_prefix("?? ")
5114 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5115 });
5116 if !only_withheld {
5117 bail!(
5118 "`{}` has uncommitted changes; refusing to touch it — commit or \
5119 discard them first",
5120 winner.worktree.display()
5121 );
5122 }
5123 git::worktree_remove(&self.state.repo, &winner.worktree)
5124 .await
5125 .ok();
5126 }
5127 let fix_worktree = self.state.worktree_root().join("operator-fix");
5128 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5129 git::git(
5130 &self.state.repo,
5131 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5132 )
5133 .await
5134 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5135 if !git::is_clean(&fix_worktree).await? {
5136 git::worktree_remove(&self.state.repo, &fix_worktree)
5137 .await
5138 .ok();
5139 bail!(
5140 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5141 winner.branch
5142 );
5143 }
5144 let repo_for_setup = self.state.repo.clone();
5145 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5146 Ok(()) => {
5147 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5148 }
5149 Err(e) => Err(e),
5150 };
5151 if let Err(e) = setup {
5152 git::worktree_remove(&self.state.repo, &fix_worktree)
5153 .await
5154 .ok();
5155 return Err(e);
5156 }
5157
5158 let run_id = self.state.id.clone();
5159 let prompts = self.state.config.prompts.clone();
5160 let language = self.state.config.graph.language.clone();
5161 let sessions = self.state.config.graph.sessions;
5162 let artifacts = agent::artifacts_dir(&self.state.dir());
5163 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5164 .findings
5165 .iter()
5166 .map(|f| Finding {
5167 id: f.id.clone(),
5168 severity: f.severity,
5169 file: f.file.clone(),
5170 line: f.line,
5171 title: f.title.clone(),
5172 detail: f.detail.clone(),
5173 })
5174 .collect();
5175 let fix_prompt = prompt::operator_fix_in(
5176 &self.state.instruction,
5177 &finding_list,
5178 reason,
5179 &stale_details,
5180 &head_at_request,
5181 &language,
5182 &self.state.config.graph.github_language,
5183 );
5184 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5185 drop(coordination);
5186 let (job, seat, out) = self
5187 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5188 prompt: fix_prompt.clone(),
5189 spec,
5190 seat,
5191 cwd: fix_worktree.clone(),
5192 timeout,
5193 allow_write: true,
5194 sessions,
5195 artifacts: artifacts.clone(),
5196 stem: "operator-fix".to_owned(),
5197 handover: None,
5198 })
5199 .await;
5200 let agent_id = seat.agent.clone();
5201
5202 let mut fix = FixRecord {
5203 agent: agent_id,
5204 addressed: Vec::new(),
5205 rejected: Vec::new(),
5206 notes: String::new(),
5207 committed: false,
5208 failed: None,
5209 duration_ms: 0,
5210 continuation: None,
5211 };
5212 let mut final_seat = seat.clone();
5213 match out {
5214 AgentOutcome::Ok(o) => {
5215 fix.duration_ms = o.duration_ms;
5216 let parsed = verdict::extract_json::<FixReport>(&o.text);
5217 let incomplete_reason = match &parsed {
5218 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5219 "the reply parsed, but it reported a command whose own CLI \
5220 never confirmed an exit status"
5221 .to_owned(),
5222 ),
5223 Ok(_) => None,
5224 Err(e) => Some(e.to_string()),
5225 };
5226 match incomplete_reason {
5227 None => {
5228 let report = parsed.expect("checked Ok above");
5229 fix.addressed = report.addressed;
5230 fix.rejected = report.rejected;
5231 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5232 }
5233 Some(reason) => {
5234 let (resumed_seat, resolved, failure, cont) = self
5235 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5236 .await;
5237 fix.duration_ms += cont.cumulative_wait_ms;
5238 fix.continuation = Some(cont);
5239 final_seat = resumed_seat;
5240 match resolved {
5241 Some(report) => {
5242 fix.addressed = report.addressed;
5243 fix.rejected = report.rejected;
5244 fix.notes =
5245 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5246 }
5247 None => fix.failed = failure,
5248 }
5249 }
5250 }
5251 }
5252 AgentOutcome::Dropped(o) => {
5253 fix.duration_ms = o.duration_ms;
5254 let why = o
5255 .dropped
5256 .as_ref()
5257 .map(|d| d.why.as_str())
5258 .unwrap_or("the CLI ended the stream without delivering its answer");
5259 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5260 }
5261 AgentOutcome::Quota(o) => {
5262 self.state.quota.push(QuotaLoss {
5263 seat: final_seat.key.clone(),
5264 node: "fix".to_owned(),
5265 at: Timestamp::now(),
5266 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5267 });
5268 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5269 }
5270 AgentOutcome::Failed(e) => fix.failed = Some(e),
5271 AgentOutcome::ResourceBlocked(e) => {
5272 fix.failed = Some(format!("the fixer was not started: {e}"));
5273 }
5274 }
5275 if fix.continuation.is_none() {
5276 fix.continuation = Some(ContinuationRecord::not_needed());
5277 }
5278 self.state.seats.insert(final_seat.key.clone(), final_seat);
5279
5280 let rescue_message = format!(
5281 "magi: operator-selected fix ({}) (uncommitted work)",
5282 self.state.operator_fixes[request_index]
5283 .findings
5284 .iter()
5285 .map(|f| f.id.as_str())
5286 .collect::<Vec<_>>()
5287 .join(", ")
5288 );
5289 let repo = self.state.repo.clone();
5290 let coordination = self.coordinate(&repo).await?;
5291 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5292 self.state.note_withheld("fix", &r.withheld);
5293 }
5294 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5295 fix.committed = after != head_at_request;
5296 git::worktree_remove(&self.state.repo, &fix_worktree)
5297 .await
5298 .ok();
5299 drop(coordination);
5300
5301 self.state.event(
5302 "fix",
5303 match &fix.failed {
5304 Some(reason) => format!(
5305 "operator fix: adoption report was lost ({reason}); {}",
5306 if fix.committed {
5307 "committed"
5308 } else {
5309 "NO new commit"
5310 }
5311 ),
5312 None => format!(
5313 "operator fix: {} addressed, {} rejected, {}",
5314 fix.addressed.len(),
5315 fix.rejected.len(),
5316 if fix.committed {
5317 "committed"
5318 } else {
5319 "NO new commit"
5320 }
5321 ),
5322 },
5323 );
5324
5325 for f in &mut self.state.operator_fixes[request_index].findings {
5332 f.outcome = if fix.failed.is_some() {
5333 OperatorFixOutcome::Unreported
5334 } else if fix.addressed.contains(&f.id) {
5335 OperatorFixOutcome::Addressed
5336 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5337 OperatorFixOutcome::Rejected { why: r.why.clone() }
5338 } else {
5339 OperatorFixOutcome::Unreported
5340 };
5341 }
5342
5343 let committed = fix.committed;
5344 if committed {
5345 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5346 }
5347 self.state.operator_fixes[request_index].fix = Some(fix);
5348 self.state.save()?;
5351
5352 if committed {
5353 self.state.event(
5354 "fix",
5355 format!(
5356 "operator fix committed {}; opening a follow-up review-only run",
5357 short(&after)
5358 ),
5359 );
5360 let origin =
5363 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5364 match Self::review(
5365 &self.state.repo,
5366 &winner.branch,
5367 self.state.config.clone(),
5368 origin,
5369 )
5370 .await
5371 {
5372 Ok(mut follow_up) => {
5373 follow_up.state.event(
5374 "start",
5375 format!(
5376 "requested by an operator fix on run {} for finding(s) {}",
5377 self.state.id,
5378 self.state.operator_fixes[request_index]
5379 .findings
5380 .iter()
5381 .map(|f| f.id.as_str())
5382 .collect::<Vec<_>>()
5383 .join(", "),
5384 ),
5385 );
5386 follow_up.state.save()?;
5387 let follow_up_id = follow_up.state.id.clone();
5388 let adopted = match crate::direct::adopt(
5391 &follow_up.state,
5392 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5393 ) {
5394 Ok(a) => a,
5395 Err(e) => {
5396 self.state.event(
5399 "fix",
5400 format!(
5401 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5402 ),
5403 );
5404 self.state.save()?;
5405 return Ok(());
5406 }
5407 };
5408 let executed = follow_up.execute().await;
5409 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5410 if let Some(a) = adopted {
5411 a.finish(&follow_up.state, executed);
5412 }
5413 if let Some(e) = failure {
5414 self.state.event(
5415 "fix",
5416 format!(
5417 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5418 ),
5419 );
5420 }
5421 self.state.operator_fixes[request_index].follow_up_review_run =
5422 Some(follow_up_id);
5423 }
5424 Err(e) => {
5425 self.state.event(
5426 "fix",
5427 format!("committed the fix but could not open a follow-up review: {e:#}"),
5428 );
5429 }
5430 }
5431 self.state.save()?;
5432 }
5433
5434 Ok(())
5435 }
5436
5437 async fn ask_fixer(
5457 &mut self,
5458 winner: &Candidate,
5459 node: &'static str,
5460 round: Option<usize>,
5461 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5462 ) -> (SeatJob, SeatState, AgentOutcome) {
5463 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5464 self.ask_fixer_among(attempts, node, round, None, build)
5465 .await
5466 }
5467
5468 async fn ask_fixer_among(
5473 &mut self,
5474 attempts: Vec<(AgentSpec, String)>,
5475 node: &'static str,
5476 round: Option<usize>,
5477 restore: Option<(&Path, &str)>,
5478 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5479 ) -> (SeatJob, SeatState, AgentOutcome) {
5480 let run_id = self.state.id.clone();
5481 let prompts = self.state.config.prompts.clone();
5482 let cache = self.state.config.cache_dir();
5483 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5484 let mut last = None;
5485 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5486 let seat = self.seat(&key, &spec.id);
5487 let mut job = build(spec.clone(), seat);
5488 if i > 0 {
5489 job.stem = format!("{}-{}", job.stem, spec.id);
5490 }
5491 let ctx = WaveCtx {
5492 carry_seats: false,
5493 pauses: Some((&self.pause, &self.interrupt)),
5494 run: &run_id,
5495 node,
5496 prompts: &prompts,
5497 cache: cache.as_deref(),
5498 round,
5499 };
5500 let (seat, out) =
5501 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5502 let advances = match &out {
5505 AgentOutcome::Ok(o) => agent::output_advances(o),
5506 AgentOutcome::ResourceBlocked(_) => false,
5507 _ => true,
5508 };
5509 if let Some(next) = ids.get(i + 1)
5510 && advances
5511 {
5512 self.state.seats.insert(seat.key.clone(), seat.clone());
5513 let class =
5514 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5515 let reason = match &out {
5516 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5517 other => fail_reason(other),
5518 };
5519 if let Some((wt, rev)) = restore
5523 && let Err(e) = self.graft_restore(wt, rev).await
5524 {
5525 let why =
5526 format!("could not restore the worktree before the next fixer: {e:#}");
5527 tracing::warn!("{why}");
5528 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5529 AgentOutcome::ResourceBlocked(why)
5530 } else {
5531 AgentOutcome::Failed(why)
5532 };
5533 last = Some((job, seat, out));
5534 break;
5535 }
5536 record_handover(
5537 &mut self.state,
5538 node,
5539 &seat.key,
5540 &spec.id,
5541 next,
5542 &class,
5543 &reason,
5544 );
5545 continue;
5546 }
5547 last = Some((job, seat, out));
5548 break;
5549 }
5550 last.expect("the fixer chain always has an entry")
5551 }
5552
5553 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5564 use crate::recovery::{Mode, Repair};
5565 if !self.state.reviews.is_empty() {
5566 return Ok(());
5567 }
5568 let Some(mut rec) = self.state.recovery.clone() else {
5569 return Ok(());
5570 };
5571 if rec.mode != Mode::VerifyRepair {
5572 return Ok(());
5573 }
5574 let wt = winner.worktree.clone();
5575 let mut head = git::rev_parse(&wt, "HEAD").await?;
5576 match rec.repair.clone() {
5577 Repair::Done { .. } => return Ok(()),
5578 Repair::WaitingForRestore { before } => {
5579 self.graft_restore(&wt, &before).await?;
5580 head = before;
5581 rec.repair = Repair::NotStarted;
5582 self.state.recovery = Some(rec.clone());
5583 self.state.save()?;
5584 }
5585 Repair::Started { before } => {
5586 if let Err(e) = self.graft_restore(&wt, &before).await {
5590 if e.is::<crate::repo_lock::ResourceBlocked>() {
5591 return Err(e);
5592 }
5593 return self
5594 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5595 .await;
5596 }
5597 rec.repair = Repair::Done {
5598 before: before.clone(),
5599 after: before,
5600 committed: false,
5601 failed: Some("interrupted; rolled back".to_owned()),
5602 };
5603 self.state
5604 .event("recovery", "the repair was interrupted and rolled back");
5605 self.state.recovery = Some(rec);
5606 self.state.save()?;
5607 return Ok(());
5608 }
5609 Repair::NotStarted => {}
5610 }
5611 if crate::recovery::recheck(&mut rec, &head) {
5612 self.state.event("recovery", rec.summary());
5613 self.state.recovery = Some(rec);
5614 self.state.save()?;
5615 return Ok(());
5616 }
5617 rec.repair = Repair::Started {
5618 before: head.clone(),
5619 };
5620 self.state.recovery = Some(rec.clone());
5621 self.state.save()?;
5622
5623 let language = self.state.config.graph.language.clone();
5624 let sessions = self.state.config.graph.sessions;
5625 let artifacts = agent::artifacts_dir(&self.state.dir());
5626 let fix_prompt = prompt::recovery_fix(
5627 &self.state.instruction,
5628 &rec,
5629 self.state.config.graph.review_rounds,
5630 &language,
5631 );
5632 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5633 self.state.event(
5634 "recovery",
5635 format!(
5636 "repair: the fixer starts from the failed verification on {}",
5637 short(&head)
5638 ),
5639 );
5640 let (_job, seat, out) = self
5641 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5642 prompt: fix_prompt.clone(),
5643 spec,
5644 seat,
5645 cwd: wt.clone(),
5646 timeout,
5647 allow_write: true,
5648 sessions,
5649 artifacts: artifacts.clone(),
5650 stem: "fix-recovery".to_owned(),
5651 handover: None,
5652 })
5653 .await;
5654 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5655 rec.repair = Repair::WaitingForRestore { before: head };
5656 self.state.recovery = Some(rec);
5657 return self.stop_for_resource();
5658 }
5659 let failed = match &out {
5660 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5661 AgentOutcome::Quota(o) => {
5662 self.state.quota.push(QuotaLoss {
5663 seat: seat.key.clone(),
5664 node: "fix".to_owned(),
5665 at: Timestamp::now(),
5666 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5667 });
5668 Some("rate limited (quota); the fixer could not run".to_owned())
5669 }
5670 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5671 AgentOutcome::Failed(e) => Some(e.clone()),
5672 };
5673 self.state.seats.insert(seat.key.clone(), seat);
5674 let coordination = self.coordinate(&wt).await?;
5675 let failed = match failed {
5676 Some(why) => {
5678 if let Err(e) = git::restore_to(&wt, &head).await {
5679 return self
5680 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5681 .await;
5682 }
5683 Some(why)
5684 }
5685 None => {
5686 if let Ok(r) = git::rescue_commit(
5687 &wt,
5688 "magi: recovery repair of the failed verification (uncommitted work)",
5689 )
5690 .await
5691 {
5692 self.state.note_withheld("fix", &r.withheld);
5693 }
5694 None
5695 }
5696 };
5697 let after = git::rev_parse(&wt, "HEAD").await?;
5698 drop(coordination);
5699 self.state.event(
5700 "recovery",
5701 match &failed {
5702 Some(why) => format!("repair: nothing kept ({why})"),
5703 None if after != head => {
5704 format!("repair: committed {}", short(&after))
5705 }
5706 None => "repair: the fixer made no commit".to_owned(),
5707 },
5708 );
5709 rec.repair = Repair::Done {
5710 before: head.clone(),
5711 committed: after != head,
5712 after,
5713 failed,
5714 };
5715 self.state.recovery = Some(rec);
5716 self.state.save()?;
5717 Ok(())
5718 }
5719
5720 async fn stop_recovery(
5723 &mut self,
5724 mut rec: crate::recovery::RecoveryRecord,
5725 head: &str,
5726 why: String,
5727 ) -> Result<()> {
5728 rec.repair = crate::recovery::Repair::Started {
5730 before: head.to_owned(),
5731 };
5732 self.state.recovery = Some(rec);
5733 self.state
5734 .event("recovery", format!("repair could not be undone: {why}"));
5735 self.state.status = RunStatus::Blocked;
5736 self.state.save()?;
5737 bail!("recovery repair could not be rolled back: {why}")
5738 }
5739
5740 async fn review_loop(&mut self) -> Result<()> {
5741 if self
5746 .state
5747 .base_sync
5748 .as_ref()
5749 .is_some_and(|s| s.conflict.is_some())
5750 {
5751 return Ok(());
5752 }
5753 let run_id = self.state.id.clone();
5758 let prompts = self.state.config.prompts.clone();
5759 let Some(winner) = self.state.winner().cloned() else {
5760 return Ok(());
5761 };
5762 let max_rounds = self.state.config.graph.review_rounds;
5763 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5773 if status == RunStatus::Gating {
5776 self.record_contested_handoff();
5777 }
5778 self.state.status = status;
5779 self.state.save()?;
5780 return Ok(());
5781 }
5782 self.state.status = RunStatus::Reviewing;
5783 if self
5793 .state
5794 .reviews
5795 .last()
5796 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5797 {
5798 let shell = self.state.config.shell();
5799 return self
5800 .stop_reviewing(
5801 "the last round's own verification never resolved",
5802 &shell,
5803 &winner.worktree,
5804 )
5805 .await;
5806 }
5807
5808 self.recovery_repair(&winner).await?;
5809 if self.resource_halt()? {
5810 return Ok(());
5811 }
5812
5813 let repo = self.state.repo.clone();
5814 let root = self.state.worktree_root();
5815 let language = self.state.config.graph.language.clone();
5816 let sessions = self.state.config.graph.sessions;
5817 let artifacts = self.merge_repair_artifacts();
5818 let base = self.landing_base();
5819 let base_short = short(&base);
5820 let reviewers = self.roles.reviewers.clone();
5821 let shell = self.state.config.shell();
5822
5823 for round in (self.state.reviews.len() + 1)..=max_rounds {
5824 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5825 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5826 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5827 let prev_verification = self
5836 .state
5837 .reviews
5838 .last()
5839 .and_then(|r| r.verification_summary(&head));
5840 let prior_section = match (&self.state.recovery, round) {
5843 (Some(rec), 1) if self.state.reviews.is_empty() => {
5844 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5845 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5846 Some(crate::recovery::head_of(
5847 &git::diff(&winner.worktree, reviewed, "HEAD")
5848 .await
5849 .unwrap_or_default(),
5850 crate::recovery::SINCE_DIFF_MAX,
5851 ))
5852 }
5853 _ => None,
5854 };
5855 Some(prompt::recovery_review(rec, since.as_deref()))
5856 }
5857 _ => None,
5858 };
5859
5860 let mut jobs = Vec::new();
5864 let coordination = self.coordinate(&repo).await?;
5865 for (r, spec) in reviewers.iter().cloned().enumerate() {
5866 let wt = root.join(format!("review-{}", r + 1));
5867 if wt.exists() {
5870 git::reset_detached(&wt, &head).await?;
5871 } else {
5872 git::worktree_add_detached(&repo, &wt, &head).await?;
5873 }
5874 self.setup_seat_worktree(&repo, &wt, false).await?;
5875 let seat_key = format!("review-{}", r + 1);
5876 let spec = pick_start_spec(
5879 &self.roles.reviewer_roster,
5880 spec,
5881 self.state.seat_history.get(&seat_key),
5882 );
5883 let seat = self.seat(&seat_key, &spec.id);
5884 jobs.push(SeatJob {
5885 prompt: prompt::review(&prompt::ReviewCtx {
5886 instruction: &self.state.instruction,
5887 branch: &winner.branch,
5888 base_short: &base_short,
5889 stat: &stat,
5890 patch: &patch,
5891 verification: prev_verification.as_ref(),
5892 prior: prior_section.as_deref(),
5893 reviewers: reviewers.len(),
5894 round,
5895 rounds: max_rounds,
5896 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5899 lens: Lens::for_seat(r),
5900 language: &language,
5901 github_language: &self.state.config.graph.github_language,
5902 }),
5903 spec,
5904 seat,
5905 cwd: wt,
5906 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5907 allow_write: false,
5908 sessions,
5909 artifacts: artifacts.clone(),
5910 stem: format!("review-{round}-{}", r + 1),
5911 handover: None,
5912 });
5913 }
5914
5915 drop(coordination);
5916 self.state.event(
5917 "review",
5918 format!(
5919 "round {round}: {} reviewers on {}",
5920 jobs.len(),
5921 short(&head)
5922 ),
5923 );
5924 let mut quota_losses = Vec::new();
5925 let review_retries = self.state.config.graph.retries;
5926 let review_cache = self.state.config.cache_dir();
5927 let ctx = WaveCtx {
5928 carry_seats: true,
5929 pauses: Some((&self.pause, &self.interrupt)),
5930 run: &run_id,
5931 node: "review",
5932 prompts: &prompts,
5933 cache: review_cache.as_deref(),
5934 round: Some(round),
5935 };
5936 let results = ask_json_wave::<Review>(
5937 jobs,
5938 Arc::clone(&self.sem),
5939 review_retries,
5940 &self.roles.reviewer_roster,
5941 &ctx,
5942 &mut quota_losses,
5943 &mut self.state,
5944 &|_: &Review| Ok(()),
5945 )
5946 .await;
5947 if self.state.resource_block.is_some() {
5948 return self.stop_for_resource();
5949 }
5950 let round_quota_missing = quota_losses.len();
5954 self.state.quota.extend(quota_losses);
5955
5956 let mut records = Vec::new();
5957 let mut all_findings = Vec::new();
5958 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5959 let agent_id = seat.agent.clone();
5960 self.state.seats.insert(seat.key.clone(), seat);
5961 let mut record = ReviewRecord {
5962 reviewer: r + 1,
5963 agent: agent_id,
5964 summary: String::new(),
5965 findings: Vec::new(),
5966 vote: None,
5967 failed: None,
5968 duration_ms: 0,
5969 attempts,
5975 };
5976 match res {
5977 Ok((review, out)) => {
5978 record.summary =
5986 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5987 record.vote = Some(review.vote);
5988 record.duration_ms = out.duration_ms;
5989 for (n, mut f) in review.findings.into_iter().enumerate() {
5990 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
5993 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
5994 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
5995 f.file = f
6001 .file
6002 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6003 all_findings.push(f.clone());
6004 record.findings.push(f);
6005 }
6006 self.state.event(
6007 "review",
6008 format!(
6009 "round {round}: reviewer {} voted {} with {} finding(s)",
6010 r + 1,
6011 review.vote.label(),
6012 record.findings.len()
6013 ),
6014 );
6015 }
6016 Err(e) => {
6017 record.failed = Some(e.to_string());
6018 self.state.event(
6019 "review",
6020 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6021 );
6022 }
6023 }
6024 records.push(record);
6025 }
6026
6027 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6034 let vote_split =
6035 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6036 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6037 if vote_split {
6038 self.state.event(
6039 "review",
6040 format!(
6041 "round {round}: votes split ({}) — one round of reconsideration",
6042 initial_votes
6043 .iter()
6044 .map(|v| v.label())
6045 .collect::<Vec<_>>()
6046 .join(", ")
6047 ),
6048 );
6049 let panel: Vec<ReviewSeatReport<'_>> = records
6052 .iter()
6053 .filter_map(|r| {
6054 r.vote.map(|vote| ReviewSeatReport {
6055 reviewer: r.reviewer,
6056 vote,
6057 summary: &r.summary,
6058 findings: &r.findings,
6059 })
6060 })
6061 .collect();
6062
6063 let mut jobs = Vec::new();
6064 let mut seats_at = Vec::new();
6065 for (r, spec) in reviewers.iter().cloned().enumerate() {
6066 if records[r].vote.is_none() {
6070 continue;
6071 }
6072 let wt = root.join(format!("review-{}", r + 1));
6073 let seat_key = format!("review-{}", r + 1);
6074 let spec = self.occupant(&seat_key, spec);
6075 let seat = self.seat(&seat_key, &spec.id);
6076 let build = |with_patch: bool| {
6084 prompt::review_reconsider(&ReviewReconsiderCtx {
6085 instruction: &self.state.instruction,
6086 reviewer: r + 1,
6087 lens: Lens::for_seat(r),
6088 panel: &panel,
6089 patch: with_patch.then_some(ReviewPatch {
6090 branch: &winner.branch,
6091 base_short: &base_short,
6092 stat: &stat,
6093 patch: &patch,
6094 }),
6095 round,
6096 rounds: max_rounds,
6097 language: &language,
6098 })
6099 };
6100 let full = build(true);
6101 let prompt = if has_context(&spec, &seat, sessions) {
6102 build(false)
6103 } else {
6104 full.clone()
6105 };
6106 jobs.push(SeatJob {
6107 prompt,
6108 spec,
6109 seat,
6110 cwd: wt,
6111 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6112 allow_write: false,
6113 sessions,
6114 artifacts: artifacts.clone(),
6115 stem: format!("review-{round}-reconsider-{}", r + 1),
6116 handover: Some(full),
6117 });
6118 seats_at.push(r);
6119 }
6120
6121 let mut recon_quota_losses = Vec::new();
6122 let recon_cache = self.state.config.cache_dir();
6123 let recon_ctx = WaveCtx {
6124 carry_seats: true,
6125 pauses: Some((&self.pause, &self.interrupt)),
6126 run: &run_id,
6127 node: "review",
6128 prompts: &prompts,
6129 cache: recon_cache.as_deref(),
6130 round: Some(round),
6131 };
6132 let recon_results = ask_json_wave::<ReviewRevote>(
6133 jobs,
6134 Arc::clone(&self.sem),
6135 review_retries,
6136 &self.roles.reviewer_roster,
6137 &recon_ctx,
6138 &mut recon_quota_losses,
6139 &mut self.state,
6140 &|_: &ReviewRevote| Ok(()),
6141 )
6142 .await;
6143 if self.state.resource_block.is_some() {
6144 return self.stop_for_resource();
6145 }
6146 self.state.quota.extend(recon_quota_losses);
6147
6148 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6149 let agent_id = seat.agent.clone();
6150 self.state.seats.insert(seat.key.clone(), seat);
6151 let mut rec = ReviewRevoteRecord {
6152 reviewer: r + 1,
6153 agent: agent_id,
6154 vote: None,
6155 reason: String::new(),
6156 failed: None,
6157 };
6158 match res {
6159 Ok((rv, _)) => {
6160 rec.vote = Some(rv.vote);
6161 rec.reason =
6162 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6163 self.state.event(
6164 "review",
6165 format!(
6166 "round {round}: reviewer {} revoted {}",
6167 r + 1,
6168 rv.vote.label()
6169 ),
6170 );
6171 }
6172 Err(e) => {
6173 rec.failed = Some(e.to_string());
6174 self.state.event(
6175 "review",
6176 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6177 );
6178 }
6179 }
6180 reconsideration.push(rec);
6181 }
6182 } else if initial_votes.len() > 1 {
6183 self.state.event(
6184 "review",
6185 format!(
6186 "round {round}: votes agreed ({}) — no reconsideration",
6187 initial_votes[0].label()
6188 ),
6189 );
6190 }
6191
6192 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6193 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6194 let defer_e2e =
6205 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6206 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6207 let reason =
6208 format!("{blocking} blocking finding(s) already required a fix this round");
6209 self.state.event(
6210 "verify",
6211 format!(
6212 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6213 {}); it will run once a round has none left",
6214 short(&head)
6215 ),
6216 );
6217 (Vec::new(), false, true, Some(reason))
6218 } else {
6219 let e2e_commands = self.state.config.verify.e2e.clone();
6220 let cache_dir = self.state.config.cache_dir();
6221 let context = format!("round {round}");
6222 let wait = self.repository_wait();
6223 let (e2e, verify_retried) = with_cache_lease(
6224 &mut self.state,
6225 cache_dir.as_deref(),
6226 "e2e",
6227 "e2e",
6228 &winner.worktree,
6229 &head,
6230 verify_timeout,
6231 &context,
6232 &wait,
6233 |state, budget| {
6234 let shell = shell.clone();
6235 let e2e_commands = e2e_commands.clone();
6236 let worktree = winner.worktree.clone();
6237 let context = context.clone();
6238 async move {
6239 run_e2e_with_retry(
6240 state,
6241 &shell,
6242 &e2e_commands,
6243 &worktree,
6244 budget,
6245 &context,
6246 )
6247 .await
6248 }
6249 },
6250 )
6251 .await?;
6252 (e2e, verify_retried, false, None)
6253 };
6254
6255 let expected = records.len();
6256 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6257 let incomplete = answered < expected;
6258 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6259 let policy = self.state.config.graph.incomplete_review;
6260 let clean = round_is_clean(
6261 blocking,
6262 e2e_ok,
6263 answered,
6264 expected,
6265 round_quota_missing,
6266 policy,
6267 );
6268
6269 let mut round_record = ReviewRound {
6270 round,
6271 head: head.clone(),
6272 verified_head: None,
6273 verified_at: None,
6274 reviews: records,
6275 e2e,
6276 verify_retried,
6277 e2e_deferred,
6278 e2e_defer_reason,
6279 fix: None,
6280 blocking,
6281 answered,
6282 expected,
6283 clean,
6284 progressed: false,
6285 vote_split,
6286 reconsideration,
6287 verdict: None,
6288 };
6289 round_record.verdict = ReviewVote::worst(
6293 round_record
6294 .final_votes()
6295 .into_iter()
6296 .map(|(_, _, vote)| vote),
6297 );
6298 if !matches!(
6309 round_record.e2e_status(),
6310 E2eStatus::Deferred | E2eStatus::NotConfigured
6311 ) {
6312 round_record.verified_head = Some(head.clone());
6313 round_record.verified_at = Some(Timestamp::now());
6314 }
6315 let this_round_verification = round_record.verification_summary(&head);
6316
6317 if incomplete {
6318 let missing: Vec<String> = round_record
6319 .reviews
6320 .iter()
6321 .filter(|r| r.failed.is_some())
6322 .map(|r| format!("review-{}", r.reviewer))
6323 .collect();
6324 self.state.event(
6325 "review",
6326 format!(
6327 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6328 missing.join(", ")
6329 ),
6330 );
6331 }
6332
6333 if clean {
6334 self.state.event(
6335 "review",
6336 if incomplete && policy == IncompleteReviewPolicy::Warn {
6337 format!(
6338 "round {round}: clean (warn policy, incomplete panel) — no \
6339 blocking findings from the seats that answered, verification green"
6340 )
6341 } else if incomplete {
6342 format!(
6343 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6344 quorum) — no blocking findings from the seats that answered, \
6345 verification green",
6346 expected - answered
6347 )
6348 } else {
6349 format!("round {round}: clean — no blocking findings, verification green")
6350 },
6351 );
6352 self.state.reviews.push(round_record);
6353 self.state.status = RunStatus::Gating;
6354 self.state.save()?;
6355 return Ok(());
6356 }
6357
6358 if incomplete && blocking == 0 && e2e_ok {
6366 self.state.reviews.push(round_record);
6367 self.state.save()?;
6368 if round == max_rounds {
6369 self.state.status = RunStatus::Blocked;
6370 self.state.event(
6371 "review",
6372 format!(
6373 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6374 refusing to call it clean",
6375 expected - answered
6376 ),
6377 );
6378 return Ok(());
6379 }
6380 continue;
6381 }
6382
6383 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6395 self.state.reviews.push(round_record);
6396 return self
6397 .stop_reviewing(
6398 "the round's own verification could not run",
6399 &shell,
6400 &winner.worktree,
6401 )
6402 .await;
6403 }
6404
6405 if round == max_rounds {
6406 self.state.reviews.push(round_record);
6407 return self
6408 .stop_reviewing(
6409 &format!(
6410 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6411 ),
6412 &shell,
6413 &winner.worktree,
6414 )
6415 .await;
6416 }
6417
6418 let blocking_findings: Vec<_> = all_findings
6421 .iter()
6422 .filter(|f| f.severity.blocks())
6423 .cloned()
6424 .collect();
6425 let fix_prompt = prompt::fix_in(
6426 &self.state.instruction,
6427 &blocking_findings,
6428 this_round_verification.as_ref(),
6429 round,
6430 max_rounds,
6431 &language,
6432 &self.state.config.graph.github_language,
6433 );
6434 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6435 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6436 let (job, seat, out) = self
6437 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6438 prompt: fix_prompt.clone(),
6439 spec,
6440 seat,
6441 cwd: winner.worktree.clone(),
6442 timeout,
6443 allow_write: true,
6444 sessions,
6445 artifacts: artifacts.clone(),
6446 stem: format!("fix-{round}"),
6447 handover: None,
6448 })
6449 .await;
6450 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6453 return self.stop_for_resource();
6454 }
6455 let agent_id = seat.agent.clone();
6456
6457 let mut fix = FixRecord {
6458 agent: agent_id,
6459 addressed: Vec::new(),
6460 rejected: Vec::new(),
6461 notes: String::new(),
6462 committed: false,
6463 failed: None,
6464 duration_ms: 0,
6465 continuation: None,
6466 };
6467 let mut continuation = ContinuationRecord::not_needed();
6468 let mut final_seat = seat.clone();
6469 match out {
6470 AgentOutcome::Ok(o) => {
6471 fix.duration_ms = o.duration_ms;
6472 let parsed = verdict::extract_json::<FixReport>(&o.text);
6473 let incomplete_reason = match &parsed {
6480 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6481 "the reply parsed, but it reported a command whose own CLI never \
6482 confirmed an exit status"
6483 .to_owned(),
6484 ),
6485 Ok(_) => None,
6486 Err(e) => Some(e.to_string()),
6487 };
6488 match incomplete_reason {
6489 None => {
6490 let report = parsed.expect("checked Ok above");
6491 fix.addressed = report.addressed;
6492 fix.rejected = report.rejected;
6493 fix.notes =
6494 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6495 }
6496 Some(reason) => {
6497 let (resumed_seat, resolved, failure, cont) = self
6498 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6499 .await;
6500 fix.duration_ms += cont.cumulative_wait_ms;
6501 continuation = cont;
6502 final_seat = resumed_seat;
6503 match resolved {
6504 Some(report) => {
6505 fix.addressed = report.addressed;
6506 fix.rejected = report.rejected;
6507 fix.notes = blind::sanitize_prose(
6508 &report.notes,
6509 &self.state.config.blind,
6510 );
6511 }
6512 None => fix.failed = failure,
6513 }
6514 }
6515 }
6516 }
6517 AgentOutcome::Dropped(o) => {
6519 fix.duration_ms = o.duration_ms;
6520 let why = o
6521 .dropped
6522 .as_ref()
6523 .map(|d| d.why.as_str())
6524 .unwrap_or("the CLI ended the stream without delivering its answer");
6525 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6526 }
6527 AgentOutcome::Quota(o) => {
6528 self.state.quota.push(QuotaLoss {
6529 seat: final_seat.key.clone(),
6530 node: "fix".to_owned(),
6531 at: Timestamp::now(),
6532 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6533 });
6534 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6535 }
6536 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6537 }
6538 fix.continuation = Some(continuation);
6539 self.state.seats.insert(final_seat.key.clone(), final_seat);
6540 let coordination = self.coordinate(&winner.worktree).await?;
6541 if let Ok(r) = git::rescue_commit(
6542 &winner.worktree,
6543 &format!("magi: review round {round} fixes (uncommitted work)"),
6544 )
6545 .await
6546 {
6547 self.state.note_withheld("fix", &r.withheld);
6548 }
6549 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6550 fix.committed = after != before;
6551 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6559 drop(coordination);
6560 let progressed = diff_after != patch;
6561 let commit_note = if fix.committed {
6562 "committed"
6563 } else {
6564 "NO new commit"
6565 };
6566 let tree_note = if progressed {
6567 "changed vs base"
6568 } else {
6569 "unchanged vs base"
6570 };
6571 self.state.event(
6572 "fix",
6573 match &fix.failed {
6574 Some(reason) => {
6580 format!(
6581 "round {round}: fixer's adoption report was lost ({reason}); \
6582 {commit_note}, tree {tree_note}"
6583 )
6584 }
6585 None => format!(
6586 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6587 {tree_note}{}",
6588 fix.addressed.len(),
6589 fix.rejected.len(),
6590 if continuation.outcome == ContinuationOutcome::Resumed {
6591 format!(
6592 " (adoption report recovered after {} continuation(s))",
6593 continuation.attempts
6594 )
6595 } else {
6596 String::new()
6597 },
6598 ),
6599 },
6600 );
6601 round_record.fix = Some(fix);
6602 round_record.progressed = progressed;
6603 self.state.reviews.push(round_record);
6604 self.state.save()?;
6605
6606 if matches!(
6619 continuation.outcome,
6620 ContinuationOutcome::Exhausted
6621 | ContinuationOutcome::QuotaLost
6622 | ContinuationOutcome::NoSession
6623 ) {
6624 return self
6625 .stop_reviewing(
6626 "the fixer's adoption report never came back, even after resuming its \
6627 own seat; refusing to start another round against the same worktree \
6628 while that is unresolved",
6629 &shell,
6630 &winner.worktree,
6631 )
6632 .await;
6633 }
6634
6635 let streak = self
6636 .state
6637 .reviews
6638 .iter()
6639 .rev()
6640 .take_while(|r| !r.progressed)
6641 .count();
6642 if streak >= STAGNANT_LIMIT {
6643 return self
6644 .stop_reviewing(
6645 &format!(
6646 "the tree has not moved against base for {streak} round(s) in a row"
6647 ),
6648 &shell,
6649 &winner.worktree,
6650 )
6651 .await;
6652 }
6653 }
6654 Ok(())
6655 }
6656
6657 fn record_contested_handoff(&mut self) {
6691 if self.state.contested_handoff.is_some() {
6692 return;
6693 }
6694 let Some(contested) = self
6695 .state
6696 .reviews
6697 .last()
6698 .and_then(ReviewRound::contested_handoff)
6699 else {
6700 return;
6701 };
6702 self.state.event(
6703 "review",
6704 format!(
6705 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6706 wait for the owner's approval",
6707 contested.findings.len(),
6708 contested.rejecters.len()
6709 ),
6710 );
6711 self.state.contested_handoff = Some(contested);
6712 }
6713
6714 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6715 let round_idx = self.state.reviews.len() - 1;
6716 let needs_catchup_run = matches!(
6724 self.state.reviews[round_idx].e2e_status(),
6725 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6726 );
6727 if needs_catchup_run {
6728 let round = self.state.reviews[round_idx].round;
6729 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6730 let commands = self.state.config.verify.e2e.clone();
6731 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6732 let cache_dir = self.state.config.cache_dir();
6733 let context = format!(
6734 "round {round}: verification unresolved, catching up before the final decision"
6735 );
6736 let wait = self.repository_wait();
6737 let (outcomes, verify_retried) = with_cache_lease(
6738 &mut self.state,
6739 cache_dir.as_deref(),
6740 "e2e",
6741 "e2e",
6742 worktree,
6743 &attempted_head,
6744 timeout,
6745 &context,
6746 &wait,
6747 |state, budget| {
6748 let shell = shell.to_vec();
6749 let commands = commands.clone();
6750 let context = context.clone();
6751 async move {
6752 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6753 .await
6754 }
6755 },
6756 )
6757 .await?;
6758 let last = &mut self.state.reviews[round_idx];
6759 last.e2e = outcomes;
6760 last.verify_retried = verify_retried;
6761 last.verified_head = Some(attempted_head);
6768 last.verified_at = Some(Timestamp::now());
6769 if verify_inconclusive(&last.e2e) {
6770 self.state.save()?;
6777 return Ok(());
6778 }
6779 last.e2e_deferred = false;
6780 }
6781 let last = &self.state.reviews[round_idx];
6782 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6783
6784 match last.e2e_status() {
6785 E2eStatus::Failed => {
6786 let red: Vec<String> = last
6787 .e2e
6788 .iter()
6789 .filter(|o| !o.ok())
6790 .map(|o| {
6791 format!(
6792 "`{}` -> {:?}\n{}",
6793 o.command,
6794 o.code,
6795 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6796 )
6797 })
6798 .collect();
6799 self.state
6800 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6801 self.state.status = RunStatus::Blocked;
6802 }
6803 E2eStatus::ResourceBlocked => {
6808 self.state.event(
6809 "review",
6810 format!(
6811 "{why}; e2e could not run (shared build cache unavailable); not \
6812 deciding yet"
6813 ),
6814 );
6815 }
6816 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6817 self.state.event(
6818 "review",
6819 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6820 );
6821 self.record_contested_handoff();
6822 self.state.status = RunStatus::Gating;
6823 }
6824 }
6825 self.state.save()?;
6826 Ok(())
6827 }
6828
6829 async fn gate(&mut self) -> Result<()> {
6832 if self.state.status == RunStatus::Failed
6844 || self
6845 .state
6846 .base_sync
6847 .as_ref()
6848 .is_some_and(|s| s.conflict.is_some())
6849 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6850 != Some(RunStatus::Gating)
6851 {
6852 return Ok(());
6853 }
6854 if self.state.gate_ran {
6855 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6866 self.state.status = RunStatus::Blocked;
6867 self.state.save()?;
6868 }
6869 return Ok(());
6870 }
6871 let Some(winner) = self.state.winner().cloned() else {
6872 return Ok(());
6873 };
6874 self.state.status = RunStatus::Gating;
6875 let mut outcomes = self.run_gate(&winner).await?;
6876 loop {
6877 if verify_inconclusive(&outcomes) {
6888 self.state.save()?;
6889 return Ok(());
6890 }
6891 if outcomes.iter().all(CommandOutcome::ok) {
6892 break;
6893 }
6894 match self.gate_fix_round(&winner, &outcomes).await? {
6895 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6896 GateFix::Stop => break,
6897 GateFix::Defer => {
6898 self.state.save()?;
6899 return Ok(());
6900 }
6901 }
6902 }
6903 let passed = outcomes.iter().all(CommandOutcome::ok);
6904 self.state.gate = outcomes;
6905 self.state.gate_ran = true;
6906 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6907 if !passed {
6908 self.state.status = RunStatus::Blocked;
6909 let spent = self.state.gate_fixes.len();
6910 self.state.event(
6911 "gate",
6912 if spent == 0 {
6913 "gate failed; not merging".to_owned()
6914 } else {
6915 format!("gate failed after {spent} gate-fix round(s); not merging")
6916 },
6917 );
6918 }
6919 self.state.save()?;
6920 Ok(())
6921 }
6922
6923 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6933 let commands = state.config.verify.pre_gate.clone();
6934 if commands.is_empty() {
6935 return;
6936 }
6937 let shell = state.config.shell();
6938 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6939 let (outcomes, _) = run_commands(
6940 state,
6941 "pre_gate",
6942 "pre_gate",
6943 0,
6944 &shell,
6945 &commands,
6946 &winner.worktree,
6947 timeout,
6948 )
6949 .await;
6950 for o in &outcomes {
6951 if !o.ok() {
6952 tracing::warn!(
6953 "pre_gate `{}` failed ({:?}); the gate decides",
6954 o.command,
6955 o.code
6956 );
6957 }
6958 state.event(
6959 "pre_gate",
6960 format!(
6961 "`{}` -> {}",
6962 o.command,
6963 if o.ok() {
6964 "pass".to_owned()
6965 } else {
6966 format!(
6967 "FAIL ({:?})\n{}",
6968 o.code,
6969 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6970 )
6971 }
6972 ),
6973 );
6974 }
6975 state.pre_gate = outcomes;
6976 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6977 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6978 Ok(head) => {
6979 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6980 state.pre_gate_commit = Some(head);
6981 }
6982 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6983 },
6984 Ok(false) => {}
6985 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6986 }
6987 if let Err(e) = state.save() {
6988 tracing::warn!("could not persist the pre_gate record: {e:#}");
6989 }
6990 }
6991
6992 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
6995 let shell = self.state.config.shell();
6996 let gate_commands = self.state.config.verify.gate.clone();
6997 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
6998 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
6999 Vec::new()
7000 } else {
7001 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7002 let cache_dir = self.state.config.cache_dir();
7003 let wait = self.repository_wait();
7004 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7005 let (outcomes, _) = with_cache_lease(
7006 &mut self.state,
7007 cache_dir.as_deref(),
7008 "gate",
7009 "gate",
7010 &winner.worktree,
7011 &head,
7012 timeout,
7013 "final gate",
7014 &wait,
7015 |state, budget| {
7016 let shell = shell.clone();
7017 let gate_commands = gate_commands.clone();
7018 let winner = winner.clone();
7019 async move {
7020 Self::run_pre_gate(state, &winner).await;
7021 let (outcomes, timed_out_pids) = run_commands(
7022 state,
7023 "gate",
7024 "gate",
7025 0,
7026 &shell,
7027 &gate_commands,
7028 &winner.worktree,
7029 budget,
7030 )
7031 .await;
7032 (outcomes, false, timed_out_pids)
7033 }
7034 },
7035 )
7036 .await?;
7037 outcomes
7038 };
7039 if outcomes.is_empty() {
7040 self.state
7045 .event("gate", "no gate commands configured; not tested");
7046 }
7047 for o in &outcomes {
7048 self.state.event(
7049 "gate",
7050 format!(
7051 "`{}` -> {}",
7052 o.command,
7053 if o.ok() {
7054 "pass".to_owned()
7055 } else {
7056 format!(
7057 "FAIL ({:?})\n{}",
7058 o.code,
7059 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7060 )
7061 }
7062 ),
7063 );
7064 }
7065 Ok(outcomes)
7066 }
7067
7068 async fn gate_fix_round(
7081 &mut self,
7082 winner: &Candidate,
7083 outcomes: &[CommandOutcome],
7084 ) -> Result<GateFix> {
7085 let cap = self.state.config.graph.gate_fix_rounds;
7086 let spent = self.state.gate_fixes.len();
7087 if spent >= cap {
7088 if cap > 0 {
7089 self.state.event(
7090 "gate",
7091 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7092 );
7093 }
7094 return Ok(GateFix::Stop);
7095 }
7096 if !gate_fixable(outcomes) {
7097 self.state.event(
7098 "gate",
7099 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7100 command or similar); not spending a fix round on it",
7101 );
7102 return Ok(GateFix::Stop);
7103 }
7104 let min_free = self.state.config.disk.min_free_bytes;
7105 if min_free > 0 {
7106 match crate::disk::free_bytes(&winner.worktree) {
7107 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7108 Ok(free) => {
7109 self.state.event(
7110 "gate",
7111 format!(
7112 "only {free} bytes free ({min_free} required by `[disk] \
7113 min_free_bytes`); not spending a fix round on a failure the disk \
7114 may explain"
7115 ),
7116 );
7117 return Ok(GateFix::Stop);
7118 }
7119 Err(e) => {
7120 self.state.event(
7121 "gate",
7122 format!("free disk space could not be measured ({e:#}); no fix round"),
7123 );
7124 return Ok(GateFix::Stop);
7125 }
7126 }
7127 }
7128
7129 let attempt = spent + 1;
7130 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7131 let base = self.landing_base();
7132 let fix_prompt = prompt::gate_fix_in(
7133 &self.state.instruction,
7134 &failed,
7135 attempt,
7136 cap,
7137 &self.state.config.graph.language,
7138 &self.state.config.graph.github_language,
7139 );
7140 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7141 let sessions = self.state.config.graph.sessions;
7142 let artifacts = self.merge_repair_artifacts();
7143 self.state.event(
7144 "gate",
7145 format!("gate failed; gate-fix round {attempt} of {cap}"),
7146 );
7147 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7148 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7149 let (_, seat, out) = self
7150 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7151 prompt: fix_prompt.clone(),
7152 spec,
7153 seat,
7154 cwd: winner.worktree.clone(),
7155 timeout,
7156 allow_write: true,
7157 sessions,
7158 artifacts: artifacts.clone(),
7159 stem: format!("gate-fix-{attempt}"),
7160 handover: None,
7161 })
7162 .await;
7163 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7166 return Ok(GateFix::Defer);
7167 }
7168 let mut record = GateFixRecord {
7169 agent: seat.agent.clone(),
7170 failed,
7171 notes: String::new(),
7172 committed: false,
7173 error: None,
7174 };
7175 match out {
7176 AgentOutcome::Ok(o) => {
7177 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7180 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7181 }
7182 }
7183 AgentOutcome::Dropped(_) => {
7184 record.error = Some("the CLI dropped the stream".to_owned());
7185 }
7186 AgentOutcome::Quota(o) => {
7187 self.state.quota.push(QuotaLoss {
7188 seat: seat.key.clone(),
7189 node: "gate-fix".to_owned(),
7190 at: Timestamp::now(),
7191 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7192 });
7193 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7194 }
7195 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7196 }
7197 self.state.seats.insert(seat.key.clone(), seat);
7198 let coordination = self.coordinate(&winner.worktree).await?;
7199 if let Ok(r) = git::rescue_commit(
7200 &winner.worktree,
7201 &format!("magi: gate fix {attempt} (uncommitted work)"),
7202 )
7203 .await
7204 {
7205 self.state.note_withheld("gate-fix", &r.withheld);
7206 }
7207 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7208 record.committed = after != before;
7209 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7210 let note = record.error.clone();
7211 self.state.gate_fixes.push(record);
7212 self.state.save()?;
7213 if !changed {
7214 self.state.event(
7215 "gate",
7216 match note {
7217 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7218 None => format!("gate-fix round {attempt}: the tree did not change"),
7219 },
7220 );
7221 return Ok(GateFix::Stop);
7222 }
7223 self.state.event(
7224 "gate",
7225 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7226 );
7227
7228 drop(coordination);
7229 let commands = self.state.config.verify.e2e.clone();
7230 if !commands.is_empty() {
7231 let shell = self.state.config.shell();
7232 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7233 let cache_dir = self.state.config.cache_dir();
7234 let context = format!("gate-fix round {attempt}");
7235 let wait = self.repository_wait();
7236 let (e2e, _) = with_cache_lease(
7237 &mut self.state,
7238 cache_dir.as_deref(),
7239 "e2e",
7240 "e2e",
7241 &winner.worktree,
7242 &after,
7243 timeout,
7244 &context,
7245 &wait,
7246 |state, budget| {
7247 let shell = shell.clone();
7248 let commands = commands.clone();
7249 let context = context.clone();
7250 let worktree = winner.worktree.clone();
7251 async move {
7252 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7253 .await
7254 }
7255 },
7256 )
7257 .await?;
7258 if verify_inconclusive(&e2e) {
7259 return Ok(GateFix::Defer);
7260 }
7261 if e2e.iter().any(|o| !o.ok()) {
7262 self.state.event(
7263 "gate",
7264 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7265 );
7266 return Ok(GateFix::Stop);
7267 }
7268 }
7269 Ok(GateFix::Retry)
7270 }
7271
7272 async fn guarded_pr_message(
7279 &mut self,
7280 winner: &Candidate,
7281 facts: Option<&BranchFacts>,
7282 posting: bool,
7283 ) -> Result<Option<PrMessage>> {
7284 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7285 let head = match final_head.as_deref() {
7286 Some(h) => crate::pr_outcome::FinalHead::At(h),
7287 None => crate::pr_outcome::FinalHead::Unknown,
7288 };
7289 let summarized = if posting {
7291 Some(self.summarize_change(winner, final_head.as_deref()).await)
7292 } else {
7293 None
7294 };
7295 if self.state.resource_block.is_some() {
7300 self.state.event(
7301 "pr-summary",
7302 "a shared resource was busy; nothing was published, resume to summarize again",
7303 );
7304 return Ok(None);
7305 }
7306 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7307 anyhow::bail!(
7308 "the final commit moved during public description generation; retry before publishing"
7309 );
7310 }
7311 let note = match (&summarized, &self.state.change_summary) {
7312 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7313 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7314 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7315 (None, _) => ChangeNote::NotAsked,
7316 };
7317 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7318 if !posting {
7319 let identity = crate::scrub::Identity::current();
7320 return Ok(Some(PrMessage {
7321 title: crate::scrub::scrub(&pr.title, &identity),
7322 body: crate::scrub::scrub(&pr.body, &identity),
7323 }));
7324 }
7325 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7328 if let Some(known) = self
7329 .state
7330 .github_text
7331 .clone()
7332 .filter(|g| g.fingerprint == raw_fp)
7333 {
7334 return self.settle_withheld(winner, &pr, known).await;
7335 }
7336 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7337 let mut violations =
7338 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7339 let identity = crate::scrub::Identity::current();
7340 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7341 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7342 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7343 {
7344 violations.push(crate::github_text::Violation::SensitiveData);
7345 }
7346 let has_language = violations
7350 .iter()
7351 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7352 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7353 self.state.event(
7354 "github-text",
7355 if has_language {
7356 format!("description rejected: {violations:?}; requesting one rewrite")
7357 } else {
7358 format!("description rejected: {violations:?}; asking the owner")
7359 },
7360 );
7361 let mut rewritten = false;
7362 let mut rejected_rewrite = None;
7363 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7364 {
7365 let output = text;
7366 if let Some(title) = summary_title(&output) {
7367 let raw_body = summary_without_title(&output);
7370 let clean_title = crate::scrub::scrub(&title, &identity);
7371 let mut body = crate::scrub::scrub(&raw_body, &identity);
7372 let redacted = clean_title != title || body != raw_body;
7373 let title = clean_title;
7374 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7375 if !complete {
7376 self.state.event(
7377 "github-text",
7378 "description rewrite omitted required findings",
7379 );
7380 }
7381 if !body.trim().is_empty() {
7382 body.push_str("\n\n");
7383 body.push_str(&public_pr_evidence(&self.state, head));
7384 }
7385 let marker = format!("magi:run/{}", self.state.id);
7386 if !body.trim().is_empty() && !body.contains(&marker) {
7387 body.push_str(&format!("\n\n{marker}"));
7388 }
7389 let rewrite = PrMessage {
7390 title: title.clone(),
7391 body: body.clone(),
7392 };
7393 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7394 let mut failed = crate::github_text::check_for(
7395 &self.state.config.graph,
7396 &title,
7397 &body,
7398 redecision,
7399 );
7400 if (!crate::github_text::shareable(&title)
7401 || !crate::github_text::shareable(&body))
7402 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7403 {
7404 failed.push(crate::github_text::Violation::SensitiveData);
7405 }
7406 if complete
7407 && !title.trim().is_empty()
7408 && !body.trim().is_empty()
7409 && !failed.is_empty()
7410 {
7411 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7412 title: title.clone(),
7413 body: body.clone(),
7414 failed: failed
7415 .iter()
7416 .map(|v| crate::github_text::category(*v).to_owned())
7417 .collect(),
7418 });
7419 }
7420 if complete
7421 && !title.trim().is_empty()
7422 && !raw_body.trim().is_empty()
7423 && failed.is_empty()
7424 {
7425 decision = redecision;
7426 pr = PrMessage { title, body };
7427 rewritten = true;
7428 self.state
7429 .event("github-text", "description rewrite passed");
7430 if redacted {
7431 self.state.event(
7432 "github-text",
7433 "sensitive data removed from description rewrite",
7434 );
7435 }
7436 }
7437 }
7438 }
7439 if !rewritten {
7440 self.state.event(
7441 "github-text",
7442 "description rewrite unavailable or rejected; asking the owner",
7443 );
7444 if let Some(w) =
7445 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7446 {
7447 self.retire_github_text_question("the withheld text changed");
7450 let record = crate::run::GithubTextAsk {
7451 rewrite: rejected_rewrite,
7452 fingerprint: raw_fp,
7453 title: w.title,
7454 body: w.body,
7455 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7456 question: None,
7457 asks: 0,
7458 resolved: false,
7459 chosen_title: None,
7460 choice: None,
7461 };
7462 return self.settle_withheld(winner, &pr, record).await;
7463 }
7464 }
7465 }
7466 let (title, body) =
7467 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7468 Ok(Some(PrMessage { title, body }))
7469 }
7470
7471 async fn summarize_change(
7482 &mut self,
7483 winner: &Candidate,
7484 head: Option<&str>,
7485 ) -> std::result::Result<(), String> {
7486 let Some(head) = head else {
7487 return Err("the final commit could not be read".to_owned());
7488 };
7489 let remote = self.state.config.merge.remote.clone();
7490 let tracking = format!("{remote}/{}", self.state.base_branch);
7491 if !git::rev_exists(&winner.worktree, &tracking).await {
7492 return Err(format!("the remote base `{tracking}` could not be read"));
7493 }
7494 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7495 Ok(b) if !b.is_empty() => b,
7496 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7497 };
7498 let evidence = crate::pr_outcome::summary_evidence(
7499 &self.state,
7500 crate::pr_outcome::FinalHead::At(head),
7501 );
7502 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7503 if self
7504 .state
7505 .change_summary
7506 .as_ref()
7507 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7508 {
7509 return Ok(());
7510 }
7511 if self.state.change_summary.take().is_some() {
7512 self.state.event(
7513 "pr-summary",
7514 "the final diff or evidence changed; the earlier change summary was discarded",
7515 );
7516 }
7517 if let Err(e) = write_artifact(
7519 &self.state,
7520 &format!("public-description-{fingerprint}.md"),
7521 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7522 ) {
7523 tracing::warn!("could not save public description evidence: {e:#}");
7524 }
7525 let chain = match agent::pick_chain(
7526 &self.state.config.agents,
7527 self.state.config.roles.synthesizer.as_ref(),
7528 &agent::installed,
7529 "synthesizer",
7530 ) {
7531 Ok(chain) => chain,
7532 Err(e) => {
7533 tracing::warn!("no agent for the change summary: {e:#}");
7534 self.state.event("pr-summary", "no summary agent available");
7535 return Err("no agent was available to summarize it".to_owned());
7536 }
7537 };
7538 let subjects = git::subjects(&winner.worktree, &base, head)
7539 .await
7540 .unwrap_or_default();
7541 let stat = git::diff_stat(&winner.worktree, &base, head)
7542 .await
7543 .unwrap_or_default();
7544 let stat: String = stat.chars().take(6_000).collect();
7545 let mut prompt = crate::prompt::change_summary_in(
7546 &base,
7547 head,
7548 &subjects,
7549 stat.trim(),
7550 &self.state.config.graph.github_language,
7551 );
7552 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7553 let sessions = self.state.config.graph.sessions;
7554 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7555 let prompts = self.state.config.prompts.clone();
7556 let cache = self.state.config.cache_dir();
7557 let run = self.state.id.clone();
7558 for (i, spec) in chain.into_iter().enumerate() {
7559 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7560 let job = SeatJob {
7561 spec: spec.clone(),
7562 seat,
7563 cwd: winner.worktree.clone(),
7564 prompt: prompt.clone(),
7565 timeout,
7566 allow_write: false,
7567 sessions,
7568 artifacts: agent::artifacts_dir(&self.state.dir()),
7569 stem: if i == 0 {
7570 "change-summary".to_owned()
7571 } else {
7572 format!("change-summary-{}", spec.id)
7573 },
7574 handover: None,
7575 };
7576 let ctx = WaveCtx {
7577 run: &run,
7578 node: "pr-summary",
7579 prompts: &prompts,
7580 cache: cache.as_deref(),
7581 round: None,
7582 carry_seats: false,
7583 pauses: Some((&self.pause, &self.interrupt)),
7584 };
7585 let (seat, outcome) =
7586 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7587 self.state.seats.insert(seat.key.clone(), seat);
7588 let text = match outcome {
7589 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7590 AgentOutcome::ResourceBlocked(why) => {
7591 if self.state.resource_block.is_none() {
7594 self.state.resource_block = Some(crate::run::ResourceBlock {
7595 node: "pr-summary".to_owned(),
7596 kind: "shared resource".to_owned(),
7597 detail: why,
7598 at: Timestamp::now(),
7599 });
7600 }
7601 return Err("a shared resource was busy".to_owned());
7602 }
7603 _ => continue,
7604 };
7605 let blind = &self.state.config.blind;
7606 let Some(title) = summary_title(&text) else {
7607 continue;
7608 };
7609 let body: String = summary_without_title(&text)
7610 .lines()
7611 .filter(|l| !l.contains("magi:run/"))
7612 .collect::<Vec<_>>()
7613 .join("\n");
7614 let body = blind::sanitize_prose(body.trim(), blind);
7615 let title = blind::sanitize_prose(&title, blind);
7616 if body.trim().is_empty() || title.trim().is_empty() {
7617 continue;
7618 }
7619 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7620 continue;
7621 }
7622 if git::rev_parse(&winner.worktree, "HEAD")
7624 .await
7625 .ok()
7626 .as_deref()
7627 != Some(head)
7628 {
7629 return Err("the branch moved while the summary was written".to_owned());
7630 }
7631 if git::merge_base(&winner.worktree, &tracking, head)
7632 .await
7633 .ok()
7634 .as_deref()
7635 != Some(base.as_str())
7636 || crate::pr_outcome::summary_evidence(
7637 &self.state,
7638 crate::pr_outcome::FinalHead::At(head),
7639 ) != evidence
7640 {
7641 return Err("the summary evidence changed while it was written".to_owned());
7642 }
7643 self.state.change_summary = Some(crate::run::ChangeSummary {
7644 head: head.to_owned(),
7645 base,
7646 evidence: fingerprint,
7647 title,
7648 body,
7649 });
7650 self.state.event(
7651 "pr-summary",
7652 format!("change summary written by {} for the final diff", spec.id),
7653 );
7654 if let Err(e) = self.state.save() {
7655 tracing::warn!("could not save the change summary: {e:#}");
7656 }
7657 return Ok(());
7658 }
7659 self.state
7660 .event("pr-summary", "no agent produced a change summary");
7661 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7662 }
7663
7664 async fn rewrite_pr_text(
7672 &mut self,
7673 winner: &Candidate,
7674 pr: &PrMessage,
7675 violations: &[crate::github_text::Violation],
7676 ) -> Option<String> {
7677 let prompt = format!(
7678 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7679 pr.title, pr.body
7680 );
7681 let sessions = self.state.config.graph.sessions;
7682 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7683 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7685 let key = format!("impl-{}", winner.label);
7686 if let Some(spec) = self
7687 .state
7688 .config
7689 .agents
7690 .iter()
7691 .find(|a| a.id == winner.agent)
7692 .cloned()
7693 && let Some(existing) = self.state.seats.get(&key)
7694 && existing.agent == spec.id
7695 && agent::has_session(spec.kind, existing, sessions)
7696 {
7697 attempts.push((spec, existing.clone(), true));
7698 } else {
7699 let chain = agent::pick_chain(
7700 &self.state.config.agents,
7701 self.state.config.roles.language_judge.as_ref(),
7702 &agent::installed,
7703 "language_judge",
7704 );
7705 match chain {
7706 Ok(chain) => {
7707 for spec in chain {
7708 let seat = crate::fixer::seat_for(
7709 &mut self.state,
7710 "github-text-rewrite",
7711 &spec.id,
7712 );
7713 attempts.push((spec, seat, false));
7714 }
7715 }
7716 Err(e) => {
7717 tracing::warn!("no agent for the description rewrite: {e:#}");
7718 self.state
7719 .event("github-text", "no rewrite agent available");
7720 return None;
7721 }
7722 }
7723 }
7724 let prompts = self.state.config.prompts.clone();
7725 let cache = self.state.config.cache_dir();
7726 let run = self.state.id.clone();
7727 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7728 let job = SeatJob {
7729 spec: spec.clone(),
7730 seat,
7731 cwd: winner.worktree.clone(),
7732 prompt: prompt.clone(),
7733 timeout: retry_budget(timeout_full, resumed),
7734 allow_write: false,
7735 sessions,
7736 artifacts: agent::artifacts_dir(&self.state.dir()),
7737 stem: if i == 0 {
7738 "github-text-rewrite".to_owned()
7739 } else {
7740 format!("github-text-rewrite-{}", spec.id)
7741 },
7742 handover: None,
7743 };
7744 let ctx = WaveCtx {
7745 run: &run,
7746 node: "github-text",
7747 prompts: &prompts,
7748 cache: cache.as_deref(),
7749 round: None,
7750 carry_seats: false,
7751 pauses: Some((&self.pause, &self.interrupt)),
7752 };
7753 let (seat, outcome) =
7754 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7755 self.state.seats.insert(seat.key.clone(), seat);
7756 match outcome {
7757 AgentOutcome::Ok(o)
7758 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7759 {
7760 return Some(o.text);
7761 }
7762 AgentOutcome::ResourceBlocked(_) => return None,
7763 _ => {}
7764 }
7765 }
7766 None
7767 }
7768
7769 fn retire_github_text_question(&mut self, why: &str) {
7771 let Some(id) = self
7772 .state
7773 .github_text
7774 .as_ref()
7775 .and_then(|g| g.question.clone())
7776 else {
7777 return;
7778 };
7779 let _ = ask::Questions::open().update(&id, |q| {
7780 q.abandon(why);
7781 Ok(())
7782 });
7783 }
7784
7785 async fn settle_withheld(
7789 &mut self,
7790 winner: &Candidate,
7791 pr: &PrMessage,
7792 mut g: crate::run::GithubTextAsk,
7793 ) -> Result<Option<PrMessage>> {
7794 use crate::github_text as gt;
7795 let store = ask::Questions::open();
7796 let timeout = self.state.config.graph.answer_timeout;
7797 if !g.resolved {
7798 let mut standing = g
7799 .question
7800 .as_ref()
7801 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7802 if let Some(q) = standing.as_ref()
7803 && q.status.open()
7804 && gt::expired(q, timeout)
7805 {
7806 let why = format!("no answer within {}s of asking", timeout.max(1));
7807 standing = store
7808 .update(&q.id, |q| {
7809 q.abandon(&why);
7810 Ok(())
7811 })
7812 .ok()
7813 .map(|(q, ())| q);
7814 }
7815 let mut ask_again: Option<Option<Vec<String>>> = None;
7816 match standing {
7817 None if g.asks <= 1 => {
7820 g.asks = 0;
7821 ask_again = Some(None);
7822 }
7823 None => {
7824 g.resolved = true;
7825 g.chosen_title = None;
7826 g.choice = Some(GithubTextChoice::Fallback);
7827 }
7828 Some(q) if q.status.open() => {
7829 self.park_on_github_text(&g)?;
7830 return Ok(None);
7831 }
7832 Some(q) => match gt::read_reply(&q) {
7833 gt::Reply::Fallback => {
7834 g.resolved = true;
7835 g.chosen_title = None;
7836 g.choice = Some(GithubTextChoice::Fallback);
7837 }
7838 gt::Reply::Original => {
7839 g.resolved = true;
7840 g.chosen_title = None;
7841 g.choice = Some(GithubTextChoice::Original);
7842 }
7843 gt::Reply::Rewrite => {
7844 g.resolved = true;
7845 g.chosen_title = None;
7846 g.choice = Some(if g.rewrite.is_some() {
7847 GithubTextChoice::Rewrite
7848 } else {
7849 GithubTextChoice::Fallback
7850 });
7851 }
7852 gt::Reply::Title(text) => {
7853 let first = text
7854 .lines()
7855 .map(str::trim)
7856 .find(|l| !l.is_empty())
7857 .unwrap_or("");
7858 let decision = if first.is_empty() {
7859 None
7860 } else {
7861 let candidate = PrMessage {
7862 title: first.to_owned(),
7863 body: String::new(),
7864 };
7865 self.judge_pr_language(&winner.worktree, &candidate).await
7866 };
7867 match gt::vet_title(
7868 &text,
7869 decision,
7870 self.state.config.graph.github_english(),
7871 ) {
7872 Ok(title) => {
7873 g.resolved = true;
7874 g.chosen_title = Some(title);
7875 g.choice = Some(GithubTextChoice::MyTitle);
7876 }
7877 Err(cats) if g.asks < 2 => {
7878 ask_again = Some(Some(
7879 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7880 ))
7881 }
7882 Err(_) => {
7883 g.resolved = true;
7884 g.chosen_title = None;
7885 g.choice = Some(GithubTextChoice::Fallback);
7886 }
7887 }
7888 }
7889 },
7890 }
7891 if let Some(rejected) = ask_again {
7892 let w = gt::Withheld {
7895 title: g.title,
7896 body: g.body,
7897 categories: g.categories.clone(),
7898 };
7899 let artifact = match crate::run::write_artifact(
7902 &self.state,
7903 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7904 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7905 ) {
7906 Ok(p) => Some(p),
7907 Err(e) => {
7908 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7909 None
7910 }
7911 };
7912 let (detail, comparison) = gt::question_content(
7913 &w,
7914 &pr.title,
7915 &pr.body,
7916 &self.state.id,
7917 rejected.as_deref(),
7918 artifact.as_deref(),
7919 g.rewrite.as_ref(),
7920 );
7921 let mut q = ask::Question::new(
7922 self.state.id.clone(),
7923 gt::ASK_NODE.to_owned(),
7924 gt::ASK_SEAT.to_owned(),
7925 gt::question_summary(&self.state.config.graph.language, &w),
7926 detail,
7927 gt::question_choices(&w, g.rewrite.as_ref()),
7928 );
7929 q.posting_comparison = Some(comparison);
7930 q.answer_timeout = timeout;
7931 g.question = Some(q.id.clone());
7934 g.asks += 1;
7935 self.state.github_text = Some(g.clone());
7936 self.state.event(
7937 "github-text",
7938 format!("asking the owner about the withheld text ({})", q.short()),
7939 );
7940 self.state.save()?;
7941 store
7942 .put_executable(&mut q)
7943 .context("file the posting-gate question")?;
7944 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7945 tracing::warn!(
7946 "could not notify about posting-gate question {}: {e:#}",
7947 q.short()
7948 );
7949 }
7950 self.park_on_github_text(&g)?;
7951 return Ok(None);
7952 }
7953 }
7954 let resolved = g.chosen_title.clone();
7955 let original = g.choice == Some(GithubTextChoice::Original);
7956 self.state.event(
7958 "github-text",
7959 if original {
7960 format!(
7961 "posting the original text for: {}",
7962 match (g.title, g.body) {
7963 (true, true) => "title, description",
7964 (true, false) => "title",
7965 _ => "description",
7966 }
7967 )
7968 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7969 "posting the saved rewrite title and description".to_owned()
7970 } else if resolved.is_some() {
7971 "posting the owner's replacement title".to_owned()
7972 } else {
7973 "posting the neutral text for the withheld fields".to_owned()
7974 },
7975 );
7976 self.state.github_text = Some(g.clone());
7977 self.state.save()?;
7978 if g.choice == Some(GithubTextChoice::Rewrite)
7979 && let Some(r) = g.rewrite.as_ref()
7980 {
7981 return Ok(Some(PrMessage {
7982 title: r.title.clone(),
7983 body: r.body.clone(),
7984 }));
7985 }
7986 let w = gt::Withheld {
7987 title: g.title,
7988 body: g.body,
7989 categories: g.categories.clone(),
7990 };
7991 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7992 if let Some(t) = resolved {
7993 title = t;
7994 }
7995 let english = gt::LanguageDecision {
7997 title_english: true,
7998 body_english: true,
7999 body_complete: true,
8000 };
8001 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8002 if original {
8003 if g.title {
8006 title = pr.title.clone();
8007 }
8008 if g.body {
8009 body = pr.body.clone();
8010 }
8011 }
8012 Ok(Some(PrMessage { title, body }))
8013 }
8014
8015 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8016 self.state.github_text = Some(g.clone());
8017 self.state.end_pass_parked(ParkCause::Question);
8018 self.state.parked = true;
8019 self.state.event(
8020 "github-text",
8021 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8022 );
8023 self.state.save()
8024 }
8025
8026 async fn judge_pr_language(
8029 &mut self,
8030 cwd: &Path,
8031 pr: &PrMessage,
8032 ) -> Option<crate::github_text::LanguageDecision> {
8033 self.state.config.roles.language_judge.as_ref()?;
8034 if !self.state.config.graph.language_gate() {
8035 return None;
8036 }
8037 let decision =
8038 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8039 self.state.event(
8040 "github-text",
8041 if decision.is_some() {
8042 "language decided by the language judge"
8043 } else {
8044 "language judge unavailable; using built-in heuristics"
8045 },
8046 );
8047 decision
8048 }
8049
8050 async fn merge(&mut self) -> Result<()> {
8051 if self
8066 .state
8067 .base_sync
8068 .as_ref()
8069 .is_some_and(|s| s.conflict.is_some())
8070 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8071 != Some(RunStatus::Gating)
8072 || !self.state.gate_allows_progress()
8080 {
8081 return Ok(());
8082 }
8083 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8092 return Ok(());
8093 }
8094 let Some(winner) = self.state.winner().cloned() else {
8095 return Ok(());
8096 };
8097 let repo = self.state.repo.clone();
8098 let base = self.state.base_branch.clone();
8099 let mode = self.state.config.merge.mode;
8100 let style = self.state.config.merge.style;
8101 let facts = if is_review_run(&self.state) {
8102 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8103 let start = review_base(
8104 &repo,
8105 &self.state.config.merge.remote,
8106 &base,
8107 &self.state.base_commit,
8108 &winner.branch,
8109 )
8110 .await;
8111 branch_facts(&repo, &start, &winner.branch).await
8112 } else {
8113 None
8114 };
8115 let leaked = if is_review_run(&self.state) {
8118 leaked_subjects(&self.state, &winner.branch).await
8119 } else {
8120 Some(Vec::new())
8121 };
8122 let Some(pr) = self
8126 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8127 .await?
8128 else {
8129 return Ok(());
8130 };
8131 let message = pr.commit_message();
8132
8133 let outcome = match mode {
8134 MergeMode::None => MergeOutcome {
8135 mode,
8136 ok: true,
8137 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8138 empty: false,
8139 },
8140 MergeMode::Pr | MergeMode::Local
8141 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8142 {
8143 MergeOutcome {
8144 mode,
8145 ok: false,
8146 detail: empty_candidate_detail(&self.state, &base),
8147 empty: true,
8148 }
8149 }
8150 MergeMode::Local => {
8151 let on = git::current_branch(&repo).await?;
8152 if on.as_deref() != Some(base.as_str()) {
8153 MergeOutcome {
8154 mode,
8155 ok: false,
8156 detail: format!(
8157 "{} has {} checked out, not the base branch {base}",
8158 repo.display(),
8159 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8160 ),
8161 empty: false,
8162 }
8163 } else if !git::is_clean(&repo).await? {
8164 MergeOutcome {
8165 mode,
8166 ok: false,
8167 detail: format!("{} is dirty; refusing to merge", repo.display()),
8168 empty: false,
8169 }
8170 } else {
8171 let out = match style {
8172 MergeStyle::Merge => {
8173 git::merge_no_ff(&repo, &winner.branch, &message).await?
8174 }
8175 MergeStyle::Squash => {
8176 git::merge_squash(&repo, &winner.branch, &message).await?
8177 }
8178 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8179 };
8180 MergeOutcome {
8181 mode,
8182 ok: out.ok(),
8183 detail: if out.ok() { out.stdout } else { out.stderr },
8184 empty: false,
8185 }
8186 }
8187 }
8188 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8189 self.state.pr_publication_pending = true;
8190 self.state.save()?;
8191 let out =
8192 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8193 self.state.pr_publication_pending = out.is_err();
8194 MergeOutcome {
8195 mode,
8196 ok: out.is_ok(),
8197 detail: match out {
8198 Ok(url) => url,
8199 Err(e) => format!("{e:#}"),
8200 },
8201 empty: false,
8202 }
8203 }
8204 MergeMode::Pr => {
8205 let remote = self.state.config.merge.remote.clone();
8206 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8207 if !pushed.ok() {
8208 MergeOutcome {
8209 mode,
8210 ok: false,
8211 detail: pushed.stderr,
8212 empty: false,
8213 }
8214 } else {
8215 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8221 let out = match pr_merge_plan(found) {
8222 PrPlan::Create => {
8223 gh_pr_create(
8224 &winner.worktree,
8225 &base,
8226 &winner.branch,
8227 &pr.title,
8228 &pr.body,
8229 )
8230 .await
8231 }
8232 PrPlan::Adopt { url, title } => {
8233 self.state
8234 .event("merge", format!("Pr: adopted open pull request {url}"));
8235 if self.state.github_text.as_ref().is_some_and(|g| {
8236 g.resolved
8237 && g.choice == Some(GithubTextChoice::Rewrite)
8238 && g.rewrite.is_some()
8239 }) {
8240 land::set_pr_text_verbatim(
8243 &winner.worktree,
8244 &url,
8245 &pr.title,
8246 &pr.body,
8247 )
8248 .await
8249 .map(|()| url)
8250 } else {
8251 if title != pr.title
8252 && (!is_review_run(&self.state)
8253 || leaked
8254 .as_deref()
8255 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8256 && let Err(e) =
8257 if self.state.github_text.as_ref().is_some_and(|g| {
8258 g.resolved
8259 && g.title
8260 && g.choice == Some(GithubTextChoice::Original)
8261 }) {
8262 land::set_pr_title_verbatim(
8264 &winner.worktree,
8265 &url,
8266 &pr.title,
8267 )
8268 .await
8269 } else {
8270 land::set_pr_title(
8271 &mut self.state,
8272 &winner.worktree,
8273 &url,
8274 &pr.title,
8275 )
8276 .await
8277 }
8278 {
8279 tracing::warn!("could not refresh title of {url}: {e:#}");
8280 self.state
8281 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8282 }
8283 Ok(url)
8284 }
8285 }
8286 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8287 };
8288 match out {
8289 Ok(url) => MergeOutcome {
8290 mode,
8291 ok: true,
8292 detail: url,
8293 empty: false,
8294 },
8295 Err(e) => MergeOutcome {
8296 mode,
8297 ok: false,
8298 detail: e.to_string(),
8299 empty: false,
8300 },
8301 }
8302 }
8303 }
8304 };
8305
8306 self.state.status = match (mode, outcome.ok) {
8307 (MergeMode::None, _) => RunStatus::Ready,
8308 (_, true) if outcome.empty => RunStatus::Ready,
8309 (MergeMode::Pr, true) => RunStatus::Ready,
8310 (_, true) => RunStatus::Merged,
8311 (_, false) => RunStatus::Blocked,
8312 };
8313 self.state.event(
8314 "merge",
8315 format!(
8316 "{:?}: {}",
8317 mode,
8318 outcome.detail.lines().next().unwrap_or("")
8319 ),
8320 );
8321 self.state.merge = Some(outcome);
8322 self.state.save()?;
8323
8324 if self.state.config.graph.land
8330 && mode == MergeMode::Pr
8331 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8332 {
8333 self.run_land().await?;
8334 }
8335 self.settle_questions();
8340 Ok(())
8341 }
8342
8343 async fn run_land(&mut self) -> Result<()> {
8354 crate::pr_head::ensure_owner(&self.state)?;
8355 let url = self
8356 .state
8357 .merge
8358 .as_ref()
8359 .map(|m| m.detail.clone())
8360 .unwrap_or_default();
8361 let url = url.lines().next().unwrap_or("").trim().to_owned();
8362 if !url.starts_with("http") {
8363 return Ok(());
8364 }
8365 loop {
8368 match land::land(&mut self.state, &url).await {
8369 Ok(_)
8370 if self
8371 .state
8372 .merge_repairs
8373 .last()
8374 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8375 {
8376 if self.repair_merge_review().await? {
8377 continue;
8378 }
8379 }
8380 Ok(pr) if self.state.parked => {
8381 let _ = pr;
8385 }
8386 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8387 self.state.status = RunStatus::Superseded;
8388 self.state.save()?;
8389 }
8390 Ok(pr) => {
8391 self.state.status = match pr.state {
8392 land::PrLifecycle::Merged => RunStatus::Merged,
8393 _ => RunStatus::Blocked,
8394 };
8395 if bump::should_release_bump(self.state.status)
8402 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8403 {
8404 self.state
8410 .event("bump", format!("release bump skipped: {e:#}"));
8411 }
8412 if self.state.status == RunStatus::Merged {
8415 crate::followup::after_merge(&mut self.state, &pr.url).await;
8416 }
8417 self.state.save()?;
8418 }
8419 Err(e) => {
8420 if e.is::<crate::repo_lock::ResourceBlocked>() {
8421 return Err(e);
8422 }
8423 self.state.status = RunStatus::Blocked;
8424 self.state.event("land", format!("gave up: {e}"));
8425 self.state.save()?;
8426 }
8427 }
8428 break;
8429 }
8430 Ok(())
8431 }
8432
8433 async fn repair_merge_review(&mut self) -> Result<bool> {
8436 use crate::run::MergeRepairPhase as Phase;
8437 crate::pr_head::ensure_owner(&self.state)?;
8438 let i = self.state.merge_repairs.len() - 1;
8439 let request = self.state.merge_repairs[i].clone();
8440 if request.phase == Phase::Fixing {
8441 let restored = match self.state.winner().cloned() {
8442 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8443 None => Err(anyhow::anyhow!("repair has no winning branch")),
8444 };
8445 let why = match restored {
8446 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8447 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8448 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8449 };
8450 self.hold_merge_repair(i, why)?;
8451 return Ok(false);
8452 }
8453 let failure = if i >= self.state.config.graph.land_rounds
8454 || self.state.config.graph.review_rounds == 0
8455 {
8456 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8457 } else {
8458 None
8459 };
8460 if let Some(why) = failure {
8461 self.hold_merge_repair(i, why)?;
8462 return Ok(false);
8463 }
8464 if request.phase == Phase::Requested {
8465 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8466 Ok(pr) => pr,
8467 Err(e) => {
8468 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8469 return Ok(false);
8470 }
8471 };
8472 self.state.merge_repairs[i].phase = Phase::Fixing;
8473 self.state.land_approval = None;
8474 self.state.land_armed_head = None;
8475 self.state.save()?;
8476 let evidence = serde_json::to_string_pretty(&request)?;
8479 let reason = format!(
8480 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8481 );
8482 let budget = self.state.config.graph.land_rounds;
8483 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8484 match result {
8485 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8486 Ok(land::Fixed::Failed(why)) => {
8487 self.hold_merge_repair(
8488 i,
8489 format!(
8490 "repair failed: {why}; inspect the branch and run an explicit review"
8491 ),
8492 )?;
8493 return Ok(false);
8494 }
8495 Err(e) => {
8496 self.hold_merge_repair(
8497 i,
8498 format!(
8499 "repair failed: {e:#}; inspect the branch and run an explicit review"
8500 ),
8501 )?;
8502 return Ok(false);
8503 }
8504 }
8505 self.begin_merge_repair_review(i)?;
8509 }
8510 self.review_loop().await?;
8511 self.gate().await?;
8512 if self.resource_halt()? || self.state.parked {
8513 return Ok(false);
8514 }
8515 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8516 != Some(RunStatus::Gating)
8517 || !self.state.gate_allows_progress()
8518 {
8519 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8520 return Ok(false);
8521 }
8522 let winner = self
8525 .state
8526 .winner()
8527 .cloned()
8528 .context("repair has no winning branch")?;
8529 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8530 if self.state.gate_head.as_deref() != Some(after.as_str())
8531 || !git::is_clean(&winner.worktree).await?
8532 {
8533 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8534 return Ok(false);
8535 }
8536 let expected = self
8537 .state
8538 .pr_pushed_head
8539 .as_deref()
8540 .unwrap_or(&request.head)
8541 .to_owned();
8542 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8543 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8546 let refspec = format!("{after}:refs/heads/{public}");
8547 let push = git::git_raw(
8548 &winner.worktree,
8549 &["push", &lease, &self.state.config.merge.remote, &refspec],
8550 )
8551 .await;
8552 match push {
8553 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8554 Ok(out) => {
8555 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8556 return Ok(false);
8557 }
8558 Err(e) => {
8559 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8560 return Ok(false);
8561 }
8562 }
8563 self.state.merge_repairs[i].phase = Phase::Complete;
8564 self.state.status = RunStatus::Landing;
8565 self.state.event(
8566 "repair",
8567 "new review and gate completed; fresh owner merge decision required",
8568 );
8569 self.state.save()?;
8570 Ok(true)
8571 }
8572
8573 fn merge_repair_artifacts(&self) -> PathBuf {
8574 let artifacts = agent::artifacts_dir(&self.state.dir());
8575 match self
8576 .state
8577 .merge_repairs
8578 .last()
8579 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8580 {
8581 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8582 None => artifacts,
8583 }
8584 }
8585
8586 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8587 self.state.reviews.clear();
8588 self.state.gate.clear();
8589 self.state.gate_ran = false;
8590 self.state.gate_head = None;
8591 self.state.gate_fixes.clear();
8592 self.state.pre_gate.clear();
8593 self.state.pre_gate_commit = None;
8594 self.state.contested_handoff = None;
8595 self.state.land_reruns = None;
8596 self.state.land_approval = None;
8597 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8598 self.state.status = RunStatus::Reviewing;
8599 self.state.save()
8600 }
8601
8602 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8603 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8604 self.state.merge_repairs[index].failure = Some(why.clone());
8605 self.state.status = RunStatus::Blocked;
8606 self.state.event("repair", why);
8607 self.state.save()
8608 }
8609
8610 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8614 if let Some(existing) = self.state.seats.get(key)
8615 && existing.agent == agent
8616 {
8617 return existing.clone();
8618 }
8619 let fresh = if self.state.seats.contains_key(key) {
8622 handover_seat(key, agent, self.state.next_seat_seed())
8623 } else {
8624 SeatState::new(key, agent, self.state.seed)
8625 };
8626 self.state.seats.insert(key.to_owned(), fresh.clone());
8627 fresh
8628 }
8629
8630 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8636 match self.state.seats.get(key) {
8637 Some(s) if s.agent != spec.id => {
8638 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8639 }
8640 _ => spec,
8641 }
8642 }
8643
8644 fn view(&self, c: &Candidate) -> CandidateView {
8646 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8647 .unwrap_or_default();
8648 let (patch, _) = blind::sanitize_patch(
8649 &format!("candidate {} patch", c.label),
8650 &raw,
8651 &self.state.config.blind,
8652 );
8653 CandidateView {
8654 label: c.label,
8655 branch: c.branch.clone(),
8656 summary: c.summary.clone(),
8657 stat: c.stat.clone(),
8658 patch,
8659 }
8660 }
8661
8662 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8664 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8665 prompt::judge(
8666 "(see above)",
8667 &views,
8668 self.roles.judges.len(),
8669 base_short,
8670 "en",
8671 )
8672 }
8673
8674 fn vote_prompt_full(
8680 &self,
8681 j: usize,
8682 labels: &[char],
8683 language: &str,
8684 candidates: &[Candidate],
8685 base_short: &str,
8686 ) -> String {
8687 let mut text = format!(
8688 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8689 prompt::final_vote(labels, language),
8690 self.state.instruction,
8691 self.candidate_block(candidates, base_short)
8692 );
8693 if let Some(own) = self
8694 .state
8695 .judgements
8696 .get(j)
8697 .filter(|r| !r.ranking.is_empty())
8698 {
8699 let reasons = own
8700 .reasons
8701 .iter()
8702 .map(|(k, v)| format!("- {k}: {v}"))
8703 .collect::<Vec<_>>()
8704 .join("\n");
8705 text.push_str(&format!(
8706 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8707 own.ranking.iter().collect::<String>(),
8708 if reasons.is_empty() {
8709 ""
8710 } else {
8711 ", because:\n"
8712 }
8713 ));
8714 }
8715 if !self.state.deliberation.is_empty() {
8716 text.push_str("\n# What was argued before this vote\n");
8717 for t in self.transcript(&[], j) {
8718 text.push_str(&format!(
8719 "\n## {}{}\n\n{}\n",
8720 t.who,
8721 if t.is_self { " (you)" } else { "" },
8722 t.body.trim()
8723 ));
8724 }
8725 }
8726 text
8727 }
8728
8729 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8736 let mut turns = Vec::new();
8737 for j in &self.state.judgements {
8738 if j.ranking.is_empty() {
8739 continue;
8740 }
8741 let reasons = j
8742 .reasons
8743 .iter()
8744 .map(|(k, v)| format!("- {k}: {v}"))
8745 .collect::<Vec<_>>()
8746 .join("\n");
8747 turns.push(Turn {
8748 who: format!("Judge {} (opening ranking)", j.judge),
8749 is_self: j.judge == self_idx + 1,
8750 body: format!(
8751 "Ranked {}{}{reasons}",
8752 j.ranking.iter().collect::<String>(),
8753 if reasons.is_empty() {
8754 ""
8755 } else {
8756 ", because:\n"
8757 }
8758 ),
8759 });
8760 }
8761 for t in self
8762 .state
8763 .deliberation
8764 .iter()
8765 .flat_map(|r| r.turns.iter())
8766 .chain(current)
8767 {
8768 turns.push(Turn {
8769 who: format!("Judge {}", t.judge),
8770 is_self: t.judge == self_idx + 1,
8771 body: t.body.clone(),
8772 });
8773 }
8774 turns
8775 }
8776}
8777
8778fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8780 agent::has_session(spec.kind, seat, sessions)
8781}
8782
8783fn next_untried_in_roster<'a>(
8804 roster: &'a [AgentSpec],
8805 start: usize,
8806 tried: &BTreeSet<String>,
8807) -> Option<&'a AgentSpec> {
8808 roster
8809 .get(start + 1..)?
8810 .iter()
8811 .find(|s| !tried.contains(&s.id))
8812}
8813
8814fn pick_successor<'a>(
8835 roster: &'a [AgentSpec],
8836 start: usize,
8837 tried: &BTreeSet<String>,
8838 carried: Option<&BTreeSet<String>>,
8839 others: &BTreeSet<String>,
8840) -> Option<&'a AgentSpec> {
8841 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8842 roster
8843 .get(start + 1..)
8844 .and_then(|tail| {
8845 tail.iter()
8846 .filter(free)
8847 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8848 })
8849 .or_else(|| {
8850 carried
8851 .is_some()
8852 .then(|| roster.iter().find(free))
8853 .flatten()
8854 })
8855 .or_else(|| match carried {
8856 Some(c) => next_for_seat(roster, start, tried, c),
8857 None => next_untried_in_roster(roster, start, tried),
8858 })
8859}
8860
8861fn next_for_seat<'a>(
8871 roster: &'a [AgentSpec],
8872 start: usize,
8873 round_tried: &BTreeSet<String>,
8874 carried_failed: &BTreeSet<String>,
8875) -> Option<&'a AgentSpec> {
8876 roster
8877 .get(start + 1..)?
8878 .iter()
8879 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8880 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8881}
8882
8883fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8889 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8890 return spec;
8891 };
8892 let ok = |id: &str| !h.failed.contains(id);
8893 if let Some(last) = h.last_ok.as_deref()
8894 && ok(last)
8895 && let Some(s) = roster.iter().find(|s| s.id == last)
8896 {
8897 return s.clone();
8898 }
8899 if ok(&spec.id) {
8900 return spec;
8901 }
8902 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8903 roster
8904 .get(from + 1..)
8905 .into_iter()
8906 .flatten()
8907 .chain(roster.iter())
8908 .find(|s| ok(&s.id))
8909 .cloned()
8910 .unwrap_or(spec)
8911}
8912
8913pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8917 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8918}
8919
8920const TIMED_OUT: &str = "timed out";
8923
8924impl FailClass {
8925 fn of(out: &AgentOutcome) -> Option<Self> {
8927 match out {
8928 AgentOutcome::Ok(_) => None,
8929 AgentOutcome::Quota(_) => Some(Self::Quota),
8930 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8931 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8932 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8933 AgentOutcome::ResourceBlocked(_) => None,
8936 }
8937 }
8938
8939 fn stem_word(&self) -> &'static str {
8941 match self {
8942 Self::Quota => "quota",
8943 _ => "handover",
8944 }
8945 }
8946}
8947
8948fn failure_signature(msg: &str) -> String {
8952 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8953 if let Some(code) = line
8956 .strip_prefix("process exited with no usable output (exit code ")
8957 .and_then(|s| s.strip_suffix(')'))
8958 .and_then(|s| s.parse::<i32>().ok())
8959 {
8960 line = format!("exited with some({code}) and no usable output");
8961 } else if matches!(
8962 line.as_str(),
8963 "process exited with no usable output (no exit code reported)"
8964 | "process exited with no usable output (exit code unavailable)"
8965 ) {
8966 line = "exited with none and no usable output".to_owned();
8967 }
8968 let mut out = Vec::new();
8969 for word in line.split_whitespace() {
8970 if word.contains('/') || word.contains('\\') {
8971 out.push("<path>".to_owned());
8972 continue;
8973 }
8974 let mut w = String::new();
8975 let mut in_digits = false;
8976 for c in word.chars() {
8977 if c.is_ascii_digit() {
8978 if !in_digits {
8979 w.push('#');
8980 }
8981 in_digits = true;
8982 } else {
8983 in_digits = false;
8984 w.push(c);
8985 }
8986 }
8987 out.push(w);
8988 }
8989 out.join(" ").chars().take(120).collect()
8990}
8991
8992fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8999 match cur {
9000 FailClass::Quota | FailClass::Timeout => true,
9001 FailClass::Other(_) => prev != Some(cur),
9002 }
9003}
9004
9005fn fail_reason(out: &AgentOutcome) -> String {
9007 match out {
9008 AgentOutcome::Ok(_) => String::new(),
9009 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9010 AgentOutcome::Dropped(o) => format!(
9011 "the CLI dropped the stream ({})",
9012 o.dropped
9013 .as_ref()
9014 .map(|d| d.why.as_str())
9015 .unwrap_or("it ended without delivering its answer")
9016 ),
9017 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9018 e.lines().next().unwrap_or("").chars().take(160).collect()
9019 }
9020 }
9021}
9022
9023pub(crate) fn record_handover(
9026 state: &mut RunState,
9027 node: &str,
9028 seat: &str,
9029 from: &str,
9030 to: &str,
9031 class: &FailClass,
9032 reason: &str,
9033) {
9034 let message = if *class == FailClass::Quota {
9035 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9036 } else {
9037 format!("{seat}: handed over {from} -> {to} ({reason})")
9038 };
9039 state.event(node, message);
9040 state.handovers.push(Handover {
9041 at: Timestamp::now(),
9042 node: node.to_owned(),
9043 seat: seat.to_owned(),
9044 from: from.to_owned(),
9045 to: to.to_owned(),
9046 reason: reason.to_owned(),
9047 });
9048}
9049
9050fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9064 commands.iter().any(|c| c.exit_code.is_none())
9065}
9066
9067fn verified_noop_claim(
9080 usable: bool,
9081 commands: &[agent::CommandEvidence],
9082 text: &str,
9083) -> Option<String> {
9084 (usable && !has_unconfirmed_command(commands))
9085 .then(|| verdict::verified_noop(text))
9086 .flatten()
9087}
9088
9089fn short(commit: &str) -> String {
9090 commit.chars().take(7).collect()
9091}
9092
9093fn make_executable(path: &Path) -> Result<()> {
9094 #[cfg(unix)]
9095 {
9096 use std::os::unix::fs::PermissionsExt as _;
9097 let mut perms = std::fs::metadata(path)?.permissions();
9098 perms.set_mode(0o755);
9099 std::fs::set_permissions(path, perms)?;
9100 }
9101 #[cfg(not(unix))]
9102 {
9103 let _ = path;
9104 }
9105 Ok(())
9106}
9107
9108struct WaveCtx<'a> {
9115 run: &'a str,
9118 node: &'a str,
9120 prompts: &'a Prompts,
9121 cache: Option<&'a Path>,
9123 round: Option<usize>,
9126 carry_seats: bool,
9130 pauses: Option<(&'a Pause, &'a Pause)>,
9133}
9134
9135fn repository_wait(
9137 state: &RunState,
9138 pauses: Option<(&Pause, &Pause)>,
9139) -> crate::repo_lock::WaitPolicy {
9140 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9141 let inherited = crate::repo_lock::WaitPolicy::current();
9142 crate::repo_lock::WaitPolicy::new(
9143 Duration::from_secs(state.config.graph.timeout_resource_wait),
9144 move || {
9145 inherited.cancelled()
9146 || pauses
9147 .as_ref()
9148 .is_some_and(|(a, b)| a.parked() || b.parked())
9149 },
9150 )
9151}
9152
9153fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9154 if error.is::<crate::repo_lock::ResourceBlocked>() {
9155 state.resource_block = Some(crate::run::ResourceBlock {
9156 node: "repository".into(),
9157 kind: "repository coordination".into(),
9158 detail: error.to_string(),
9159 at: Timestamp::now(),
9160 });
9161 state.event("resource", error.to_string());
9162 }
9163}
9164
9165async fn coordinate_operation(
9166 state: &mut RunState,
9167 cwd: &Path,
9168 wait: &crate::repo_lock::WaitPolicy,
9169) -> Result<Option<crate::repo_lock::SeatLease>> {
9170 let result =
9171 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9172 .await;
9173 if let Err(e) = &result {
9174 record_repository_block(state, e);
9175 }
9176 result
9177}
9178
9179async fn run_one(
9181 job: SeatJob,
9182 sem: Arc<Semaphore>,
9183 ctx: &WaveCtx<'_>,
9184 state: &mut RunState,
9185 attempt: usize,
9186) -> (SeatState, AgentOutcome) {
9187 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9188 .await
9189 .pop()
9190 .expect("one job in, one result out");
9191 (seat, out)
9192}
9193
9194async fn wave(
9200 jobs: Vec<SeatJob>,
9201 sem: Arc<Semaphore>,
9202 ctx: &WaveCtx<'_>,
9203 state: &mut RunState,
9204 attempt: usize,
9205) -> Vec<(usize, SeatState, AgentOutcome)> {
9206 let WaveCtx {
9207 run,
9208 node,
9209 prompts,
9210 cache,
9211 round,
9212 carry_seats: _,
9213 pauses,
9214 } = *ctx;
9215 for job in &jobs {
9216 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9217 }
9218 if let Err(e) = state.save() {
9219 tracing::warn!("could not persist in-progress seats: {e:#}");
9224 }
9225 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9241 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9245 let cache_guard = if let Some(cache_dir) = cache {
9246 if jobs_had_a_writer {
9247 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9248 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9249 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9250 {
9251 Ok(Some(g)) => Some(g),
9252 Ok(None) => {
9255 state.resource_block = Some(crate::run::ResourceBlock {
9256 node: node.to_owned(),
9257 kind: "shared build cache".to_owned(),
9258 detail: "the run is parking".to_owned(),
9259 at: Timestamp::now(),
9260 });
9261 return block_wave(
9262 jobs,
9263 state,
9264 node,
9265 attempt,
9266 "parked while waiting for the shared build cache",
9267 );
9268 }
9269 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9272 Err(e) => {
9273 let why = format!("{e:#}");
9277 state.resource_block = Some(crate::run::ResourceBlock {
9278 node: node.to_owned(),
9279 kind: "shared build cache".to_owned(),
9280 detail: why.clone(),
9281 at: Timestamp::now(),
9282 });
9283 state.event(
9284 "resource",
9285 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9286 );
9287 return block_wave(jobs, state, node, attempt, &why);
9288 }
9289 }
9290 } else {
9291 None
9292 }
9293 } else {
9294 None
9295 };
9296 state.seats_spawning(node, attempt);
9299 if let Err(e) = state.save() {
9300 tracing::warn!("could not persist the start of a wave: {e:#}");
9301 }
9302 let slot_limit = resource_limit
9306 + jobs
9307 .iter()
9308 .map(|j| j.timeout)
9309 .max()
9310 .unwrap_or_default()
9311 .saturating_mul(jobs.len() as u32);
9312 let mut set = tokio::task::JoinSet::new();
9313 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9316 let overlay = prompts.overlay(node);
9317 let repository_wait = repository_wait(state, pauses);
9318 for (i, mut job) in jobs.into_iter().enumerate() {
9319 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9320 if cache.is_some() {
9321 job.prompt.push('\n');
9322 job.prompt
9323 .push_str(&prompt::build_cache_note(node, job.allow_write));
9324 }
9325 let sem = Arc::clone(&sem);
9326 let run = run.to_owned();
9327 let node = node.to_owned();
9328 let attachments = if node == "implement" {
9331 state.attachments.clone()
9332 } else {
9333 Vec::new()
9334 };
9335 let cache = cache
9346 .filter(|_| job.allow_write && cache_guard.is_some())
9347 .map(Path::to_path_buf);
9348 let spawned_tx = spawned_tx.clone();
9349 let repository_wait = repository_wait.clone();
9350 let coordination = state.config.coordination.clone();
9351 let fetched_base = state.fetched_base.clone();
9352 set.spawn(async move {
9353 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9357 Ok(p) => p,
9358 Err(_) => {
9359 return (
9360 i,
9361 job.seat,
9362 AgentOutcome::ResourceBlocked(format!(
9363 "no parallel slot became free within {}s",
9364 slot_limit.as_secs()
9365 )),
9366 );
9367 }
9368 };
9369 spawned_tx.send(job.seat.key.clone()).ok();
9370 let mut seat = job.seat;
9371 let out = repository_wait
9372 .scope(agent::invoke(
9373 &job.spec,
9374 &mut seat,
9375 &Invocation {
9376 scope: crate::agent::InvocationScope::Repository,
9377 coordination: coordination.as_ref(),
9378 fetched_base: fetched_base.as_ref(),
9379 cwd: &job.cwd,
9380 prompt: &job.prompt,
9381 timeout: job.timeout,
9382 allow_write: job.allow_write,
9383 unsandboxed: false,
9384 sessions: job.sessions,
9385 artifacts: &job.artifacts,
9386 stem: &job.stem,
9387 run: &run,
9388 node: &node,
9389 cache_dir: cache.as_deref(),
9390 attachments: &attachments,
9391 writable: &[],
9392 },
9393 ))
9394 .await;
9395 let out = match out {
9396 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9397 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9398 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9406 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9407 Ok(o) => AgentOutcome::Failed(format!(
9408 "process exited with no usable output ({})",
9409 crate::notices::exit_status(o.exit_code)
9410 )),
9411 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9412 AgentOutcome::ResourceBlocked(e.to_string())
9413 }
9414 Err(e) => AgentOutcome::Failed(e.to_string()),
9415 };
9416 (i, seat, out)
9417 });
9418 }
9419 drop(spawned_tx);
9420 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9421 loop {
9422 let joined = tokio::select! {
9423 Some(key) = spawned_rx.recv() => {
9424 state.seat_spawned(&key);
9426 if let Err(e) = state.save() {
9427 tracing::warn!("could not persist a seat's start: {e:#}");
9428 }
9429 continue;
9430 }
9431 joined = set.join_next() => joined,
9432 };
9433 let Some(joined) = joined else {
9434 break;
9435 };
9436 let (i, seat, out) = match joined {
9437 Ok(v) => v,
9438 Err(e) => {
9442 tracing::error!("agent task panicked: {e}");
9443 continue;
9444 }
9445 };
9446 state.seat_finished(&seat.key);
9447 if let AgentOutcome::ResourceBlocked(why) = &out
9448 && state.resource_block.is_none()
9449 {
9450 state.resource_block = Some(crate::run::ResourceBlock {
9451 node: node.to_owned(),
9452 kind: if why.contains("repository coordination") {
9453 "repository coordination"
9454 } else {
9455 "parallel slot"
9456 }
9457 .to_owned(),
9458 detail: why.clone(),
9459 at: Timestamp::now(),
9460 });
9461 state.event("resource", format!("{node}: no agent was started - {why}"));
9462 }
9463 record_jobs(state, node, round, &seat.key, &out);
9464 if let Err(e) = state.save() {
9465 tracing::warn!("could not persist a seat's completion: {e:#}");
9466 }
9467 if collected.len() <= i {
9468 collected.resize_with(i + 1, || None);
9469 }
9470 collected[i] = Some((i, seat, out));
9471 }
9472 if state
9478 .active
9479 .values()
9480 .any(|a| a.node == node && a.attempt == attempt)
9481 {
9482 state
9483 .active
9484 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9485 if let Err(e) = state.save() {
9486 tracing::warn!("could not persist the end of a wave: {e:#}");
9487 }
9488 }
9489 if let Some(cache_dir) = cache
9496 && jobs_had_a_writer
9497 {
9498 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9499 }
9500 if let Some(guard) = cache_guard {
9501 guard.release();
9502 }
9503 collected.into_iter().flatten().collect()
9504}
9505
9506fn record_jobs(
9517 state: &mut RunState,
9518 node: &str,
9519 round: Option<usize>,
9520 seat: &str,
9521 out: &AgentOutcome,
9522) {
9523 let commands: &[agent::CommandEvidence] = match out {
9524 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9525 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9526 };
9527 let checked_at = Timestamp::now();
9528 for c in commands {
9529 state.jobs.push(JobRecord {
9530 node: node.to_owned(),
9531 round,
9532 seat: seat.to_owned(),
9533 id: c.id.clone(),
9534 description: c.description.clone(),
9535 checked_at,
9536 status: match c.exit_code {
9537 Some(0) => JobStatus::Completed,
9538 Some(_) => JobStatus::Failed,
9539 None => JobStatus::Unknown,
9540 },
9541 exit_code: c.exit_code,
9542 result_summary: c.result_summary.clone(),
9543 source: c.source.clone(),
9544 });
9545 }
9546}
9547
9548fn round_is_clean(
9569 blocking: usize,
9570 e2e_ok: bool,
9571 answered: usize,
9572 expected: usize,
9573 quota_missing: usize,
9574 policy: IncompleteReviewPolicy,
9575) -> bool {
9576 if blocking != 0 || !e2e_ok {
9577 return false;
9578 }
9579 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9580 return true;
9581 }
9582 answered > 0 && expected - answered <= quota_missing
9583}
9584
9585fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9609 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9610 return Some(RunStatus::Gating);
9611 }
9612 let last = reviews.last()?;
9613 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9614 if reviews.len() < max_rounds && !stagnant {
9615 return None;
9616 }
9617 if last.incomplete() && last.blocking == 0 {
9618 return Some(RunStatus::Blocked);
9619 }
9620 if last.e2e_status() == E2eStatus::ResourceBlocked {
9621 return None;
9622 }
9623 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9624 RunStatus::Gating
9625 } else {
9626 RunStatus::Blocked
9627 })
9628}
9629
9630pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9645 if nudged {
9646 (full / 4).max(Duration::from_secs(120)).min(full)
9647 } else {
9648 full
9649 }
9650}
9651
9652#[allow(clippy::too_many_arguments)]
9681async fn ask_json_wave<T>(
9682 jobs: Vec<SeatJob>,
9683 sem: Arc<Semaphore>,
9684 retries: usize,
9685 roster: &[AgentSpec],
9686 ctx: &WaveCtx<'_>,
9687 losses: &mut Vec<QuotaLoss>,
9688 state: &mut RunState,
9689 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9690) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9691where
9692 T: serde::de::DeserializeOwned + Send + 'static,
9693{
9694 ask_wave_with(
9695 jobs,
9696 sem,
9697 retries,
9698 roster,
9699 ctx,
9700 losses,
9701 state,
9702 &|text: &str| {
9703 let v = verdict::extract_json::<T>(text)?;
9704 validate(&v)?;
9705 Ok(v)
9706 },
9707 )
9708 .await
9709}
9710
9711#[allow(clippy::too_many_arguments)]
9720async fn ask_wave_with<T>(
9721 jobs: Vec<SeatJob>,
9722 sem: Arc<Semaphore>,
9723 retries: usize,
9724 roster: &[AgentSpec],
9725 ctx: &WaveCtx<'_>,
9726 losses: &mut Vec<QuotaLoss>,
9727 state: &mut RunState,
9728 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9729) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9730where
9731 T: Send + 'static,
9732{
9733 let n = jobs.len();
9734 let originals: Vec<SeatJob> = jobs;
9735 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9736 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9737 let mut nudges: Vec<usize> = vec![0; n];
9744 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9748 let mut tried: Vec<BTreeSet<String>> = specs
9749 .iter()
9750 .map(|s| BTreeSet::from([s.id.clone()]))
9751 .collect();
9752 let starts: Vec<usize> = specs
9753 .iter()
9754 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9755 .collect();
9756 let carry = ctx.carry_seats && !roster.is_empty();
9757 let carried: Vec<BTreeSet<String>> = originals
9760 .iter()
9761 .map(|j| {
9762 state
9763 .seat_history
9764 .get(&j.seat.key)
9765 .filter(|_| carry)
9766 .map(|h| h.failed.clone())
9767 .unwrap_or_default()
9768 })
9769 .collect();
9770 let mut prev: Vec<Option<FailClass>> = originals
9771 .iter()
9772 .map(|j| {
9773 state
9774 .seat_history
9775 .get(&j.seat.key)
9776 .filter(|_| carry)
9777 .and_then(|h| h.last_fail.clone())
9778 })
9779 .collect();
9780 let next_agent =
9781 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9782 let others: BTreeSet<String> = specs
9783 .iter()
9784 .enumerate()
9785 .filter(|(j, _)| *j != i)
9786 .map(|(_, s)| s.id.clone())
9787 .collect();
9788 pick_successor(
9789 roster,
9790 starts[i],
9791 tried,
9792 carry.then(|| &carried[i]),
9793 &others,
9794 )
9795 .cloned()
9796 };
9797 let mut fresh: Vec<Option<String>> = vec![None; n];
9798 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9799 let mut pending: Vec<usize> = (0..n).collect();
9800
9801 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9806 for round in 0..max_rounds {
9807 if pending.is_empty() {
9808 break;
9809 }
9810 let mut batch = Vec::with_capacity(pending.len());
9811 let mut renudged: Vec<&str> = Vec::new();
9812 for &i in &pending {
9813 let src = &originals[i];
9814 let full: &str = match &src.handover {
9819 Some(h) if specs[i].id != src.spec.id => h,
9820 _ => &src.prompt,
9821 };
9822 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9825 (
9826 full.to_owned(),
9827 src.timeout,
9828 format!("{}-{word}-{}", src.stem, specs[i].id),
9829 )
9830 } else if nudges[i] == 0 {
9831 (full.to_owned(), src.timeout, src.stem.clone())
9832 } else {
9833 renudged.push(src.seat.key.as_str());
9834 let why = done[i]
9835 .as_ref()
9836 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9837 .unwrap_or_else(|| "no parsable answer".to_owned());
9838 let nudge = prompt::nudge(&why);
9839 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9840 let prompt = if nudged {
9841 nudge
9842 } else {
9843 format!("{full}\n\n---\n\n{nudge}")
9844 };
9845 (
9846 prompt,
9847 retry_budget(src.timeout, nudged),
9848 format!("{}-retry{}", src.stem, nudges[i]),
9849 )
9850 };
9851 batch.push(SeatJob {
9852 spec: specs[i].clone(),
9853 seat: seats[i].clone(),
9854 cwd: src.cwd.clone(),
9855 prompt,
9856 timeout,
9857 allow_write: src.allow_write,
9858 sessions: src.sessions,
9859 artifacts: src.artifacts.clone(),
9860 stem,
9861 handover: None,
9862 });
9863 }
9864
9865 if !renudged.is_empty() {
9866 state.event(
9867 ctx.node,
9868 format!("retry {round}: re-asking {}", renudged.join(", ")),
9869 );
9870 }
9871 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9872 let mut still = Vec::new();
9873 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9874 seats[i] = seat;
9875 if let AgentOutcome::ResourceBlocked(why) = &out {
9878 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9879 continue;
9880 }
9881 let class = FailClass::of(&out);
9882 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9885 && nudges[i] < retries
9886 && !roster.is_empty();
9887 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9888 let next = should_hand_over(prev[i].as_ref(), &cur)
9889 .then(|| next_agent(i, &tried[i], &specs))
9890 .flatten();
9891 if carry {
9892 let h = state
9893 .seat_history
9894 .entry(originals[i].seat.key.clone())
9895 .or_default();
9896 h.failed.insert(specs[i].id.clone());
9897 h.last_fail = Some(cur.clone());
9898 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9899 h.last_ok = None;
9900 }
9901 if let Err(e) = state.save() {
9904 tracing::warn!("could not persist a seat's failure history: {e:#}");
9905 }
9906 }
9907 if let Some(next) = next {
9908 record_handover(
9909 state,
9910 ctx.node,
9911 &originals[i].seat.key,
9912 &specs[i].id,
9913 &next.id,
9914 &cur,
9915 &fail_reason(&out),
9916 );
9917 tried[i].insert(next.id.clone());
9918 prev[i] = Some(cur.clone());
9919 seats[i] =
9920 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9921 specs[i] = next;
9922 fresh[i] = Some(cur.stem_word().to_owned());
9923 nudges[i] = 0;
9924 done[i] = Some(Err(anyhow::anyhow!(
9925 "handed over after: {}",
9926 fail_reason(&out)
9927 )));
9928 still.push(i);
9929 continue;
9930 }
9931 }
9932 if carry
9933 && !nudge_first
9934 && let Some(cur) = class.clone()
9935 {
9936 let h = state
9938 .seat_history
9939 .entry(originals[i].seat.key.clone())
9940 .or_default();
9941 h.failed.insert(specs[i].id.clone());
9942 h.last_fail = Some(cur);
9943 }
9944 let parsed = match out {
9945 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9946 AgentOutcome::Quota(o) => {
9947 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9948 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9949 }
9950 AgentOutcome::Dropped(o) => {
9954 let why = o
9955 .dropped
9956 .as_ref()
9957 .map(|d| d.why.as_str())
9958 .unwrap_or("the CLI ended the stream without delivering its answer");
9959 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9960 }
9961 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9962 Err(anyhow::anyhow!(e))
9963 }
9964 };
9965 let quota = class == Some(FailClass::Quota);
9966 let failed = parsed.is_err();
9967 done[i] = Some(parsed);
9968 if carry && !failed {
9969 let h = state
9970 .seat_history
9971 .entry(originals[i].seat.key.clone())
9972 .or_default();
9973 h.failed.remove(&specs[i].id);
9974 h.last_ok = Some(specs[i].id.clone());
9975 h.last_fail = None;
9976 }
9977 let agent_failure = class.is_some()
9985 && !nudge_first
9986 && !roster.is_empty()
9987 && next_agent(i, &tried[i], &specs).is_some();
9988 if failed && !quota && !agent_failure && nudges[i] < retries {
9989 nudges[i] += 1;
9990 still.push(i);
9991 }
9992 }
9993 pending = still;
9994 }
9995
9996 for (i, q) in last_quota.into_iter().enumerate() {
9999 if let Some(reset) = q {
10000 losses.push(QuotaLoss {
10001 seat: originals[i].seat.key.clone(),
10002 node: ctx.node.to_owned(),
10003 at: Timestamp::now(),
10004 reset,
10005 });
10006 }
10007 }
10008
10009 seats
10010 .into_iter()
10011 .zip(done)
10012 .zip(nudges)
10013 .map(|((seat, res), attempts)| {
10014 (
10015 seat,
10016 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10017 attempts,
10018 )
10019 })
10020 .collect()
10021}
10022
10023fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10025 results
10026 .iter()
10027 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10028}
10029
10030fn block_wave(
10034 jobs: Vec<SeatJob>,
10035 state: &mut RunState,
10036 node: &str,
10037 attempt: usize,
10038 why: &str,
10039) -> Vec<(usize, SeatState, AgentOutcome)> {
10040 state.resource_wait = None;
10041 state
10042 .active
10043 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10044 if let Err(e) = state.save() {
10045 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10046 }
10047 jobs.into_iter()
10048 .enumerate()
10049 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10050 .collect()
10051}
10052
10053async fn acquire_cache_lease(
10066 state: &mut RunState,
10067 cache_dir: &Path,
10068 owner: &crate::cache::Owner,
10069 budget: Duration,
10070 context: &str,
10071 cancel: &(dyn Fn() -> bool + Sync),
10072) -> Result<Option<crate::cache::Guard>> {
10073 let home = crate::run::home();
10074 let started = Instant::now();
10075 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10076 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10077 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10078 Err(e) => {
10079 state.event(
10080 "verify",
10081 format!("{context}: could not check the shared build cache: {e:#}"),
10082 );
10083 if let Err(e2) = state.save() {
10084 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10085 }
10086 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10087 }
10088 };
10089 state.event(
10090 "verify",
10091 format!(
10092 "{context}: waiting for the shared build cache at {} ({})",
10093 cache_dir.display(),
10094 busy.describe()
10095 ),
10096 );
10097 state.resource_wait = Some(crate::run::ResourceWait {
10100 node: context.to_owned(),
10101 kind: "shared build cache".to_owned(),
10102 holder: busy.describe(),
10103 since: Timestamp::now(),
10104 limit_secs: budget.as_secs(),
10105 });
10106 if let Err(e) = state.save() {
10107 tracing::warn!("could not persist a cache wait: {e:#}");
10108 }
10109 let remaining = budget.saturating_sub(started.elapsed());
10110 let waited = crate::cache::wait_for_unless(
10111 &home,
10112 cache_dir,
10113 owner,
10114 remaining,
10115 Duration::from_secs(5),
10116 cancel,
10117 )
10118 .await;
10119 state.resource_wait = None;
10120 match waited {
10121 Ok(g) => Ok(g),
10122 Err(e) => {
10123 state.event("verify", format!("{context}: {e:#}"));
10124 if let Err(e2) = state.save() {
10125 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10126 }
10127 Err(e)
10128 }
10129 }
10130}
10131
10132#[derive(Debug)]
10137struct CacheCheckError(String);
10138
10139impl std::fmt::Display for CacheCheckError {
10140 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10141 f.write_str(&self.0)
10142 }
10143}
10144
10145impl std::error::Error for CacheCheckError {}
10146
10147#[allow(clippy::too_many_arguments)]
10168async fn with_cache_lease<'s, F, Fut>(
10169 state: &'s mut RunState,
10170 cache_dir: Option<&Path>,
10171 node: &str,
10172 seat: &str,
10173 worktree: &Path,
10174 head: &str,
10175 budget: Duration,
10176 context: &str,
10177 wait: &crate::repo_lock::WaitPolicy,
10178 body: F,
10179) -> Result<(Vec<CommandOutcome>, bool)>
10180where
10181 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10182 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10183{
10184 let started = Instant::now();
10187 let home = crate::run::home();
10188 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10189 let guard = if let Some(cache_dir) = cache_dir {
10190 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10191 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10192 wait.cancelled()
10193 })
10194 .await
10195 .and_then(|g| {
10196 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10197 }) {
10198 Ok(g) => Some(g),
10199 Err(e) => {
10200 return Ok((
10201 vec![CommandOutcome {
10202 command: "(waiting for the shared build cache)".into(),
10203 code: None,
10204 output_tail: e.to_string(),
10205 duration_ms: started.elapsed().as_millis() as u64,
10206 resource_blocked: true,
10207 }],
10208 false,
10209 ));
10210 }
10211 }
10212 } else {
10213 None
10214 };
10215 let _coordination = coordinate_operation(state, worktree, wait).await?;
10216 if let Some(cache_dir) = cache_dir {
10217 let identity = crate::cache::Identity::new(worktree, head);
10218 if let Err(e) = crate::cache::ensure_fresh(
10219 &home,
10220 cache_dir,
10221 &identity,
10222 &state.config.verify.cargo_manifests(),
10223 ) {
10224 state.event(
10225 "verify",
10226 format!(
10227 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10228 worktree.display(),
10229 short(head)
10230 ),
10231 );
10232 if let Some(guard) = guard {
10233 guard.release();
10234 }
10235 return Ok((
10236 vec![CommandOutcome {
10237 command: "(confirming the shared build cache is fresh)".into(),
10238 code: None,
10239 output_tail: e.to_string(),
10240 duration_ms: started.elapsed().as_millis() as u64,
10241 resource_blocked: true,
10242 }],
10243 false,
10244 ));
10245 }
10246 }
10247 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10248 if !timed_out_pids.is_empty() {
10249 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10250 }
10251 drop(_coordination);
10254 if let Some(guard) = guard {
10255 guard.release();
10256 }
10257 Ok((outcomes, retried))
10258}
10259
10260async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10272 wait_for_pids_with(
10273 pids,
10274 crate::proc::pid_alive,
10275 LEASE_RELEASE_POLL,
10276 LEASE_RELEASE_MAX_WAIT,
10277 )
10278 .await;
10279}
10280
10281async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10287 pids: &[u32],
10288 alive: F,
10289 poll: Duration,
10290 max_wait: Duration,
10291) {
10292 let deadline = Instant::now() + max_wait;
10293 loop {
10294 if pids.iter().all(|&pid| !alive(pid)) {
10295 return;
10296 }
10297 if Instant::now() >= deadline {
10298 return;
10299 }
10300 tokio::time::sleep(poll).await;
10301 }
10302}
10303
10304fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10311 outcomes.iter().any(|o| o.resource_blocked)
10312}
10313
10314enum GateFix {
10316 Retry,
10318 Stop,
10321 Defer,
10324}
10325
10326fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10334 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10335 red.peek().is_some()
10336 && red.all(|o| {
10337 !o.resource_blocked
10338 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10339 && !o.output_tail.trim().is_empty()
10340 })
10341}
10342
10343fn e2e_outcome_label(o: &CommandOutcome) -> String {
10347 if o.ok() {
10348 return "pass".to_owned();
10349 }
10350 let reason = if o.build_failed() {
10351 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10352 } else {
10353 format!("FAIL ({:?})", o.code)
10354 };
10355 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10356}
10357
10358async fn run_e2e_with_retry(
10366 state: &mut RunState,
10367 shell: &[String],
10368 commands: &[String],
10369 worktree: &Path,
10370 timeout: Duration,
10371 context: &str,
10372) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10373 let (mut e2e, mut timed_out_pids) = run_commands(
10374 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10375 )
10376 .await;
10377 for o in &e2e {
10378 state.event(
10379 "verify",
10380 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10381 );
10382 }
10383 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10387 if verify_retried {
10388 state.event(
10389 "verify",
10390 format!(
10391 "{context}: verify could not build/link, not a test result — retrying once \
10392 before concluding"
10393 ),
10394 );
10395 let retried = run_commands(
10396 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10397 )
10398 .await;
10399 e2e = retried.0;
10400 timed_out_pids.extend(retried.1);
10403 for o in &e2e {
10404 state.event(
10405 "verify",
10406 format!(
10407 "{context}: retry `{}` -> {}",
10408 o.command,
10409 e2e_outcome_label(o)
10410 ),
10411 );
10412 }
10413 }
10414 (e2e, verify_retried, timed_out_pids)
10415}
10416
10417#[allow(clippy::too_many_arguments)]
10433async fn run_commands(
10434 state: &mut RunState,
10435 node: &str,
10436 task: &str,
10437 attempt: usize,
10438 shell: &[String],
10439 commands: &[String],
10440 cwd: &Path,
10441 timeout: Duration,
10442) -> (Vec<CommandOutcome>, Vec<u32>) {
10443 if commands.is_empty() {
10444 return (Vec::new(), Vec::new());
10449 }
10450 let mut out = Vec::new();
10451 let mut timed_out_pids = Vec::new();
10452 let total = commands.len();
10453 for (idx, command) in commands.iter().enumerate() {
10454 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10455 if let Err(e) = state.save() {
10456 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10457 }
10458 let started = Instant::now();
10459 let mut cmd = tokio::process::Command::new(&shell[0]);
10460 cmd.quiet();
10461 cmd.args(&shell[1..])
10462 .arg(command)
10463 .current_dir(cwd)
10464 .stdin(std::process::Stdio::null())
10465 .stdout(std::process::Stdio::piped())
10466 .stderr(std::process::Stdio::piped())
10467 .kill_on_drop(true);
10468 let spawned = cmd.spawn();
10469 let (code, body) = match spawned {
10470 Ok(child) => {
10471 let pid = child.id();
10476 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10477 Ok(Ok(o)) => {
10478 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10479 body.push_str(&String::from_utf8_lossy(&o.stderr));
10480 (o.status.code(), body)
10481 }
10482 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10483 Err(_) => {
10484 if let Some(pid) = pid {
10485 timed_out_pids.push(pid);
10486 }
10487 (None, format!("timed out after {}s", timeout.as_secs()))
10488 }
10489 }
10490 }
10491 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10492 };
10493 out.push(CommandOutcome {
10494 command: command.clone(),
10495 code,
10496 output_tail: tail(&body, OUTPUT_TAIL),
10497 duration_ms: started.elapsed().as_millis() as u64,
10498 resource_blocked: false,
10499 });
10500 }
10501 state.task_finished(task);
10502 if let Err(e) = state.save() {
10503 tracing::warn!("could not persist the end of {task}: {e:#}");
10504 }
10505 (out, timed_out_pids)
10506}
10507
10508fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10520 let repo = repo.display();
10521 match style {
10522 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10523 MergeStyle::Squash => {
10524 let subject = message
10527 .lines()
10528 .next()
10529 .unwrap_or(branch)
10530 .replace(['\\', '"', '$', '`'], "");
10531 format!(
10532 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10533 )
10534 }
10535 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10536 }
10537}
10538
10539const PR_TITLE_MAX: usize = 240;
10550
10551fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10562 let line = queue::first_line(text)?;
10563 let chars: Vec<char> = line.chars().collect();
10564 let (kept, mark) = if chars.len() <= max {
10565 (line.to_owned(), "")
10566 } else if let Some(end) = sentence_end(&chars, max) {
10567 (chars[..end].iter().collect::<String>(), "")
10568 } else {
10569 let room = max.saturating_sub(3);
10570 let cut = if chars[room].is_whitespace() {
10573 room
10574 } else {
10575 chars[..room]
10576 .iter()
10577 .rposition(|c| c.is_whitespace())
10578 .unwrap_or(room)
10579 };
10580 let head: String = chars[..cut].iter().collect();
10581 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10582 (head.to_owned(), "...")
10583 };
10584 if kept.is_empty() {
10585 return None;
10586 }
10587 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10588 return None;
10589 }
10590 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10591 return None;
10592 }
10593 Some(format!("{kept}{mark}"))
10594}
10595
10596fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10600 const MIN: usize = 20;
10601 for i in MIN..max.min(chars.len()) {
10602 if !matches!(chars[i], '.' | '!' | '?') {
10603 continue;
10604 }
10605 let Some(&next) = chars.get(i + 1) else {
10606 continue;
10607 };
10608 if !next.is_whitespace() {
10609 continue;
10610 }
10611 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10612 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10613 continue;
10614 }
10615 let word: String = chars[..i]
10616 .iter()
10617 .rev()
10618 .take_while(|c| !c.is_whitespace())
10619 .collect::<Vec<_>>()
10620 .into_iter()
10621 .rev()
10622 .collect();
10623 let word = word.to_ascii_lowercase();
10624 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10625 continue;
10626 }
10627 let end = chars[..i]
10628 .iter()
10629 .rposition(|c| !c.is_whitespace())
10630 .map_or(i, |p| p + 1);
10631 return Some(end);
10632 }
10633 None
10634}
10635
10636struct PrMessage {
10641 title: String,
10642 body: String,
10643}
10644
10645impl PrMessage {
10646 fn commit_message(&self) -> String {
10650 format!("{}\n\n{}", self.title, self.body)
10651 }
10652}
10653
10654fn title_marker(line: &str) -> Option<&str> {
10656 let line = line.trim();
10657 let head = line.get(..6)?;
10658 head.eq_ignore_ascii_case("title:")
10659 .then(|| line[6..].trim())
10660}
10661
10662fn summary_title(summary: &str) -> Option<String> {
10667 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10668 let raw = title_marker(first)?;
10669 if raw.is_empty() {
10670 return None;
10671 }
10672 let title = queue::title_from(raw, PR_TITLE_MAX);
10673 let lower = title.to_ascii_lowercase();
10674 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10675 return None;
10676 }
10677 Some(title)
10678}
10679
10680const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10682
10683pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10685
10686fn is_review_run(state: &RunState) -> bool {
10690 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10691}
10692
10693fn review_title(state: &RunState) -> Option<String> {
10698 english_subject(
10699 state.reviewed_commits.as_ref()?.first()?,
10700 state.config.graph.github_english(),
10701 )
10702}
10703
10704fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10707 let raw = raw.trim();
10708 if raw.is_empty() {
10709 return None;
10710 }
10711 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10712 return None;
10713 }
10714 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10715 return None;
10716 }
10717 let title = queue::title_from(raw, PR_TITLE_MAX);
10718 let lower = title.to_ascii_lowercase();
10719 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10720 return None;
10721 }
10722 Some(title)
10723}
10724
10725#[derive(Debug, Clone, PartialEq, Eq)]
10728struct BranchFacts {
10729 commits: Vec<(String, String)>,
10731 stat: String,
10733}
10734
10735const STAT_FILE_LINES: usize = 25;
10737const COMMITS_MAX_CHARS: usize = 20_000;
10739
10740async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10743 let commits = git::commit_log(repo, base, branch).await.ok()?;
10744 if commits.is_empty() {
10745 return None;
10746 }
10747 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10748 let lines: Vec<&str> = stat.lines().collect();
10749 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10750 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10751 let more = format!(" ... {omitted} more file(s)");
10752 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10753 kept.push(&more);
10754 kept.push(lines[lines.len() - 1]);
10755 kept.join("\n")
10756 } else {
10757 lines.join("\n")
10758 };
10759 Some(BranchFacts { commits, stat })
10760}
10761
10762fn markdown_safe(text: &str) -> String {
10765 text.replace("</details>", "</details>")
10766 .replace("\x60\x60\x60", "~~~")
10767}
10768
10769fn neutral_title(state: &RunState, winner: char) -> String {
10770 format!(
10771 "chore: land candidate {} of run {}",
10772 winner.to_ascii_uppercase(),
10773 state.id
10774 )
10775}
10776
10777pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10783 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10784 ""
10785 } else {
10786 pr_title
10787 }
10788}
10789
10790pub fn landing_subject_source(state: &RunState) -> String {
10794 if is_review_run(state) {
10795 let winner = state.candidates.first().map_or('A', |c| c.label);
10796 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10797 }
10798 state.instruction.clone()
10799}
10800
10801fn summary_without_title(summary: &str) -> String {
10804 let mut lines = summary.trim().lines().peekable();
10805 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10806 lines.next();
10807 }
10808 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10809}
10810
10811#[cfg(test)]
10823fn pr_message(state: &RunState, winner: char) -> PrMessage {
10824 pr_message_with(state, winner, None)
10825}
10826
10827#[cfg(test)]
10830fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10831 let raw = pr_message_raw(
10832 state,
10833 winner,
10834 facts,
10835 crate::pr_outcome::FinalHead::Unreported,
10836 ChangeNote::NotAsked,
10837 );
10838 let id = crate::scrub::Identity::current();
10839 PrMessage {
10840 title: crate::scrub::scrub(&raw.title, &id),
10841 body: crate::scrub::scrub(&raw.body, &id),
10842 }
10843}
10844
10845#[derive(Debug, Clone, Copy)]
10847enum ChangeNote<'a> {
10848 NotAsked,
10850 Summary(&'a crate::run::ChangeSummary),
10852 Unavailable(&'a str),
10854}
10855
10856fn pr_message_raw(
10857 state: &RunState,
10858 winner: char,
10859 facts: Option<&BranchFacts>,
10860 head: crate::pr_outcome::FinalHead<'_>,
10861 change: ChangeNote<'_>,
10862) -> PrMessage {
10863 if !matches!(change, ChangeNote::NotAsked) {
10864 let (title, mut body) = match change {
10865 ChangeNote::Summary(c) => (
10866 c.title.clone(),
10867 format!("## Summary\n\n{}\n\n", c.body.trim()),
10868 ),
10869 ChangeNote::Unavailable(why) => (
10870 neutral_title(state, winner),
10871 format!(
10872 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10873 ),
10874 ),
10875 ChangeNote::NotAsked => unreachable!(),
10876 };
10877 body.push_str(&public_pr_evidence(state, head));
10878 body.push_str(&format!(
10879 "\n---\nmagi:run/{} magi:candidate-{}\n",
10880 state.id,
10881 winner.to_ascii_lowercase()
10882 ));
10883 return PrMessage { title, body };
10884 }
10885 pr_preview(state, winner, facts, head)
10886}
10887
10888fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10891 let mut body = crate::pr_outcome::review_rounds(state);
10892 body.push_str(&crate::pr_outcome::final_verification(state, head));
10893 let open = state.last_round_findings();
10894 if !open.is_empty() {
10895 body.push_str("## Open review findings\n\n");
10896 for f in open {
10897 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10898 }
10899 body.push('\n');
10900 }
10901 if let Some(c) = state.change_summary.as_ref()
10902 && head == crate::pr_outcome::FinalHead::At(&c.head)
10903 {
10904 body.push_str(&format!(
10905 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10906 c.base, c.head
10907 ));
10908 }
10909 body
10910}
10911
10912fn pr_preview(
10914 state: &RunState,
10915 winner: char,
10916 facts: Option<&BranchFacts>,
10917 head: crate::pr_outcome::FinalHead<'_>,
10918) -> PrMessage {
10919 let summary = state
10920 .candidates
10921 .iter()
10922 .find(|c| c.label == winner)
10923 .map(|c| c.summary.as_str())
10924 .unwrap_or_default();
10925 let review = is_review_run(state);
10928 let title = if review {
10929 facts
10930 .and_then(|f| {
10931 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10932 })
10933 .or_else(|| review_title(state))
10934 .or_else(|| {
10935 state
10936 .candidates
10937 .iter()
10938 .find(|c| c.label == winner)
10939 .filter(|c| !c.branch.starts_with("magi/"))
10940 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10941 })
10942 .unwrap_or_else(|| neutral_title(state, winner))
10943 } else {
10944 summary_title(summary).unwrap_or_else(|| {
10945 english_title(
10946 &state.instruction,
10947 PR_TITLE_MAX,
10948 state.config.graph.github_english(),
10949 )
10950 .unwrap_or_else(|| neutral_title(state, winner))
10951 })
10952 };
10953
10954 let mut body = String::new();
10955 let what = summary_without_title(summary);
10956 if !what.is_empty() {
10957 body.push_str("## Summary\n\n");
10958 body.push_str(&what);
10959 body.push_str("\n\n");
10960 }
10961
10962 body.push_str(&crate::pr_outcome::review_rounds(state));
10964 body.push_str(&crate::pr_outcome::final_verification(state, head));
10965 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10966
10967 let open = state.open_findings();
10968 if !open.is_empty() {
10969 body.push_str("## Open review findings\n\n");
10970 for f in &open {
10971 body.push_str(&format!(
10972 "- `{}` [{:?}] {}\n",
10973 f.id, f.severity, "Context retained in local review records."
10974 ));
10975 }
10976 body.push('\n');
10977 }
10978
10979 if let Some(fix) = fix
10980 && !fix.rejected.is_empty()
10981 {
10982 body.push_str("## Declined by the fixer\n\n");
10983 for r in &fix.rejected {
10984 body.push_str(&format!(
10985 "- `{}`: {}\n",
10986 r.id, "Reason retained in local review records."
10987 ));
10988 }
10989 body.push('\n');
10990 }
10991
10992 if review {
10993 body.push_str("## Commits under review\n\n");
10995 if let Some(facts) = facts {
10996 let mut left = COMMITS_MAX_CHARS;
10997 for (i, (subject, text)) in facts.commits.iter().enumerate() {
10998 let mut entry = format!("- {}\n", markdown_safe(subject));
10999 for l in markdown_safe(text).lines() {
11000 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11001 }
11002 if left == 0 {
11003 body.push_str(&format!(
11004 "- ... {} more commit(s)\n",
11005 facts.commits.len() - i
11006 ));
11007 break;
11008 }
11009 if entry.len() > left {
11010 let mut end = left;
11013 while !entry.is_char_boundary(end) {
11014 end -= 1;
11015 }
11016 entry.truncate(end);
11017 entry.push_str("\n ... (truncated)\n");
11018 left = 0;
11019 } else {
11020 left -= entry.len();
11021 }
11022 body.push_str(&entry);
11023 }
11024 if !facts.stat.trim().is_empty() {
11025 body.push_str(&format!(
11026 "\n## Diff stat\n\n```\n{}\n```\n",
11027 markdown_safe(facts.stat.trim())
11028 ));
11029 }
11030 } else {
11031 match &state.reviewed_commits {
11032 Some(subjects) => {
11033 for s in subjects {
11034 body.push_str(&format!("- {}\n", s.trim()));
11035 }
11036 }
11037 None => {
11038 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11041 body.push_str(rest.trim());
11042 body.push('\n');
11043 }
11044 }
11045 }
11046 } else {
11047 let task = state.instruction.trim();
11048 let task = if task.is_empty() {
11049 "(empty task)"
11050 } else {
11051 task
11052 };
11053 body.push_str(&format!(
11054 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11055 task.replace("</details>", "</details>")
11056 ));
11057 }
11058
11059 body.push_str(&format!(
11060 "\n---\nmagi:run/{} magi:candidate-{}\n",
11061 state.id,
11062 winner.to_ascii_lowercase()
11063 ));
11064
11065 PrMessage { title, body }
11066}
11067
11068fn seeded_instruction(state: &RunState) -> String {
11072 match refs::describe(&state.seeds) {
11073 Some(facts) => format!(
11074 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11075 Candidates start from the unmerged branch named above, when there \
11076 is one, and carry any unmerged commit named by sha as a \
11077 cherry-pick. Check that this is what the task meant before \
11078 building on it.",
11079 state.instruction
11080 ),
11081 None => state.instruction.clone(),
11082 }
11083}
11084
11085async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11089 let base = &state.base_branch;
11090 let mut against = base.clone();
11091 if mode == MergeMode::Pr {
11092 let remote = &state.config.merge.remote;
11096 let tracking = format!("{remote}/{base}");
11097 let fetched = git::fetch(repo, remote, base).await;
11098 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11099 against = tracking;
11100 } else {
11101 return false;
11102 }
11103 }
11104 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11105}
11106
11107fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11110 let mut detail = format!(
11111 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11112 nothing to open a pull request for"
11113 );
11114 match refs::describe(&state.seeds) {
11115 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11116 None => detail.push_str(
11117 "\nThe task names no existing branch or commit; if it means to land work \
11118 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11119 ),
11120 }
11121 detail
11122}
11123
11124#[derive(Debug, PartialEq, Eq)]
11127enum PrPlan {
11128 Create,
11129 Adopt { url: String, title: String },
11130 Stop(String),
11131}
11132
11133fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11136 match found {
11137 Ok(land::OpenPr::None) => PrPlan::Create,
11138 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11139 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11140 "several open pull requests exist for this branch, not picking one: {}",
11141 urls.join(" ")
11142 )),
11143 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11144 }
11145}
11146
11147pub(crate) async fn gh_pr_create(
11149 cwd: &Path,
11150 base: &str,
11151 head: &str,
11152 title: &str,
11153 body: &str,
11154) -> Result<String> {
11155 let out = tokio::process::Command::new("gh")
11156 .args([
11157 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11158 ])
11159 .current_dir(cwd)
11160 .quiet()
11161 .stdin(std::process::Stdio::null())
11162 .output()
11163 .await
11164 .context("spawn gh")?;
11165 if out.status.success() {
11166 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11167 } else {
11168 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11169 }
11170}
11171
11172async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11174 match std::fs::symlink_metadata(path) {
11175 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11176 git::worktree_remove_clean(repo, path).await?;
11179 return Ok(());
11180 }
11181 Err(e) => return Err(e.into()),
11182 Ok(_) => {}
11183 }
11184 if !git::is_clean(path).await? {
11185 bail!(
11186 "worktree has uncommitted work; refusing removal: {}",
11187 path.display()
11188 );
11189 }
11190 if !git::worktree_remove_clean(repo, path).await? {
11191 bail!("worktree removal refused: {}", path.display());
11192 }
11193 Ok(())
11194}
11195
11196pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11205 if crate::pr_recovery::sources_retained(home, &state.id) {
11206 return Ok(Vec::new());
11207 }
11208 if !state.candidate_preservation.is_empty() {
11209 bail!("candidate preservation incomplete; refusing cleanup");
11210 }
11211 let repo = state.repo.clone();
11212 let wait = repository_wait(state, None);
11215 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11216 Ok(lease) => lease,
11217 Err(e) => {
11218 state.save_under(home)?;
11219 return Err(e);
11220 }
11221 };
11222 let root = state.worktree_root();
11223 let winner = state.tally.as_ref().map(|t| t.winner);
11224 let mut removed = Vec::new();
11225
11226 for i in 0..state.candidates.len() {
11227 let c = state.candidates[i].clone();
11228 let is_winner = Some(c.label) == winner;
11229 if is_winner && !drop_winner {
11230 continue;
11231 }
11232 let existed = c.worktree.exists();
11233 remove_preserved_worktree(&repo, &c.worktree).await?;
11236 if existed {
11237 removed.push(c.worktree.to_string_lossy().into_owned());
11238 }
11239 let handed_over = state.released_branches.contains(&c.branch);
11242 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11243 if !git::branch_delete(&repo, &c.branch).await? {
11244 bail!("candidate branch removal refused");
11245 }
11246 removed.push(c.branch.clone());
11247 }
11248 state.candidates[i].folded = true;
11249 }
11250
11251 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11252 let path = name.path();
11253 let keep = !drop_winner
11254 && winner.is_some_and(|w| {
11255 path.file_name()
11256 .is_some_and(|n| n == format!("cand-{w}").as_str())
11257 });
11258 if keep {
11259 continue;
11260 }
11261 remove_preserved_worktree(&repo, &path).await?;
11262 removed.push(path.to_string_lossy().into_owned());
11263 }
11264
11265 remove_if_empty(&root);
11274
11275 if state.enabled_worktree_config && drop_winner {
11276 git::release_worktree_config(&repo).await.ok();
11280 state.enabled_worktree_config = false;
11281 }
11282 state.save_under(home)?;
11283 Ok(removed)
11284}
11285
11286fn remove_if_empty(dir: &Path) {
11297 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11298 std::fs::remove_dir(dir).ok();
11299 }
11300}
11301
11302pub fn worst_open(state: &RunState) -> Option<Severity> {
11304 state
11305 .reviews
11306 .last()?
11307 .reviews
11308 .iter()
11309 .flat_map(|r| r.findings.iter())
11310 .map(|f| f.severity)
11311 .max()
11312}
11313
11314async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11318 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11319 git::acquire_worktree_config(repo).await?;
11320 state.enabled_worktree_config = true;
11321 }
11322 Ok(())
11323}
11324
11325#[cfg(test)]
11326mod tests {
11327 #[tokio::test]
11328 async fn launcher_base_fetch_requires_remote_success() {
11329 let tmp = tempfile::tempdir().unwrap();
11330 let repo = tmp.path().join("repo");
11331 let remote = tmp.path().join("remote");
11332 std::fs::create_dir(&repo).unwrap();
11333 std::fs::create_dir(&remote).unwrap();
11334 git::git(&remote, &["init", "--bare", "-b", "main"])
11335 .await
11336 .unwrap();
11337 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11338 git::git(
11339 &repo,
11340 &[
11341 "-c",
11342 "user.name=Fixture",
11343 "-c",
11344 "user.email=fixture@example.invalid",
11345 "commit",
11346 "--allow-empty",
11347 "-m",
11348 "fixture",
11349 ],
11350 )
11351 .await
11352 .unwrap();
11353 git::git(
11354 &repo,
11355 &["remote", "add", "origin", remote.to_str().unwrap()],
11356 )
11357 .await
11358 .unwrap();
11359 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11360 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11361 assert_eq!(
11362 resolve_base(&repo, "main", "origin", None)
11363 .await
11364 .unwrap()
11365 .commit,
11366 head
11367 );
11368 #[cfg(unix)]
11369 {
11370 let policy: crate::config::Coordination = toml::from_str(
11371 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11372 ).unwrap();
11373 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11374 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11375 let held = crate::repo_lock::SeatLease::operation_with(
11376 &repo,
11377 Some(&policy),
11378 &crate::repo_lock::WaitPolicy::default(),
11379 )
11380 .await
11381 .unwrap();
11382 assert!(held.is_some());
11383 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11384 tokio::pin!(fetch);
11385 assert!(
11386 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11387 .await
11388 .is_err()
11389 );
11390 assert!(
11391 !repo.join(".git/FETCH_HEAD").exists(),
11392 "no fetch before admission"
11393 );
11394 drop(held);
11395 let fetched = fetch.await.unwrap();
11396 assert_eq!(fetched.commit, head);
11397 assert_eq!(fetched.remote, "origin");
11398 assert_eq!(fetched.branch, "main");
11399 }
11400 git::git(
11402 &repo,
11403 &[
11404 "remote",
11405 "set-url",
11406 "origin",
11407 tmp.path().join("missing").to_str().unwrap(),
11408 ],
11409 )
11410 .await
11411 .unwrap();
11412 let error = resolve_base(&repo, "main", "origin", None)
11413 .await
11414 .unwrap_err()
11415 .to_string();
11416 assert!(error.contains("cannot read origin/main"));
11417 assert!(error.contains("refusing to branch off"));
11418 }
11419
11420 #[test]
11421 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11422 let leaked = vec!["chore(deps): update a crate".to_owned()];
11423 let own = "fix(daemon): apply a chosen action";
11424 assert!(should_retitle("", own, &leaked));
11425 assert!(should_retitle(
11426 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11427 own,
11428 &leaked
11429 ));
11430 assert!(should_retitle(
11431 "chore: land candidate A of run 1",
11432 own,
11433 &leaked
11434 ));
11435 assert!(should_retitle(
11436 "magi: candidate A (uncommitted work)",
11437 own,
11438 &leaked
11439 ));
11440 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11441 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11442 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11443 }
11444
11445 #[test]
11446 fn pr_merge_plan_creates_adopts_or_stops() {
11447 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11448 assert_eq!(
11449 pr_merge_plan(Ok(land::OpenPr::One {
11450 url: "u".into(),
11451 title: "t".into()
11452 })),
11453 PrPlan::Adopt {
11454 url: "u".into(),
11455 title: "t".into()
11456 }
11457 );
11458 let PrPlan::Stop(many) =
11459 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11460 else {
11461 panic!("many must stop");
11462 };
11463 assert!(many.contains('a') && many.contains('b'));
11464 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11465 panic!("a failed lookup must stop");
11466 };
11467 assert!(err.contains("bad token"));
11468 }
11469
11470 use super::*;
11471 use crate::run::GateStatus;
11472 use std::collections::BTreeMap;
11473 use std::time::Duration;
11474
11475 fn conductor() -> AgentSpec {
11476 AgentSpec {
11477 id: "conductor".to_owned(),
11478 kind: crate::config::AgentKind::Command,
11479 model: None,
11480 command: vec!["true".to_owned()],
11481 extra_args: Vec::new(),
11482 env: BTreeMap::new(),
11483 prompt_delivery: None,
11484 }
11485 }
11486
11487 fn spec(id: &str) -> AgentSpec {
11488 AgentSpec {
11489 id: id.to_owned(),
11490 kind: crate::config::AgentKind::Command,
11491 model: None,
11492 command: vec!["true".to_owned()],
11493 extra_args: Vec::new(),
11494 env: BTreeMap::new(),
11495 prompt_delivery: None,
11496 }
11497 }
11498
11499 fn ids(xs: &[&str]) -> BTreeSet<String> {
11500 xs.iter().map(|s| (*s).to_owned()).collect()
11501 }
11502
11503 #[test]
11504 fn pick_successor_skips_an_agent_another_seat_holds() {
11505 let roster = [spec("a"), spec("b"), spec("c")];
11507 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11508 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11509 }
11510
11511 #[test]
11512 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11513 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11515 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11516 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11517 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11518 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11519 }
11520
11521 #[test]
11522 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11523 let roster = [spec("a"), spec("b")];
11524 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11525 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11526 let carried = ids(&["b"]);
11527 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11528 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11529 }
11530
11531 #[test]
11532 fn pick_successor_returns_none_without_any_untried_successor() {
11533 let roster = [spec("a"), spec("b")];
11534 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11535 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11536 let carried = ids(&["a"]);
11537 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11538 }
11539
11540 #[test]
11541 fn pick_successor_rescue_avoids_another_seats_occupant() {
11542 let roster = [spec("a"), spec("b"), spec("c")];
11544 let carried = ids(&["b", "c"]);
11545 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11546 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11547 }
11548
11549 #[test]
11550 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11551 let roster = [spec("a"), spec("b"), spec("c")];
11552 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11553 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11554 }
11555
11556 #[test]
11557 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11558 let roster = [spec("a"), spec("b"), spec("c")];
11559 let failed = ids(&["a", "b", "c"]);
11560 let mut tried = ids(&["b"]);
11562 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11563 assert_eq!(first.id, "a");
11564 tried.insert(first.id.clone());
11565 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11566 assert_eq!(second.id, "c");
11567 tried.insert(second.id.clone());
11568 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11569 }
11570
11571 #[test]
11572 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11573 let roster = [spec("a"), spec("b")];
11574 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11575 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11576 }
11577
11578 #[test]
11579 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11580 let roster = [spec("a"), spec("b"), spec("c")];
11581 let h = SeatHistory {
11582 failed: ids(&["a"]),
11583 last_ok: Some("b".to_owned()),
11584 last_fail: None,
11585 };
11586 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11587 let gone = SeatHistory {
11589 last_ok: Some("zzz".to_owned()),
11590 ..h.clone()
11591 };
11592 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11593 let failed = SeatHistory {
11594 failed: ids(&["a", "b"]),
11595 last_ok: Some("b".to_owned()),
11596 last_fail: None,
11597 };
11598 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11599 }
11600
11601 #[test]
11602 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11603 let first = SeatState::new("review-1", "alpha", 7);
11604 let next = handover_seat("review-1", "gamma", 7);
11605 assert_ne!(first.claude_session, next.claude_session);
11606 }
11607
11608 #[test]
11609 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11610 let mut state = state_with_summary("x", "y");
11611 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11612 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11613 assert_ne!(a.claude_session, b.claude_session);
11614 }
11615
11616 #[test]
11617 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11618 let roster = [spec("a"), spec("b")];
11619 let h = SeatHistory {
11620 failed: ids(&["a", "b"]),
11621 ..SeatHistory::default()
11622 };
11623 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11624 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11625 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11626 }
11627
11628 #[test]
11635 fn failure_signature_ignores_numbers_and_paths() {
11636 assert_eq!(
11637 failure_signature("exited with Some(2) and no usable output"),
11638 failure_signature("exited with Some(137) and no usable output")
11639 );
11640 assert_eq!(
11641 failure_signature("process exited with no usable output (exit code 2)"),
11642 failure_signature("process exited with no usable output (exit code 137)")
11643 );
11644 for (old, new) in [
11645 (
11646 "exited with Some(2) and no usable output",
11647 "process exited with no usable output (exit code 2)",
11648 ),
11649 (
11650 "exited with None and no usable output",
11651 "process exited with no usable output (exit code unavailable)",
11652 ),
11653 ] {
11654 let previous = FailClass::Other(failure_signature(old));
11655 let current = FailClass::Other(failure_signature(new));
11656 assert_eq!(previous, current, "resumption retains the failure class");
11657 assert!(!should_hand_over(Some(&previous), ¤t));
11658 }
11659 assert_eq!(
11660 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11661 failure_signature("cannot open /var/x.txt: denied")
11662 );
11663 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11664 }
11665
11666 #[test]
11667 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11668 let other = FailClass::Other("x".into());
11669 assert!(should_hand_over(None, &FailClass::Quota));
11670 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11671 assert!(should_hand_over(
11672 Some(&FailClass::Timeout),
11673 &FailClass::Timeout
11674 ));
11675 assert!(should_hand_over(None, &other));
11676 assert!(!should_hand_over(Some(&other), &other));
11677 assert!(should_hand_over(
11678 Some(&other),
11679 &FailClass::Other("y".into())
11680 ));
11681 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11683 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11684 }
11685
11686 #[test]
11687 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11688 let a = SeatState::new("judge-1", "alpha", 7);
11689 let b = handover_seat("judge-1", "beta", 7);
11690 assert_ne!(a.claude_session, b.claude_session);
11691 assert_eq!(b.turns, 0);
11692 }
11693
11694 #[test]
11695 fn a_timeout_is_classified_apart_from_other_failures() {
11696 assert_eq!(
11697 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11698 Some(FailClass::Timeout)
11699 );
11700 assert!(matches!(
11701 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11702 Some(FailClass::Other(_))
11703 ));
11704 }
11705
11706 #[test]
11707 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11708 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11709 let tried = BTreeSet::from(["beta".to_owned()]);
11710 let next = next_untried_in_roster(&roster, 1, &tried);
11713 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11714 }
11715
11716 #[test]
11717 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11718 let roster = vec![spec("alpha"), spec("beta")];
11719 let tried = BTreeSet::from(["beta".to_owned()]);
11720 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11724 }
11725
11726 #[test]
11727 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11728 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11729 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11730 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11734 }
11735
11736 #[test]
11737 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11738 let roster = vec![spec("a"), spec("a"), spec("b")];
11739 let tried = BTreeSet::from(["a".to_owned()]);
11740 let next = next_untried_in_roster(&roster, 0, &tried);
11741 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11742 }
11743
11744 #[test]
11745 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11746 let roster = vec![spec("a"), spec("b")];
11747 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11748 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11749 }
11750
11751 #[test]
11752 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11753 let dir = tempfile::tempdir().unwrap();
11754 let bay = dir.path().join("ffff");
11755
11756 remove_if_empty(&bay);
11758 assert!(!bay.exists());
11759
11760 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11763 remove_if_empty(&bay);
11764 assert!(bay.exists(), "non-empty directory must survive");
11765
11766 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11768 remove_if_empty(&bay);
11769 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11770 }
11771
11772 #[test]
11781 fn a_full_panel_that_found_nothing_is_clean() {
11782 assert!(round_is_clean(
11783 0,
11784 true,
11785 2,
11786 2,
11787 0,
11788 IncompleteReviewPolicy::Block
11789 ));
11790 }
11791
11792 #[test]
11793 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11794 assert!(!round_is_clean(
11795 0,
11796 true,
11797 1,
11798 2,
11799 0,
11800 IncompleteReviewPolicy::Block
11801 ));
11802 }
11803
11804 #[test]
11805 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11806 assert!(!round_is_clean(
11807 1,
11808 true,
11809 1,
11810 2,
11811 0,
11812 IncompleteReviewPolicy::Warn
11813 ));
11814 }
11815
11816 #[test]
11817 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11818 assert!(round_is_clean(
11819 0,
11820 true,
11821 1,
11822 2,
11823 0,
11824 IncompleteReviewPolicy::Warn
11825 ));
11826 }
11827
11828 #[test]
11829 fn a_full_panel_with_an_open_finding_is_not_clean() {
11830 assert!(!round_is_clean(
11831 1,
11832 true,
11833 2,
11834 2,
11835 0,
11836 IncompleteReviewPolicy::Block
11837 ));
11838 }
11839
11840 #[test]
11841 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11842 assert!(!round_is_clean(
11843 0,
11844 false,
11845 2,
11846 2,
11847 0,
11848 IncompleteReviewPolicy::Block
11849 ));
11850 }
11851
11852 #[test]
11859 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11860 assert!(round_is_clean(
11863 0,
11864 true,
11865 1,
11866 2,
11867 1,
11868 IncompleteReviewPolicy::Block
11869 ));
11870 }
11871
11872 #[test]
11873 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11874 assert!(!round_is_clean(
11877 0,
11878 true,
11879 1,
11880 2,
11881 0,
11882 IncompleteReviewPolicy::Block
11883 ));
11884 }
11885
11886 #[test]
11887 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11888 assert!(!round_is_clean(
11889 1,
11890 true,
11891 1,
11892 2,
11893 1,
11894 IncompleteReviewPolicy::Block
11895 ));
11896 assert!(!round_is_clean(
11897 0,
11898 false,
11899 1,
11900 2,
11901 1,
11902 IncompleteReviewPolicy::Block
11903 ));
11904 }
11905
11906 #[test]
11907 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11908 assert!(!round_is_clean(
11912 0,
11913 true,
11914 0,
11915 2,
11916 2,
11917 IncompleteReviewPolicy::Block
11918 ));
11919 }
11920
11921 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11922 CommandOutcome {
11923 command: "test".to_owned(),
11924 code,
11925 output_tail: String::new(),
11926 duration_ms: 0,
11927 resource_blocked,
11928 }
11929 }
11930
11931 #[test]
11932 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11933 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11934 assert!(
11935 !verify_inconclusive(&[outcome(Some(1), false)]),
11936 "an ordinary failure is still evidence about the patch"
11937 );
11938 assert!(verify_inconclusive(&[outcome(None, true)]));
11939 assert!(
11940 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11941 "one inconclusive outcome taints the whole batch"
11942 );
11943 assert!(!verify_inconclusive(&[]));
11944 }
11945
11946 #[tokio::test]
11947 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11948 let calls = std::sync::atomic::AtomicUsize::new(0);
11952 let started = Instant::now();
11953 wait_for_pids_with(
11954 &[123],
11955 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11956 Duration::from_millis(5),
11957 Duration::from_secs(5),
11958 )
11959 .await;
11960 assert!(
11961 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11962 "must keep checking rather than deciding on the first answer"
11963 );
11964 assert!(
11965 started.elapsed() < Duration::from_secs(1),
11966 "must return the moment it is confirmed dead, not wait out the ceiling"
11967 );
11968 }
11969
11970 #[tokio::test]
11971 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11972 let started = Instant::now();
11973 wait_for_pids_with(
11974 &[123],
11975 |_| true, Duration::from_millis(5),
11977 Duration::from_millis(30),
11978 )
11979 .await;
11980 let elapsed = started.elapsed();
11981 assert!(
11982 elapsed >= Duration::from_millis(30),
11983 "must not give up before its own ceiling: {elapsed:?}"
11984 );
11985 assert!(
11986 elapsed < Duration::from_secs(1),
11987 "must not wait past its own ceiling either: {elapsed:?}"
11988 );
11989 }
11990
11991 #[tokio::test]
11992 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
11993 let started = Instant::now();
11994 wait_for_pids_with(
11995 &[],
11996 |_| true,
11997 Duration::from_secs(5),
11998 Duration::from_secs(5),
11999 )
12000 .await;
12001 assert!(
12002 started.elapsed() < Duration::from_millis(200),
12003 "an empty pid list has nothing to confirm"
12004 );
12005 }
12006
12007 fn review_round(
12013 clean: bool,
12014 blocking: usize,
12015 answered: usize,
12016 expected: usize,
12017 progressed: bool,
12018 e2e_ok: bool,
12019 ) -> ReviewRound {
12020 ReviewRound {
12021 round: 1,
12022 head: "h".to_owned(),
12023 verified_head: None,
12024 verified_at: None,
12025 reviews: Vec::new(),
12026 e2e: vec![CommandOutcome {
12027 command: "test".to_owned(),
12028 code: Some(if e2e_ok { 0 } else { 1 }),
12029 output_tail: String::new(),
12030 duration_ms: 0,
12031 resource_blocked: false,
12032 }],
12033 verify_retried: false,
12034 e2e_deferred: false,
12035 e2e_defer_reason: None,
12036 fix: None,
12037 blocking,
12038 answered,
12039 expected,
12040 clean,
12041 progressed,
12042 vote_split: false,
12043 reconsideration: Vec::new(),
12044 verdict: None,
12045 }
12046 }
12047
12048 #[test]
12049 fn review_conclusion_is_none_when_nothing_has_run() {
12050 assert_eq!(review_conclusion(&[], 3), None);
12051 }
12052
12053 #[test]
12054 fn review_conclusion_is_none_while_rounds_remain() {
12055 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12056 assert_eq!(review_conclusion(&rounds, 3), None);
12057 }
12058
12059 #[test]
12060 fn review_conclusion_is_gating_once_a_round_is_clean() {
12061 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12062 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12063 }
12064
12065 #[test]
12066 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12067 let rounds = vec![
12068 review_round(false, 1, 2, 2, true, true),
12069 review_round(false, 1, 2, 2, true, true),
12070 ];
12071 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12072 }
12073
12074 #[test]
12075 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12076 let rounds = vec![
12077 review_round(false, 1, 2, 2, true, true),
12078 review_round(false, 1, 2, 2, true, false),
12079 ];
12080 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12081 }
12082
12083 #[test]
12084 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12085 let mut blocked = review_round(false, 1, 2, 2, true, false);
12092 blocked.e2e[0].resource_blocked = true;
12093 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12094 assert_eq!(review_conclusion(&rounds, 2), None);
12095 }
12096
12097 #[test]
12098 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12099 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12101 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12102 }
12103
12104 #[test]
12105 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12106 let rounds = vec![
12107 review_round(false, 1, 2, 2, false, true),
12108 review_round(false, 1, 2, 2, false, true),
12109 ];
12110 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12111 }
12112
12113 fn secs(n: u64) -> Duration {
12114 Duration::from_secs(n)
12115 }
12116
12117 #[tokio::test]
12120 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12121 ask_test_home();
12122 let tmp = tempfile::tempdir().unwrap();
12123 let repo = tmp.path().join("repo");
12124 std::fs::create_dir_all(&repo).unwrap();
12125 init_repo(&repo);
12126 let g = |args: &[&str]| {
12127 let out = std::process::Command::new("git")
12128 .args(args)
12129 .current_dir(&repo)
12130 .quiet()
12131 .output()
12132 .unwrap();
12133 assert!(
12134 out.status.success(),
12135 "{}",
12136 String::from_utf8_lossy(&out.stderr)
12137 );
12138 };
12139 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12140 g(&["checkout", "-b", "historical"]);
12141 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12142 g(&["add", "."]);
12143 g(&["commit", "-m", "old alternative"]);
12144 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12145 g(&["checkout", "-b", "magi/task-selected", &base]);
12146 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12147 g(&["add", "."]);
12148 g(&["commit", "-m", "measurement infrastructure"]);
12149 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12150 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12151 g(&["add", "."]);
12152 g(&["commit", "-m", "dependent implementation"]);
12153 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12154 g(&["checkout", "main"]);
12155 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12156 g(&["add", "."]);
12157 g(&["commit", "-m", "unrelated base change"]);
12158 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12159 let origin = tmp.path().join("origin.git");
12160 g(&[
12161 "clone",
12162 "--bare",
12163 repo.to_str().unwrap(),
12164 origin.to_str().unwrap(),
12165 ]);
12166 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12167 g(&["fetch", "origin"]);
12168 let selection = crate::pr_recovery::Selection {
12169 url: "https://example.test/project/pull/1".into(),
12170 branch: "magi/task-selected".into(),
12171 head: head.clone(),
12172 source_head: None,
12173 disposition: Some(crate::pr_recovery::Disposition::Review),
12174 instruction: "preserve measured behavior".into(),
12175 approved_at: jiff::Timestamp::now(),
12176 };
12177 let mut template = runner_at(RunStatus::Reviewing);
12178 template.state.repo = repo.clone();
12179 template.state.base_commit = current.clone();
12180 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12181 template.state.origin = Some(crate::run::Origin::queue("selected"));
12182 template.state.pr_head = Some(selection.branch.clone());
12183 template.state.pr_recovery = Some(selection.clone());
12184 let mut runner =
12185 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12186 .await
12187 .unwrap();
12188 let wt = runner.state.winner().unwrap().worktree.clone();
12189 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12190 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12191 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12192 assert!(runner.state.seeds.is_empty());
12193 assert_eq!(runner.state.instruction, "preserve measured behavior");
12194 runner.sync_to_base().await.unwrap();
12195 assert_eq!(
12196 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12197 "prerequisite"
12198 );
12199 assert_eq!(
12200 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12201 "measured implementation"
12202 );
12203 assert_eq!(
12204 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12205 "current main change"
12206 );
12207 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12208 assert_eq!(
12209 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12210 head
12211 );
12212
12213 let mut partial = runner_at(RunStatus::Reviewing);
12215 partial.state.repo = repo.clone();
12216 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12217 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12218 run: Some("exit 1".into()),
12219 ..Default::default()
12220 }];
12221 let selection = runner.state.pr_recovery.clone().unwrap();
12222 partial.state.pr_recovery = Some(selection.clone());
12223 partial.state.pr_head = Some(selection.branch.clone());
12224 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12225 let id = partial.state.id.clone();
12226 let error = match Runner::open_selected(
12227 &repo,
12228 partial.state,
12229 partial.roles,
12230 1,
12231 runner.state.base_commit.clone(),
12232 selection,
12233 )
12234 .await
12235 {
12236 Ok(_) => panic!("setup must fail"),
12237 Err(e) => e,
12238 };
12239 assert_eq!(
12240 error
12241 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12242 .unwrap()
12243 .run,
12244 id
12245 );
12246 let failed = RunState::load(&id).unwrap();
12247 assert_eq!(failed.status, RunStatus::Blocked);
12248 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12249 assert!(Runner::resume(&id).is_err());
12250 runner.state.status = RunStatus::Ready;
12251 runner.state.driver_exited = true;
12252 runner.state.save().unwrap();
12253 let mut retry = runner_at(RunStatus::Reviewing);
12254 retry.state.repo = repo.clone();
12255 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12256 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12257 reselected.approved_at = jiff::Timestamp::now();
12258 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12259 retry.state.pr_head = Some(reselected.branch.clone());
12260 retry.state.pr_recovery = Some(reselected.clone());
12261 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12262 .await
12263 .unwrap();
12264 let restarted = Runner::open_selected(
12265 &repo,
12266 retry.state,
12267 retry.roles,
12268 1,
12269 runner.state.base_commit.clone(),
12270 reselected,
12271 )
12272 .await
12273 .unwrap();
12274 assert_eq!(
12275 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12276 .await
12277 .unwrap(),
12278 head
12279 );
12280 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12281 assert_eq!(
12282 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12283 head
12284 );
12285 }
12286
12287 #[tokio::test]
12288 #[cfg(unix)]
12289 async fn fallback_and_interrupted_restoration_require_coordination() {
12290 use std::os::fd::AsRawFd as _;
12291 ask_test_home();
12292 let tmp = tempfile::tempdir().unwrap();
12293 let mut runner = registered_runner(tmp.path()).await;
12294 runner.state.config.graph.timeout_resource_wait = 1;
12295 let winner = runner.state.candidates[0].clone();
12296 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12297 let leftover = winner.worktree.join("leftover");
12298 std::fs::write(&leftover, "untrusted edits").unwrap();
12299 let next = tmp.path().join("next-fixer");
12300 let mut failed = spec("first");
12301 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12302 let mut successor = spec("second");
12303 successor.command = vec![
12304 "sh".into(),
12305 "-c".into(),
12306 format!("touch '{}'; echo done", next.display()),
12307 ];
12308 let attempts = vec![
12309 (failed.clone(), "first".into()),
12310 (successor.clone(), "second".into()),
12311 ];
12312 let lock_path = runner.state.repo.join(".git/updater.lock");
12313 let external = std::fs::File::create(&lock_path).unwrap();
12314 assert_eq!(
12315 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12316 0
12317 );
12318 let artifacts = tmp.path().join("artifacts");
12319 let build = |spec, seat| SeatJob {
12320 spec,
12321 seat,
12322 cwd: winner.worktree.clone(),
12323 prompt: "fixture".into(),
12324 timeout: Duration::from_secs(10),
12325 allow_write: true,
12326 sessions: false,
12327 artifacts: artifacts.clone(),
12328 stem: "fix".into(),
12329 handover: None,
12330 };
12331 let (_, _, out) = runner
12332 .ask_fixer_among(
12333 attempts.clone(),
12334 "graft",
12335 None,
12336 Some((&winner.worktree, &head)),
12337 build,
12338 )
12339 .await;
12340 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12341 assert!(leftover.exists(), "contention must prevent clean");
12342 assert!(!next.exists());
12343 assert!(runner.state.handovers.is_empty());
12344 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12346 from_run: "earlier".into(),
12347 generation: 1,
12348 mode: crate::recovery::Mode::VerifyRepair,
12349 reasons: Vec::new(),
12350 reviewed_head: None,
12351 rounds: Vec::new(),
12352 open_findings: Vec::new(),
12353 evidence: None,
12354 criteria_inherited: true,
12355 repair: crate::recovery::Repair::Started {
12356 before: head.clone(),
12357 },
12358 });
12359 assert!(
12360 runner
12361 .recovery_repair(&winner)
12362 .await
12363 .unwrap_err()
12364 .is::<crate::repo_lock::ResourceBlocked>()
12365 );
12366 assert!(matches!(
12367 runner.state.recovery.as_ref().unwrap().repair,
12368 crate::recovery::Repair::Started { .. }
12369 ));
12370 assert!(leftover.exists());
12371 drop(external);
12372 runner.recovery_repair(&winner).await.unwrap();
12373 assert!(!leftover.exists());
12374 assert!(matches!(
12375 runner.state.recovery.as_ref().unwrap().repair,
12376 crate::recovery::Repair::Done { .. }
12377 ));
12378 std::fs::remove_file(&lock_path).unwrap();
12380 std::fs::create_dir(&lock_path).unwrap();
12381 std::fs::write(&leftover, "untrusted edits").unwrap();
12382 let (_, _, out) = runner
12383 .ask_fixer_among(
12384 attempts,
12385 "graft",
12386 None,
12387 Some((&winner.worktree, &head)),
12388 build,
12389 )
12390 .await;
12391 assert!(matches!(out, AgentOutcome::Failed(_)));
12392 assert!(leftover.exists());
12393 assert!(!next.exists());
12394 assert_eq!(
12395 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12396 head
12397 );
12398 }
12399
12400 #[cfg(unix)]
12401 async fn registered_runner(root: &Path) -> Runner {
12402 ask_test_home();
12403 let repo = root.join("repo");
12404 std::fs::create_dir_all(&repo).unwrap();
12405 init_repo(&repo);
12406 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12407 let mut runner = runner_at(RunStatus::Prep);
12408 runner.state.repo = repo.clone();
12409 runner.state.config.coordination = crate::config::test_coordination();
12410 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12411 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12412 runner.state.config.graph.advise = false;
12413 runner.roles.implementers = vec![spec("implement")];
12414 runner.roles.judges = vec![spec("judge")];
12415 runner.prep().await.unwrap();
12416 runner
12417 }
12418
12419 #[tokio::test]
12420 #[cfg(unix)]
12421 async fn parked_implementer_persists_completed_work() {
12422 for registered in [false, true] {
12423 let tmp = tempfile::tempdir().unwrap();
12424 let mut runner = registered_runner(tmp.path()).await;
12425 if !registered {
12426 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12427 }
12428 let marker = tmp.path().join("started");
12429 let release = tmp.path().join("release");
12430 let mut implementer = spec("implement");
12431 implementer.command = vec![
12432 "sh".into(),
12433 "-c".into(),
12434 format!(
12435 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12436 marker.display(),
12437 release.display()
12438 ),
12439 ];
12440 runner.roles.implementers = vec![implementer.clone()];
12441 runner.state.config.agents = vec![implementer];
12442 let pause = runner.pause.clone();
12443 let (result, ()) = tokio::join!(runner.implement(), async {
12444 tokio::time::timeout(Duration::from_secs(10), async {
12445 while !marker.exists() {
12446 tokio::time::sleep(Duration::from_millis(10)).await;
12447 }
12448 })
12449 .await
12450 .unwrap();
12451 pause.park();
12452 std::fs::write(&release, "go").unwrap();
12453 });
12454 result.unwrap();
12455 let candidate = &runner.state.candidates[0];
12456 assert!(candidate.commits > 0, "{candidate:?}");
12457 assert!(candidate.summary.contains("saved-result"));
12458 assert_eq!(candidate.files, 1);
12459 let before = candidate.commits;
12460 runner.implement().await.unwrap();
12461 assert_eq!(runner.state.candidates[0].commits, before);
12462 }
12463 }
12464
12465 #[tokio::test]
12466 #[cfg(unix)]
12467 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12468 let tmp = tempfile::tempdir().unwrap();
12469 let mut runner = registered_runner(tmp.path()).await;
12470 let winner = runner.state.candidates[0].clone();
12471 let cache = tmp.path().join("cache");
12472 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12473 runner.state.config.graph.timeout_resource_wait = 0;
12474 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12475 let crate::cache::AcquireOutcome::Acquired(held) =
12476 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12477 else {
12478 panic!("cache must be free")
12479 };
12480 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12481 from_run: "earlier".into(),
12482 generation: 1,
12483 mode: crate::recovery::Mode::VerifyRepair,
12484 reasons: Vec::new(),
12485 reviewed_head: None,
12486 rounds: Vec::new(),
12487 open_findings: Vec::new(),
12488 evidence: None,
12489 criteria_inherited: true,
12490 repair: crate::recovery::Repair::NotStarted,
12491 });
12492 runner.recovery_repair(&winner).await.unwrap();
12493 assert!(matches!(
12494 runner.state.recovery.as_ref().unwrap().repair,
12495 crate::recovery::Repair::WaitingForRestore { .. }
12496 ));
12497 drop(held);
12498 runner.state.resource_block = None;
12499 runner.state.config.graph.timeout_resource_wait = 1;
12500 runner.recovery_repair(&winner).await.unwrap();
12501 assert!(matches!(
12502 runner.state.recovery.as_ref().unwrap().repair,
12503 crate::recovery::Repair::Done { .. }
12504 ));
12505 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12506 }
12507
12508 #[tokio::test]
12509 #[cfg(unix)]
12510 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12511 use std::os::fd::AsRawFd as _;
12512 for graft in [false, true] {
12513 let tmp = tempfile::tempdir().unwrap();
12514 let mut runner = registered_runner(tmp.path()).await;
12515 runner.state.config.graph.timeout_resource_wait = 1;
12516 let winner = runner.state.candidates[0].clone();
12517 let marker = tmp.path().join("fixer-started");
12518 let mut fixer = spec("fixer");
12519 fixer.command = vec![
12520 "sh".into(),
12521 "-c".into(),
12522 format!("echo started >> '{}'; echo done", marker.display()),
12523 ];
12524 runner.roles.fixer = Some(vec![fixer]);
12525 if graft {
12526 let mut loser = winner.clone();
12527 loser.label = 'B';
12528 loser.index = 1;
12529 runner.state.candidates.push(loser);
12530 runner.state.config.graph.graft_runner_up = true;
12531 runner.state.tally = Some(Tally {
12532 first_choice: BTreeMap::new(),
12533 borda: BTreeMap::new(),
12534 winner: winner.label,
12535 rankings: 1,
12536 unanimous_initial: true,
12537 deliberated: false,
12538 changed_votes: 0,
12539 unanimous_final: true,
12540 tie_break: None,
12541 judges: 1,
12542 present: 1,
12543 quorum: 1,
12544 met_quorum: true,
12545 uncontested: None,
12546 });
12547 runner.state.judgements.push(crate::run::Judgement {
12548 judge: 1,
12549 seat: "judge-1".into(),
12550 agent: "judge".into(),
12551 ranking: vec![winner.label, 'B'],
12552 reasons: BTreeMap::new(),
12553 confidence: None,
12554 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12555 order: vec![0, 1],
12556 failed: None,
12557 duration_ms: 0,
12558 });
12559 } else {
12560 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12561 from_run: "earlier".into(),
12562 generation: 1,
12563 mode: crate::recovery::Mode::VerifyRepair,
12564 reasons: Vec::new(),
12565 reviewed_head: None,
12566 rounds: Vec::new(),
12567 open_findings: Vec::new(),
12568 evidence: None,
12569 criteria_inherited: true,
12570 repair: crate::recovery::Repair::NotStarted,
12571 });
12572 }
12573 let reader = crate::repo_lock::SeatLease::for_checkout(
12574 &winner.worktree,
12575 false,
12576 crate::config::test_coordination().as_ref(),
12577 )
12578 .await
12579 .unwrap();
12580 let pause = runner.pause.clone();
12581 let (result, ()) = tokio::join!(
12582 async {
12583 if graft {
12584 runner.graft().await
12585 } else {
12586 runner.recovery_repair(&winner).await
12587 }
12588 },
12589 async {
12590 tokio::time::sleep(Duration::from_millis(100)).await;
12591 pause.park();
12592 }
12593 );
12594 result.unwrap();
12595 assert!(!marker.exists(), "a blocked seat must not spawn");
12596 runner.state = RunState::load(&runner.state.id).unwrap();
12598 if graft {
12599 assert_eq!(
12600 runner.state.graft.as_ref().unwrap().outcome,
12601 GraftOutcome::WaitingForRestore
12602 );
12603 } else {
12604 assert!(matches!(
12605 runner.state.recovery.as_ref().unwrap().repair,
12606 crate::recovery::Repair::WaitingForRestore { .. }
12607 ));
12608 }
12609 drop(reader);
12610 runner.pause = Pause::new();
12611 runner.state.resource_block = None;
12612 let external =
12614 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12615 assert_eq!(
12616 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12617 0
12618 );
12619 let leftover = winner.worktree.join("leftover");
12620 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12621 let result = if graft {
12622 runner.graft().await
12623 } else {
12624 runner.recovery_repair(&winner).await
12625 };
12626 assert!(
12627 result
12628 .unwrap_err()
12629 .is::<crate::repo_lock::ResourceBlocked>()
12630 );
12631 assert!(leftover.exists());
12632 assert!(!marker.exists());
12633 drop(external);
12634 runner.state.resource_block = None;
12635 let result = if graft {
12636 runner.graft().await
12637 } else {
12638 runner.recovery_repair(&winner).await
12639 };
12640 result.unwrap();
12641 assert!(!leftover.exists(), "restore must precede retry");
12642 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12643 if graft {
12644 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12645 } else {
12646 assert!(matches!(
12647 runner.state.recovery.as_ref().unwrap().repair,
12648 crate::recovery::Repair::Done { .. }
12649 ));
12650 }
12651 }
12652 }
12653
12654 #[tokio::test]
12655 #[cfg(unix)]
12656 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12657 use std::os::fd::AsRawFd as _;
12658 let tmp = tempfile::tempdir().unwrap();
12659 let mut runner = registered_runner(tmp.path()).await;
12660 runner.state.config.graph.timeout_resource_wait = 1;
12661 let winner = runner.state.candidates[0].clone();
12662 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12663 runner.state.tally = Some(Tally {
12664 first_choice: BTreeMap::new(),
12665 borda: BTreeMap::new(),
12666 winner: winner.label,
12667 rankings: 0,
12668 unanimous_initial: false,
12669 deliberated: false,
12670 changed_votes: 0,
12671 unanimous_final: false,
12672 tie_break: None,
12673 judges: 0,
12674 present: 0,
12675 quorum: 0,
12676 met_quorum: true,
12677 uncontested: Some("fixture".into()),
12678 });
12679 runner.state.graft = Some(GraftRecord {
12680 pre: pre.clone(),
12681 post: None,
12682 lines: None,
12683 limit: None,
12684 agent: None,
12685 outcome: GraftOutcome::Running,
12686 requested: Vec::new(),
12687 adoption: None,
12688 adoption_error: None,
12689 });
12690 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12691 git::commit_all(&winner.worktree, "unfinished graft")
12692 .await
12693 .unwrap();
12694 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12695 let leftover = winner.worktree.join("leftover");
12696 std::fs::write(&leftover, "untrusted edits").unwrap();
12697 let path = runner.state.repo.join(".git/updater.lock");
12698 let external = std::fs::File::create(&path).unwrap();
12699 assert_eq!(
12700 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12701 0
12702 );
12703 assert!(
12704 runner
12705 .graft()
12706 .await
12707 .unwrap_err()
12708 .is::<crate::repo_lock::ResourceBlocked>()
12709 );
12710 assert!(matches!(
12711 runner.state.graft.as_ref().unwrap().outcome,
12712 GraftOutcome::Running
12713 ));
12714 assert_eq!(
12715 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12716 unfinished
12717 );
12718 assert!(leftover.exists());
12719 drop(external);
12720 std::fs::remove_file(&path).unwrap();
12721 std::fs::create_dir(&path).unwrap();
12722 assert!(runner.graft().await.is_err());
12723 assert!(matches!(
12724 runner.state.graft.as_ref().unwrap().outcome,
12725 GraftOutcome::Running
12726 ));
12727 assert_eq!(
12728 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12729 unfinished
12730 );
12731 assert!(leftover.exists());
12732 std::fs::remove_dir(&path).unwrap();
12733 runner.graft().await.unwrap();
12734 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12735 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12736 assert!(!leftover.exists());
12737 }
12738
12739 #[tokio::test]
12740 #[cfg(unix)]
12741 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12742 use std::os::fd::AsRawFd as _;
12743 let tmp = tempfile::tempdir().unwrap();
12744 let mut runner = registered_runner(tmp.path()).await;
12745 runner.state.prep_complete = Some(false);
12746 runner.state.config.graph.timeout_resource_wait = 1;
12747 runner.state.save().unwrap();
12748 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12749 assert_eq!(
12750 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12751 0
12752 );
12753 runner.execute().await.unwrap();
12754 let disk = RunState::load(&runner.state.id).unwrap();
12755 assert!(disk.driver_exited);
12756 assert_eq!(disk.driver_pid, None);
12757 assert_eq!(runner.state.status, RunStatus::Blocked);
12758 assert_eq!(
12759 runner.state.resource_block.as_ref().unwrap().kind,
12760 "repository coordination"
12761 );
12762 assert!(runner.state.prep_failure.is_none());
12763 assert!(runner.state.seats.is_empty());
12764 runner.pause.park();
12765 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12768 .await
12769 .unwrap()
12770 .unwrap();
12771 assert!(runner.state.parked);
12772 assert!(runner.state.prep_failure.is_none());
12773 let disk = RunState::load(&runner.state.id).unwrap();
12774 assert!(disk.driver_exited);
12775 assert_eq!(disk.driver_pid, None);
12776 }
12777
12778 #[tokio::test]
12779 #[cfg(unix)]
12780 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12781 use std::os::fd::AsRawFd as _;
12782 let tmp = tempfile::tempdir().unwrap();
12783 let mut runner = registered_runner(tmp.path()).await;
12784 runner.state.prep_complete = Some(false);
12785 runner.state.config.graph.timeout_resource_wait = 30;
12786 runner.state.save().unwrap();
12787 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12788 assert_eq!(
12789 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12790 0
12791 );
12792 assert!(
12793 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12794 .await
12795 .is_err()
12796 );
12797 let disk = RunState::load(&runner.state.id).unwrap();
12798 assert!(disk.driver_exited);
12799 assert_eq!(disk.driver_pid, None);
12800 assert!(disk.status.resumable());
12801 assert!(runner.state.driver_exited);
12802 assert_eq!(runner.state.driver_pid, None);
12803 runner.state.save().unwrap();
12804 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12805 }
12806
12807 #[tokio::test]
12808 #[cfg(unix)]
12809 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12810 let tmp = tempfile::tempdir().unwrap();
12811 let mut runner = registered_runner(tmp.path()).await;
12812 let winner = runner.state.candidates[0].clone();
12813 let cache = tmp.path().join("cache");
12814 let started = tmp.path().join("pre-gate-started");
12815 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12816 runner.state.config.verify.gate =
12817 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12818 let home = crate::run::home();
12819 let owner =
12820 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12821 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12822 crate::cache::AcquireOutcome::Acquired(g) => g,
12823 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12824 };
12825 let mut gate = Box::pin(runner.run_gate(&winner));
12826 assert!(
12827 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12828 .await
12829 .is_err()
12830 );
12831 assert!(!started.exists(), "pre-gate must wait for the cache too");
12832 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12836 let editor = crate::repo_lock::SeatLease::checkout_with(
12837 &winner.worktree,
12838 true,
12839 crate::config::test_coordination().as_ref(),
12840 &wait,
12841 )
12842 .await
12843 .unwrap();
12844 drop(editor);
12845 guard.release();
12846 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12847 .await
12848 .unwrap()
12849 .unwrap();
12850 drop(gate);
12851 assert!(outcomes.iter().all(CommandOutcome::ok));
12852 assert!(started.exists());
12853 }
12854
12855 #[tokio::test]
12856 #[cfg(unix)]
12857 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12858 use std::os::fd::AsRawFd as _;
12859
12860 ask_test_home();
12861 let tmp = tempfile::tempdir().unwrap();
12862 let repo = tmp.path().join("repo");
12863 std::fs::create_dir_all(&repo).unwrap();
12864 init_repo(&repo);
12865 let common = repo.join(".git");
12866 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12867 let mut runner = runner_at(RunStatus::Prep);
12868 runner.state.repo = repo.clone();
12869 runner.state.config.coordination = crate::config::test_coordination();
12870 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12871 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12872 runner.state.config.graph.advise = false;
12873 runner.roles.implementers = vec![spec("implement")];
12874 runner.roles.judges = vec![spec("judge")];
12875 runner.prep().await.unwrap();
12876 let winner = runner.state.candidates[0].clone();
12877 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12880 let completed = tmp.path().join("seat-completed");
12881 runner.roles.implementers[0].command = vec![
12882 "sh".into(),
12883 "-c".into(),
12884 format!("touch '{}'; echo completed", completed.display()),
12885 ];
12886 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12887 assert_eq!(
12888 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12889 0
12890 );
12891 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12892 let mut implement = Box::pin(runner.implement());
12893 tokio::select! {
12894 result = &mut implement => panic!("rescue must wait: {result:?}"),
12895 result = tokio::time::timeout(Duration::from_secs(10), async {
12896 while !completed.exists() {
12897 tokio::time::sleep(Duration::from_millis(20)).await;
12898 }
12899 }) => result.unwrap(),
12900 }
12901 assert!(
12902 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12903 .await
12904 .is_err()
12905 );
12906 assert_eq!(
12907 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12908 before
12909 );
12910 assert_eq!(
12911 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12912 0
12913 );
12914 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12915 .await
12916 .unwrap()
12917 .unwrap();
12918 drop(implement);
12919 assert_ne!(
12920 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12921 before
12922 );
12923 assert!(git::is_clean(&winner.worktree).await.unwrap());
12924
12925 let started = tmp.path().join("pre-gate-started");
12926 let proceed = tmp.path().join("pre-gate-proceed");
12927 let checked = tmp.path().join("gate-ran");
12928 runner.state.config.verify.pre_gate = vec![format!(
12929 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12930 started.display(),
12931 proceed.display()
12932 )];
12933 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12934 assert_eq!(
12935 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12936 0
12937 );
12938 let mut gate = Box::pin(runner.run_gate(&winner));
12939 assert!(
12940 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12941 .await
12942 .is_err()
12943 );
12944 assert!(!started.exists());
12945 assert!(!checked.exists());
12946 assert!(git::is_clean(&winner.worktree).await.unwrap());
12947 assert_eq!(
12948 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12949 0
12950 );
12951 tokio::select! {
12952 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12953 result = tokio::time::timeout(Duration::from_secs(10), async {
12954 while !started.exists() {
12955 tokio::time::sleep(Duration::from_millis(20)).await;
12956 }
12957 }) => result.unwrap(),
12958 }
12959 assert_eq!(
12960 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12961 -1,
12962 "mechanical edits must hold the real lock"
12963 );
12964 std::fs::write(proceed, "ready").unwrap();
12965 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12966 .await
12967 .unwrap()
12968 .unwrap();
12969 drop(gate);
12970 assert!(outcomes.iter().all(CommandOutcome::ok));
12971 assert!(checked.exists());
12972 assert!(runner.state.pre_gate_commit.is_some());
12973 assert!(git::is_clean(&winner.worktree).await.unwrap());
12974 assert_eq!(
12975 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12976 0,
12977 "gate completion must release coordination"
12978 );
12979 drop(outsider);
12980 std::fs::remove_file(common.join("updater.lock")).unwrap();
12981 std::fs::create_dir(common.join("updater.lock")).unwrap();
12982 std::fs::remove_file(&started).unwrap();
12983 std::fs::remove_file(&checked).unwrap();
12984 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12985 let error = runner.run_gate(&winner).await.unwrap_err();
12986 assert!(
12987 error
12988 .to_string()
12989 .contains("acquire mandatory repository lock")
12990 );
12991 assert!(
12992 !started.exists(),
12993 "failed acquisition must prevent pre-gate"
12994 );
12995 assert!(!checked.exists(), "failed acquisition must prevent gate");
12996 assert!(git::is_clean(&winner.worktree).await.unwrap());
12997 assert_eq!(
12998 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12999 head
13000 );
13001 }
13002
13003 #[tokio::test]
13004 #[cfg(unix)]
13005 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13006 use std::os::fd::AsRawFd as _;
13007 use std::time::Duration;
13008
13009 ask_test_home();
13010 let tmp = tempfile::tempdir().unwrap();
13011 let repo = tmp.path().join("repo");
13012 std::fs::create_dir_all(&repo).unwrap();
13013 init_repo(&repo);
13014 let common = repo.join(".git");
13015 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13016 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13017 assert_eq!(
13018 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13019 0
13020 );
13021 let mut runner = runner_at(RunStatus::Prep);
13022 runner.state.repo = repo.clone();
13023 runner.state.config.coordination = crate::config::test_coordination();
13024 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13025 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13026 runner.state.config.graph.advise = true;
13027 runner.state.config.graph.advisors = 1;
13028 runner.roles.implementers = vec![spec("implement")];
13029 runner.roles.judges = vec![spec("judge")];
13030 let root = runner.state.worktree_root();
13031 let started = tmp.path().join("setup-started");
13032 let proceed = tmp.path().join("setup-proceed");
13033 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13034 run: Some(format!(
13035 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13036 started.display(),
13037 proceed.display()
13038 )),
13039 ..Default::default()
13040 }];
13041 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13042 let config = git::git(&repo, &["config", "--local", "--list"])
13043 .await
13044 .unwrap();
13045 let mut prep = Box::pin(runner.prep());
13046 assert!(
13047 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13048 .await
13049 .is_err()
13050 );
13051 assert!(
13052 !root.exists(),
13053 "contention must prevent even checkout creation"
13054 );
13055 assert!(!started.exists(), "contention must prevent setup");
13056 assert_eq!(
13057 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13058 branches
13059 );
13060 assert_eq!(
13061 git::git(&repo, &["config", "--local", "--list"])
13062 .await
13063 .unwrap(),
13064 config
13065 );
13066 drop(lock);
13067 tokio::select! {
13068 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13069 result = tokio::time::timeout(Duration::from_secs(10), async {
13070 while !started.exists() {
13071 tokio::time::sleep(Duration::from_millis(20)).await;
13072 }
13073 }) => result.unwrap(),
13074 }
13075 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13076 assert_eq!(
13077 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13078 -1
13079 );
13080 assert!(
13081 tokio::time::timeout(
13082 Duration::from_millis(100),
13083 crate::repo_lock::SeatLease::for_checkout(
13084 &repo,
13085 true,
13086 crate::config::test_coordination().as_ref()
13087 )
13088 )
13089 .await
13090 .is_err(),
13091 "preparation must also exclude local editors"
13092 );
13093 std::fs::write(proceed, "ready").unwrap();
13094 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13095 .await
13096 .unwrap()
13097 .unwrap();
13098 drop(prep);
13099 assert_eq!(runner.state.prep_complete, Some(true));
13100 assert!(root.join("judge-1").exists());
13101 assert!(root.join("advisor-1").exists());
13102 assert_eq!(
13103 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13104 0
13105 );
13106 }
13107
13108 #[tokio::test]
13109 #[cfg(unix)]
13110 async fn preparation_lock_failure_prevents_repository_changes() {
13111 ask_test_home();
13112 let tmp = tempfile::tempdir().unwrap();
13113 let repo = tmp.path().join("repo");
13114 std::fs::create_dir_all(&repo).unwrap();
13115 init_repo(&repo);
13116 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13117 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13120 let mut runner = runner_at(RunStatus::Prep);
13121 runner.state.repo = repo.clone();
13122 runner.state.config.coordination = crate::config::test_coordination();
13123 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13124 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13125 runner.roles.implementers = vec![spec("implement")];
13126 let error = runner.prep().await.unwrap_err();
13127 assert!(
13128 error
13129 .to_string()
13130 .contains("acquire mandatory repository lock")
13131 );
13132 assert!(!runner.state.worktree_root().exists());
13133 assert!(runner.state.candidates.is_empty());
13134 assert!(!repo.join(".git/config.worktree").exists());
13135 assert_eq!(
13136 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13137 "* main"
13138 );
13139 }
13140
13141 #[tokio::test]
13142 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13143 ask_test_home();
13144 let tmp = tempfile::tempdir().unwrap();
13145 let repo = tmp.path().join("repo");
13146 std::fs::create_dir_all(&repo).unwrap();
13147 init_repo(&repo);
13148 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13149 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13150 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13151 git::git(&repo, &["add", "."]).await.unwrap();
13152 git::git(&repo, &["commit", "-m", "required seed"])
13153 .await
13154 .unwrap();
13155 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13156 git::git(&repo, &["checkout", "main"]).await.unwrap();
13157 let mut runner = runner_at(RunStatus::Prep);
13158 runner.state.repo = repo.clone();
13159 runner.state.base_commit = base.clone();
13160 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13161 runner.state.config.graph.advise = false;
13162 runner.state.config.blind.commit_msg_hook = false;
13163 runner.state.seeds = vec![refs::Seed {
13164 token: seed.clone(),
13165 sha: seed.clone(),
13166 kind: refs::SeedKind::Unmerged,
13167 branch: false,
13168 detail: String::new(),
13169 }];
13170 runner.roles.implementers = vec![spec("one"), spec("two")];
13171 runner.roles.judges = vec![spec("judge")];
13172 let label = blind::assign_labels(2, runner.state.seed)[0];
13173 let branch = runner.state.branch_for(label);
13174 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13175 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13176 .await
13177 .unwrap();
13178 runner.state.candidates.push(Candidate {
13179 index: 0,
13180 label,
13181 agent: "one".into(),
13182 branch,
13183 worktree,
13184 summary: String::new(),
13185 stat: String::new(),
13186 files: 0,
13187 commits: 0,
13188 empty: false,
13189 failed: None,
13190 verified_noop: None,
13191 duration_ms: 0,
13192 folded: false,
13193 });
13194 runner.state.save().unwrap();
13195 runner.state = RunState::load(&runner.state.id).unwrap();
13197 assert_eq!(runner.state.prep_complete, Some(false));
13198 runner.prep().await.unwrap();
13199 assert_eq!(runner.state.prep_complete, Some(true));
13200 assert_eq!(runner.state.candidates.len(), 2);
13201 for candidate in &runner.state.candidates {
13202 assert_eq!(
13203 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13204 git::tree_of(&repo, &seed).await.unwrap()
13205 );
13206 assert_eq!(
13207 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13208 "required commit"
13209 );
13210 }
13211 assert!(runner.state.worktree_root().join("judge-1").exists());
13212 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13213 runner.prep().await.unwrap();
13214 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13215
13216 runner.state.prep_complete = Some(false);
13217 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13218 run: Some(format!(
13219 "test -f '{}'",
13220 tmp.path().join("dependency-ready").display()
13221 )),
13222 ..Default::default()
13223 }];
13224 assert!(runner.prep().await.is_err());
13225 assert!(runner.state.prep_failure.is_some());
13226 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13227 runner.state = RunState::load(&runner.state.id).unwrap();
13228 assert!(
13229 runner.prep().await.is_err(),
13230 "correcting the cause alone must not auto-retry"
13231 );
13232 let failed_at = runner.state.events.last().unwrap().at;
13233 assert!(
13234 runner.retry_preparation(failed_at).is_err(),
13235 "old approvals cannot unlock the hold"
13236 );
13237 let mut task = crate::queue::Task::new(
13238 "retry".into(),
13239 "retry setup".into(),
13240 repo.clone(),
13241 crate::queue::Source::Human,
13242 );
13243 task.start(runner.state.id.clone());
13244 task.hold_machine(Some("setup failed".into()));
13245 task.release_operator("cli", None);
13246 runner
13247 .retry_preparation(task.resumption.as_ref().unwrap().at)
13248 .unwrap();
13249 runner.prep().await.unwrap();
13250 assert!(runner.state.prep_failure.is_none());
13251 assert_eq!(runner.state.prep_complete, Some(true));
13252 assert_eq!(runner.state.candidates.len(), 2);
13253 assert!(runner.state.worktree_root().join("judge-1").exists());
13254 for candidate in &runner.state.candidates {
13255 assert_eq!(
13256 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13257 git::tree_of(&repo, &seed).await.unwrap()
13258 );
13259 }
13260 }
13261
13262 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13263 let mut heads = Vec::new();
13264 for candidate in candidates {
13265 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13266 }
13267 heads
13268 }
13269
13270 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13271 ask_test_home();
13272 let tmp = tempfile::tempdir().unwrap();
13273 let repo = tmp.path().join("repo");
13274 std::fs::create_dir(&repo).unwrap();
13275 init_repo(&repo);
13276 let mut runner = runner_at(RunStatus::Reviewing);
13277 runner.state.repo = repo.clone();
13278 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13279 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13280 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13281 let branch = format!("candidate-{label}");
13282 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13283 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13284 .await
13285 .unwrap();
13286 if label == 'B' {
13287 std::fs::write(worktree.join("change"), "winner").unwrap();
13288 git::rescue_commit(&worktree, "candidate change")
13289 .await
13290 .unwrap();
13291 }
13292 runner.state.candidates.push(Candidate {
13293 index,
13294 label,
13295 agent: "alpha".into(),
13296 branch,
13297 worktree,
13298 summary: "completed reply".into(),
13299 stat: String::new(),
13300 files: usize::from(label == 'B'),
13301 commits: usize::from(label == 'B'),
13302 empty: label == 'A',
13303 failed: None,
13304 verified_noop: None,
13305 duration_ms: 1,
13306 folded: false,
13307 });
13308 }
13309 runner.state.judge_skipped = true;
13310 runner.state.tally = Some(Tally {
13311 first_choice: BTreeMap::new(),
13312 borda: BTreeMap::new(),
13313 winner: 'B',
13314 rankings: 0,
13315 unanimous_initial: false,
13316 deliberated: false,
13317 changed_votes: 0,
13318 unanimous_final: false,
13319 tie_break: None,
13320 judges: 0,
13321 present: 0,
13322 quorum: 0,
13323 met_quorum: true,
13324 uncontested: Some("only B produced a change".into()),
13325 });
13326 runner.state.save().unwrap();
13327 (tmp, runner)
13328 }
13329
13330 #[tokio::test]
13331 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13332 let (_tmp, mut runner) = preservation_resume_fixture().await;
13333 runner.implement().await.unwrap();
13335 assert!(runner.state.candidate_preservation.is_empty());
13336 assert!(
13337 !runner
13338 .state
13339 .dir()
13340 .join("artifacts/candidate-A-recovery")
13341 .exists()
13342 );
13343 runner.fold_losers().await.unwrap();
13344 assert!(!runner.state.candidates[0].worktree.exists());
13345 for schema in [14, crate::run::SCHEMA] {
13346 runner.state.schema = schema;
13347 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13348 runner.state.save().unwrap();
13349 runner.state = RunState::load(&runner.state.id).unwrap();
13350 runner.implement().await.unwrap();
13351 assert!(runner.state.candidate_preservation.is_empty());
13352 assert!(runner.state.candidates[0].folded);
13353 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13354 }
13355 }
13356
13357 #[tokio::test]
13358 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13359 let (_tmp, mut runner) = preservation_resume_fixture().await;
13360 let wt = runner.state.candidates[0].worktree.clone();
13361 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13362 runner.state.schema = 14;
13363 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13364 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13365 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13366 for _ in 0..2 {
13367 assert!(runner.implement().await.is_err());
13368 assert_eq!(runner.state.status, RunStatus::Blocked);
13369 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13370 assert!(
13371 runner.state.candidates[0].empty,
13372 "old verdict must remain unusable"
13373 );
13374 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13375 assert!(runner.state.judge_skipped);
13376 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13377 assert!(
13378 crate::run::read_artifact(&runner.state, "cand-A.patch")
13379 .unwrap()
13380 .contains("owned edits")
13381 );
13382 assert!(wt.exists());
13383 assert_eq!(
13384 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13385 .await
13386 .unwrap(),
13387 1
13388 );
13389 runner.state = RunState::load(&runner.state.id).unwrap();
13390 }
13391 }
13392
13393 #[tokio::test]
13394 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13395 let (_tmp, mut runner) = preservation_resume_fixture().await;
13396 let wt = runner.state.candidates[0].worktree.clone();
13397 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13398 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13399 let index_lock = PathBuf::from(
13400 git::git(
13401 &wt,
13402 &[
13403 "rev-parse",
13404 "--path-format=absolute",
13405 "--git-path",
13406 "index.lock",
13407 ],
13408 )
13409 .await
13410 .unwrap(),
13411 );
13412 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13413 runner.state.schema = 14;
13414 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13415 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13416 assert!(runner.implement().await.is_err());
13417 assert!(runner.after_implement().is_err());
13418 assert!(runner.fold_losers().await.is_err());
13419 assert_eq!(
13420 std::fs::read_to_string(wt.join("recovered")).unwrap(),
13421 "failed seat edits"
13422 );
13423 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13424 runner.state = RunState::load(&runner.state.id).unwrap();
13425 std::fs::remove_file(index_lock).unwrap();
13426 assert!(runner.implement().await.is_err());
13428 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13429 assert!(runner.state.candidates[0].empty);
13430 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13431 assert!(
13432 crate::run::read_artifact(&runner.state, "cand-A.patch")
13433 .unwrap()
13434 .contains("failed seat edits")
13435 );
13436 assert!(wt.exists());
13437 }
13438
13439 #[tokio::test]
13440 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13441 let (_tmp, mut runner) = preservation_resume_fixture().await;
13442 let repo = runner.state.repo.clone();
13443 let common = git::git(
13444 &repo,
13445 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13446 )
13447 .await
13448 .unwrap();
13449 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13450 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13451 assert!(runner.fold_losers().await.is_err());
13452 assert!(!runner.state.candidates[0].worktree.exists());
13453 assert!(!runner.state.candidates[0].folded);
13454 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13455 runner.state = RunState::load(&runner.state.id).unwrap();
13456 std::fs::remove_file(branch_lock).unwrap();
13457 let judge = runner.state.worktree_root().join("judge-1");
13458 git::worktree_add_detached(&repo, &judge, "main")
13459 .await
13460 .unwrap();
13461 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13462 runner.roles.judges = vec![spec("judge")];
13463 assert!(runner.fold_losers().await.is_err());
13464 runner.state = RunState::load(&runner.state.id).unwrap();
13465 assert!(
13466 runner.state.candidates[0].folded,
13467 "candidate progress saved before judge failure"
13468 );
13469 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13470 assert!(judge.join("uncommitted").exists());
13471 std::fs::remove_file(judge.join("uncommitted")).unwrap();
13472 runner.fold_losers().await.unwrap();
13473 assert!(!judge.exists());
13474 runner.fold_losers().await.unwrap();
13475 }
13476
13477 #[tokio::test]
13478 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13479 let (_tmp, mut runner) = preservation_resume_fixture().await;
13480 runner.state.candidates.retain(|c| c.label == 'B');
13481 runner.state.tally = None;
13482 runner.state.judge_skipped = false;
13483 runner.state.prep_complete = Some(true);
13484 runner.state.config.graph.advise = false;
13485 runner.state.config.graph.selected_implementer = Some("alpha".into());
13486 let wt = runner.state.candidates[0].worktree.clone();
13487 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13488 runner
13489 .state
13490 .candidate_preservation
13491 .insert('B', "awaiting persistence".into());
13492 let index_lock = git::git(
13493 &wt,
13494 &[
13495 "rev-parse",
13496 "--path-format=absolute",
13497 "--git-path",
13498 "index.lock",
13499 ],
13500 )
13501 .await
13502 .unwrap();
13503 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13504 runner.state.save().unwrap();
13505 assert!(runner.execute_graph().await.is_err());
13506 assert_eq!(runner.state.status, RunStatus::Blocked);
13507 assert!(runner.state.candidate_preservation.contains_key(&'B'));
13508 std::fs::remove_file(index_lock).unwrap();
13509 for _ in 0..2 {
13512 runner.state = RunState::load(&runner.state.id).unwrap();
13513 runner.pause.park();
13514 runner.execute_graph().await.unwrap();
13515 assert_eq!(runner.state.status, RunStatus::Implementing);
13516 assert!(runner.state.candidate_preservation.is_empty());
13517 assert_eq!(runner.state.candidates[0].commits, 2);
13518 assert!(runner.state.tally.is_none());
13519 }
13520 runner.implement().await.unwrap();
13521 assert_eq!(runner.state.status, RunStatus::Judging);
13522 runner.state.status = RunStatus::Blocked;
13524 runner.state.candidates[0].failed = Some("implementation failed".into());
13525 runner.state.save().unwrap();
13526 let error = runner.execute_graph().await.unwrap_err();
13527 assert!(
13528 error
13529 .to_string()
13530 .contains("selected implementation is held")
13531 );
13532 }
13533
13534 #[tokio::test]
13535 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13536 let (_tmp, mut runner) = preservation_resume_fixture().await;
13537 let repo = runner.state.repo.clone();
13538 let candidate = runner.state.candidates[0].clone();
13539 let registered = git::worktree_holding(&repo, &candidate.branch)
13542 .await
13543 .unwrap()
13544 .unwrap();
13545 assert_eq!(
13546 registered.canonicalize().unwrap(),
13547 candidate.worktree.canonicalize().unwrap()
13548 );
13549 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13550 assert_eq!(
13551 git::worktree_holding(&repo, &candidate.branch)
13552 .await
13553 .unwrap(),
13554 Some(registered)
13555 );
13556 assert!(
13557 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13558 "the stale registration initially keeps the branch checked out"
13559 );
13560 runner.fold_losers().await.unwrap();
13561 assert!(runner.state.candidates[0].folded);
13562 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13563 assert_eq!(
13564 git::worktree_holding(&repo, &candidate.branch)
13565 .await
13566 .unwrap(),
13567 None
13568 );
13569 runner.state = RunState::load(&runner.state.id).unwrap();
13570 runner.fold_losers().await.unwrap();
13571 }
13572
13573 #[tokio::test]
13574 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13575 let (_tmp, mut runner) = preservation_resume_fixture().await;
13576 let repo = runner.state.repo.clone();
13577 let candidate = runner.state.candidates[0].clone();
13578 let winner = runner.state.candidates[1].clone();
13579 let registered = git::worktree_holding(&repo, &candidate.branch)
13580 .await
13581 .unwrap()
13582 .unwrap();
13583 assert_eq!(
13584 registered.canonicalize().unwrap(),
13585 candidate.worktree.canonicalize().unwrap()
13586 );
13587 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13588 assert_eq!(
13589 git::worktree_holding(&repo, &candidate.branch)
13590 .await
13591 .unwrap(),
13592 Some(registered)
13593 );
13594 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13595
13596 for _ in 0..2 {
13597 fold_run(&mut runner.state, false, &crate::run::home())
13598 .await
13599 .unwrap();
13600 runner.state = RunState::load(&runner.state.id).unwrap();
13601 assert!(runner.state.candidates[0].folded);
13602 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13603 assert_eq!(
13604 git::worktree_holding(&repo, &candidate.branch)
13605 .await
13606 .unwrap(),
13607 None
13608 );
13609 assert!(!runner.state.candidates[1].folded);
13610 assert!(winner.worktree.exists());
13611 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13612 }
13613 }
13614
13615 #[tokio::test]
13616 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13617 ask_test_home();
13618 let tmp = tempfile::tempdir().unwrap();
13619 let repo = tmp.path().join("repo");
13620 std::fs::create_dir(&repo).unwrap();
13621 init_repo(&repo);
13622 let mut runner = runner_at(RunStatus::Implementing);
13623 runner.state.repo = repo.clone();
13624 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13625 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13626 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13627 let branch = format!("candidate-{label}");
13628 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13629 git::worktree_add_branch(&repo, &wt, &branch, "main")
13630 .await
13631 .unwrap();
13632 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13633 git::git(&wt, &["add", "README.md"]).await.unwrap();
13634 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13635 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13636 runner.state.candidates.push(Candidate {
13637 index,
13638 label,
13639 agent: "alpha".to_owned(),
13640 branch,
13641 worktree: wt,
13642 summary: "completed implementation".to_owned(),
13643 stat: String::new(),
13644 files: 0,
13645 commits: 0,
13646 empty: false,
13647 failed: None,
13648 verified_noop: None,
13649 duration_ms: 42,
13650 folded: false,
13651 });
13652 runner
13653 .state
13654 .candidate_preservation
13655 .insert(label, "awaiting persistence".to_owned());
13656 }
13657 let wt = runner.state.candidates[0].worktree.clone();
13658 let index_lock = git::git(
13659 &wt,
13660 &[
13661 "rev-parse",
13662 "--path-format=absolute",
13663 "--git-path",
13664 "index.lock",
13665 ],
13666 )
13667 .await
13668 .unwrap();
13669 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13670 runner.state.save().unwrap();
13671 assert!(runner.persist_candidates().await.is_err());
13672 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13673 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13674 assert_eq!(
13675 runner.state.candidates[1].commits, 1,
13676 "other results are still collected"
13677 );
13678 assert!(
13679 !runner.state.candidates[0].empty,
13680 "failed rescue is not no change"
13681 );
13682 assert_eq!(
13683 runner.state.candidates[0].summary,
13684 "completed implementation"
13685 );
13686 assert!(runner.after_implement().is_err());
13687 let home = tmp.path().join("fold-home");
13688 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13689 assert!(wt.join("new-file").exists());
13690 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13691 assert!(recovery.join("staged.patch").exists());
13692 assert!(recovery.join("unstaged.patch").exists());
13693 assert_eq!(
13694 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13695 "untracked edits\n"
13696 );
13697 runner.state.begin_pass();
13698 runner.state.end_pass_parked(ParkCause::Upgrade {
13699 from: "1.0".into(),
13700 to: Some("1.1".into()),
13701 });
13702 runner.state.save().unwrap();
13703 runner.state = RunState::load(&runner.state.id).unwrap();
13704 assert!(matches!(
13705 runner.state.passes.last().unwrap().end,
13706 Some(crate::run::PassEnd::Parked {
13707 cause: ParkCause::Upgrade { .. }
13708 })
13709 ));
13710 std::fs::remove_file(index_lock).unwrap();
13711 runner.implement().await.unwrap();
13713 assert!(runner.state.candidate_preservation.is_empty());
13714 assert!(
13715 runner
13716 .state
13717 .candidates
13718 .iter()
13719 .all(|c| c.commits == 1 && !c.empty)
13720 );
13721 assert!(git::is_clean(&wt).await.unwrap());
13722 assert!(
13723 crate::run::read_artifact(&runner.state, "cand-A.patch")
13724 .unwrap()
13725 .contains("later tracked edits")
13726 );
13727 }
13728
13729 #[tokio::test]
13730 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13731 ask_test_home();
13732 let tmp = tempfile::tempdir().unwrap();
13733 let repo = tmp.path().join("repo");
13734 std::fs::create_dir(&repo).unwrap();
13735 init_repo(&repo);
13736 let wt = tmp.path().join("candidate");
13737 git::worktree_add_branch(&repo, &wt, "candidate", "main")
13738 .await
13739 .unwrap();
13740 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13741 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13742 assert!(wt.join("uncommitted").exists());
13743 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13744 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13745 let mut runner = runner_at(RunStatus::Implementing);
13746 runner.state.repo = repo.clone();
13747 runner.state.base_commit = "missing-base".to_owned();
13748 runner.state.candidates.push(Candidate {
13749 index: 0,
13750 label: 'A',
13751 agent: "alpha".into(),
13752 branch: "candidate".into(),
13753 worktree: wt.clone(),
13754 summary: "completed".into(),
13755 stat: String::new(),
13756 files: 0,
13757 commits: 0,
13758 empty: false,
13759 failed: None,
13760 verified_noop: None,
13761 duration_ms: 1,
13762 folded: false,
13763 });
13764 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13765 assert!(
13766 fold_run(&mut runner.state, true, &tmp.path().join("home"))
13767 .await
13768 .is_err()
13769 );
13770 assert!(!runner.state.candidates[0].folded);
13771 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13772 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13773 runner
13774 .state
13775 .candidate_preservation
13776 .insert('A', "pending".into());
13777 assert!(runner.persist_candidates().await.is_err());
13778 assert!(!runner.state.candidates[0].empty);
13779 assert!(!runner.state.candidates[0].folded);
13780 assert!(wt.exists());
13781 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13782 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13783 std::fs::create_dir_all(&patch_path).unwrap();
13784 assert!(
13785 runner.persist_candidates().await.is_err(),
13786 "artifact failure stays pending"
13787 );
13788 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13789 assert!(!runner.state.candidates[0].empty);
13790 std::fs::remove_dir(patch_path).unwrap();
13791 runner.persist_candidates().await.unwrap();
13792 assert!(
13793 runner.state.candidates[0].empty,
13794 "only successful inspection confirms no change"
13795 );
13796 }
13797
13798 fn init_repo(dir: &Path) {
13799 let run = |args: &[&str]| {
13800 let out = std::process::Command::new("git")
13801 .args(args)
13802 .current_dir(dir)
13803 .quiet()
13804 .output()
13805 .expect("spawn git");
13806 assert!(
13807 out.status.success(),
13808 "git {args:?} failed: {}",
13809 String::from_utf8_lossy(&out.stderr)
13810 );
13811 };
13812 run(&["init", "-b", "main"]);
13813 run(&["config", "user.name", "magi test"]);
13814 run(&["config", "user.email", "magi@example.com"]);
13815 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13816 run(&["add", "-A"]);
13817 run(&["commit", "-m", "init"]);
13818 }
13819
13820 fn ask_test_home() {
13828 crate::run::pin_test_home();
13829 }
13830
13831 fn runner_at(status: RunStatus) -> Runner {
13834 let mut state = RunState::new(
13835 PathBuf::from("/nonexistent/repo"),
13836 "main".to_owned(),
13837 "deadbeef".to_owned(),
13838 "task".to_owned(),
13839 Config::default(),
13840 );
13841 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13844 state.id = format!(
13845 "20000101-000000-{:04x}",
13846 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13847 );
13848 state.status = status;
13849 Runner {
13850 state,
13851 roles: ResolvedRoles {
13852 implementers: Vec::new(),
13853 judges: Vec::new(),
13854 reviewers: Vec::new(),
13855 fixer: None,
13856 conductor: conductor(),
13857 implementer_roster: Vec::new(),
13858 judge_roster: Vec::new(),
13859 reviewer_roster: Vec::new(),
13860 },
13861 sem: Arc::new(Semaphore::new(1)),
13862 pause: Pause::new(),
13863 interrupt: Pause::new(),
13864 }
13865 }
13866
13867 #[test]
13868 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13869 crate::run::pin_test_home();
13870 let mut runner = runner_at(RunStatus::Gating);
13871 runner.state.save().unwrap();
13872 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13875 assert!(runner.claim_driver().is_err());
13876 assert!(runner.state.driver_exited);
13877 assert_eq!(runner.state.driver_pid, None);
13878 let disk = RunState::load(&runner.state.id).unwrap();
13879 assert!(disk.driver_exited);
13880 assert_eq!(disk.driver_pid, None);
13881 assert_eq!(disk.status, RunStatus::Gating);
13882 }
13883
13884 #[tokio::test]
13885 async fn an_early_graph_error_releases_its_driver() {
13886 crate::run::pin_test_home();
13887 let mut runner = runner_at(RunStatus::Blocked);
13888 runner.state.config.graph.selected_implementer = Some("selected".into());
13889 runner.state.save().unwrap();
13890 assert!(runner.execute().await.is_err());
13891 let disk = RunState::load(&runner.state.id).unwrap();
13892 assert!(disk.driver_exited);
13893 assert_eq!(disk.driver_pid, None);
13894 assert_eq!(disk.status, RunStatus::Blocked);
13895 }
13896
13897 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13898 crate::run::MergeRepair {
13899 question: "repair-question".into(),
13900 head: "old-head".into(),
13901 pr: "https://github.com/o/r/pull/42".into(),
13902 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13903 gate: vec![outcome(Some(0), false)],
13904 gate_head: Some("old-head".into()),
13905 gate_fixes: Vec::new(),
13906 pre_gate: Vec::new(),
13907 phase,
13908 failure: None,
13909 }
13910 }
13911
13912 #[tokio::test]
13913 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13914 crate::run::pin_test_home();
13915 let mut runner = runner_at(RunStatus::Landing);
13916 runner.state.merge_repairs.push(merge_repair_request(
13917 crate::run::MergeRepairPhase::Requested,
13918 ));
13919 runner.state.save().unwrap();
13920 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13921 assert!(
13922 runner
13923 .execute()
13924 .await
13925 .unwrap_err()
13926 .to_string()
13927 .contains("already running")
13928 );
13929 assert_eq!(
13930 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13931 crate::run::MergeRepairPhase::Requested
13932 );
13933 drop(claim);
13934 let mut disk = runner.state.clone();
13935 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13936 disk.save().unwrap();
13937 assert!(
13938 runner
13939 .execute()
13940 .await
13941 .unwrap_err()
13942 .to_string()
13943 .contains("decision changed")
13944 );
13945 assert_eq!(
13946 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13947 crate::run::MergeRepairPhase::Complete
13948 );
13949 }
13950
13951 #[test]
13952 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13953 crate::run::pin_test_home();
13954 let mut runner = runner_at(RunStatus::Landing);
13955 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13956 runner.state.reviews = request.reviews.clone();
13957 runner.state.gate = request.gate.clone();
13958 runner.state.gate_ran = true;
13959 runner.state.land_approval = Some(crate::run::LandApproval {
13960 question: "old".into(),
13961 head: "old-head".into(),
13962 });
13963 runner.state.merge_repairs.push(request);
13964 runner.begin_merge_repair_review(0).unwrap();
13965 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13966 assert!(!runner.state.gate_status().ok());
13967 assert!(runner.state.land_approval.is_none());
13968 let disk = RunState::load(&runner.state.id).unwrap();
13969 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13970 assert!(disk.merge_repairs[0].gate[0].ok());
13971 assert_eq!(
13972 disk.merge_repairs[0].phase,
13973 crate::run::MergeRepairPhase::Reviewing
13974 );
13975 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13976 assert!(disk.reviews.is_empty());
13977 }
13978
13979 #[tokio::test]
13980 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13981 crate::run::pin_test_home();
13982 for phase in [
13983 crate::run::MergeRepairPhase::Fixing,
13984 crate::run::MergeRepairPhase::Requested,
13985 ] {
13986 let mut runner = runner_at(RunStatus::Landing);
13987 runner.state.config.graph.land_rounds = 0;
13988 runner.state.merge_repairs.push(merge_repair_request(phase));
13989 runner.state.save().unwrap();
13990 assert!(!runner.repair_merge_review().await.unwrap());
13991 assert_eq!(runner.state.status, RunStatus::Blocked);
13992 assert_eq!(
13993 runner.state.merge_repairs[0].phase,
13994 crate::run::MergeRepairPhase::Failed
13995 );
13996 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
13997 assert!(
13998 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
13999 "interrupted"
14000 } else {
14001 "budget exhausted"
14002 })
14003 );
14004 assert!(runner.state.merge.is_none());
14005 }
14006 }
14007
14008 #[test]
14012 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14013 crate::run::pin_test_home();
14014 let mut runner = runner_at(RunStatus::Implementing);
14015 let interrupt = Pause::new();
14016 runner.watch_interrupt(interrupt.clone());
14017
14018 interrupt.park_because("task a1b2 asked to run first");
14019
14020 assert!(runner.park_here().expect("park_here"));
14021 assert!(runner.state.parked);
14022 let last = runner.state.events.last().expect("a park event");
14023 assert_eq!(last.node, "park");
14024 assert!(
14025 last.message.contains("task a1b2 asked to run first"),
14026 "expected the interrupt reason in {:?}",
14027 last.message
14028 );
14029 }
14030
14031 #[test]
14034 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14035 crate::run::pin_test_home();
14036 let mut runner = runner_at(RunStatus::Implementing);
14037 runner.state.begin_pass();
14038 let shutdown = Pause::new();
14039 runner.on_pause(shutdown.clone());
14040 let upgrade = ParkCause::Upgrade {
14041 from: "0.121.0".to_owned(),
14042 to: Some("0.121.1".to_owned()),
14043 };
14044 shutdown.park_for(upgrade.clone());
14045 shutdown.park_for(ParkCause::Operator);
14046
14047 assert!(runner.park_here().expect("park_here"));
14048 assert_eq!(
14049 runner.state.passes[0].end,
14050 Some(crate::run::PassEnd::Parked { cause: upgrade })
14051 );
14052 let last = runner.state.events.last().expect("a park event");
14053 assert!(last.message.contains("0.121.0"), "{}", last.message);
14054 }
14055
14056 #[test]
14064 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14065 crate::run::pin_test_home();
14066 let mut runner = runner_at(RunStatus::Implementing);
14067 let shutdown = Pause::new();
14068 runner.on_pause(shutdown.clone());
14069 let interrupt = Pause::new();
14070 runner.watch_interrupt(interrupt.clone());
14071
14072 assert!(!runner.park_here().expect("park_here"));
14074 assert!(!runner.state.parked);
14075
14076 interrupt.park_because("test");
14078 assert!(!shutdown.parked());
14079 assert!(runner.park_here().expect("park_here"));
14080 }
14081
14082 #[tokio::test]
14096 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14097 crate::run::pin_test_home();
14098 let mut runner = runner_at(RunStatus::Implementing);
14099 let interrupt = Pause::new();
14100 runner.watch_interrupt(interrupt.clone());
14101
14102 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14103 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14104
14105 let node = async move {
14109 started_tx.send(()).expect("send started");
14110 finish_rx.await.expect("recv finish");
14111 "node finished"
14112 };
14113
14114 let interrupter = async move {
14115 started_rx.await.expect("recv started");
14116 interrupt.park_because("higher-priority task waiting");
14118 tokio::task::yield_now().await;
14122 finish_tx.send(()).expect("send finish");
14123 };
14124
14125 let (node_result, ()) = tokio::join!(node, interrupter);
14126 assert_eq!(
14127 node_result, "node finished",
14128 "the in-flight call ran to completion"
14129 );
14130
14131 assert!(runner.park_here().expect("park_here"));
14134 assert!(runner.state.parked);
14135 }
14136
14137 #[test]
14143 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14144 crate::run::pin_test_home();
14145 let mut runner = runner_at(RunStatus::Judging);
14146 runner.state.config.agents = vec![conductor()];
14150 runner.state.candidates = vec![Candidate {
14151 index: 0,
14152 label: 'A',
14153 agent: "alpha".to_owned(),
14154 branch: "magi/x/A".to_owned(),
14155 worktree: PathBuf::from("/nonexistent/worktree"),
14156 summary: "did the thing".to_owned(),
14157 stat: "1 file changed".to_owned(),
14158 files: 1,
14159 commits: 1,
14160 empty: false,
14161 failed: None,
14162 verified_noop: None,
14163 duration_ms: 1234,
14164 folded: false,
14165 }];
14166 let run_id = runner.state.id.clone();
14167
14168 let interrupt = Pause::new();
14169 runner.watch_interrupt(interrupt.clone());
14170 interrupt.park_because("task c3d4 asked to run first");
14171 assert!(runner.park_here().expect("park_here"));
14172
14173 let resumed = Runner::resume(&run_id).expect("resume");
14174 assert_eq!(resumed.state.candidates.len(), 1);
14175 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14176 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14177 assert_eq!(resumed.state.status, runner.state.status);
14178 assert!(
14179 resumed.state.parked,
14180 "still parked until `execute` actually walks the graph again"
14181 );
14182 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14183 }
14184
14185 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14187 let mut q = ask::Question::new(
14188 run.to_owned(),
14189 "implement".to_owned(),
14190 "impl-A".to_owned(),
14191 "Which storage backend should the cache use?".to_owned(),
14192 String::new(),
14193 vec!["SQLite".to_owned(), "Redis".to_owned()],
14194 );
14195 store.put(&mut q).unwrap();
14196 q
14197 }
14198
14199 #[test]
14200 fn a_failed_runs_open_question_is_abandoned() {
14201 ask_test_home();
14202 let store = ask::Questions::open();
14203 let mut runner = runner_at(RunStatus::Failed);
14204 let run = runner.state.id.clone();
14205 let q = ask_open_question(&store, &run);
14206
14207 runner.settle_questions();
14208
14209 let back = store.get(&q.id).unwrap();
14210 assert!(
14211 !back.status.open(),
14212 "the seat that asked died with the run; nobody is left to read an answer"
14213 );
14214 assert!(
14215 back.detail.contains(&run) && back.detail.contains("failed"),
14216 "the reason names what the run became, not just that it is gone: {}",
14217 back.detail
14218 );
14219 }
14220
14221 #[test]
14222 fn a_merged_runs_open_question_is_abandoned_too() {
14223 ask_test_home();
14224 let store = ask::Questions::open();
14225 for status in [RunStatus::Merged, RunStatus::Ready] {
14228 let mut runner = runner_at(status);
14229 let run = runner.state.id.clone();
14230 let q = ask_open_question(&store, &run);
14231
14232 runner.settle_questions();
14233
14234 let back = store.get(&q.id).unwrap();
14235 assert!(
14236 !back.status.open(),
14237 "{status:?} run's question must not outlive the run"
14238 );
14239 }
14240 }
14241
14242 #[test]
14243 fn a_still_resumable_runs_open_question_is_left_alone() {
14244 ask_test_home();
14245 let store = ask::Questions::open();
14246 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14252 let mut runner = runner_at(status);
14253 let run = runner.state.id.clone();
14254 let q = ask_open_question(&store, &run);
14255
14256 runner.settle_questions();
14257
14258 let back = store.get(&q.id).unwrap();
14259 assert!(
14260 back.status.open(),
14261 "{status:?} is still alive; the question must still be waiting"
14262 );
14263 }
14264 }
14265
14266 #[test]
14267 fn settle_questions_never_touches_an_already_answered_question() {
14268 ask_test_home();
14269 let store = ask::Questions::open();
14270 let mut runner = runner_at(RunStatus::Failed);
14271 let run = runner.state.id.clone();
14272 let mut q = ask_open_question(&store, &run);
14273 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14274 .unwrap();
14275 store.put(&mut q).unwrap();
14276
14277 runner.settle_questions();
14282 runner.settle_questions();
14283
14284 let back = store.get(&q.id).unwrap();
14285 assert_eq!(
14286 back.status,
14287 ask::QuestionStatus::Answered,
14288 "a real answer is a decision on record, never overwritten by a sweep"
14289 );
14290 }
14291
14292 #[tokio::test]
14303 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14304 crate::run::pin_test_home();
14305 let tmp = tempfile::tempdir().expect("tempdir");
14306 let repo = tmp.path().join("repo");
14307 std::fs::create_dir_all(&repo).unwrap();
14308 init_repo(&repo);
14309
14310 let mut config = Config::default();
14311 config.graph.worktree_root = Some(tmp.path().join("wt"));
14312
14313 let mut state = RunState::new(
14314 repo.clone(),
14315 "main".to_owned(),
14316 "deadbeef".to_owned(),
14317 "task".to_owned(),
14318 config,
14319 );
14320 let root = state.worktree_root();
14321 let wt_a = root.join("cand-A");
14322 let wt_b = root.join("cand-B");
14323 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14324 .await
14325 .expect("worktree A");
14326 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14327 .await
14328 .expect("worktree B");
14329
14330 state.candidates = vec![
14331 Candidate {
14332 index: 0,
14333 label: 'A',
14334 agent: "alpha".to_owned(),
14335 branch: "magi/x/A".to_owned(),
14336 worktree: wt_a.clone(),
14337 summary: String::new(),
14338 stat: String::new(),
14339 files: 0,
14340 commits: 0,
14341 empty: false,
14342 failed: None,
14343 verified_noop: None,
14344 duration_ms: 0,
14345 folded: false,
14346 },
14347 Candidate {
14348 index: 1,
14349 label: 'B',
14350 agent: "beta".to_owned(),
14351 branch: "magi/x/B".to_owned(),
14352 worktree: wt_b.clone(),
14353 summary: String::new(),
14354 stat: String::new(),
14355 files: 0,
14356 commits: 0,
14357 empty: false,
14358 failed: None,
14359 verified_noop: None,
14360 duration_ms: 0,
14361 folded: false,
14362 },
14363 ];
14364 state.tally = Some(Tally {
14365 first_choice: BTreeMap::from([('A', 1)]),
14366 borda: BTreeMap::new(),
14367 winner: 'A',
14368 rankings: 1,
14369 unanimous_initial: true,
14370 deliberated: false,
14371 changed_votes: 0,
14372 unanimous_final: true,
14373 tie_break: None,
14374 judges: 1,
14375 present: 1,
14376 quorum: 1,
14377 met_quorum: true,
14378 uncontested: None,
14379 });
14380 state.status = RunStatus::Ready;
14381
14382 fold_run(&mut state, false, &crate::run::home())
14383 .await
14384 .expect("fold_run");
14385
14386 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14387 assert!(
14388 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14389 "the unmerged winner's branch survives"
14390 );
14391 assert!(
14392 !state.candidates[0].folded,
14393 "the winner is not marked folded"
14394 );
14395
14396 assert!(!wt_b.exists(), "the loser's worktree is removed");
14397 assert!(
14398 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14399 "the loser's branch is removed"
14400 );
14401 assert!(state.candidates[1].folded, "the loser is marked folded");
14402 }
14403
14404 #[tokio::test]
14405 #[cfg(unix)]
14406 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14407 use std::os::fd::AsRawFd as _;
14408 use std::time::Duration;
14409
14410 let tmp = tempfile::tempdir().unwrap();
14411 let repo = tmp.path().join("repo");
14412 std::fs::create_dir_all(&repo).unwrap();
14413 init_repo(&repo);
14414 let common = repo.join(".git");
14415 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14416 let home = tmp.path().join("home");
14417 let mut config = Config::default();
14418 config.coordination = crate::config::test_coordination();
14419 config.graph.worktree_root = Some(tmp.path().join("wt"));
14420 config.graph.timeout_resource_wait = 1;
14421 let mut state = RunState::new(
14422 repo.clone(),
14423 "main".into(),
14424 "deadbeef".into(),
14425 "task".into(),
14426 config,
14427 );
14428 let root = state.worktree_root();
14429 let candidate = root.join("cand-A");
14430 let reviewer = root.join("review-1");
14431 let branch = state.branch_for('A');
14432 git::worktree_add_branch(&repo, &candidate, &branch, "main")
14433 .await
14434 .unwrap();
14435 git::worktree_add_detached(&repo, &reviewer, "main")
14436 .await
14437 .unwrap();
14438 git::acquire_worktree_config(&repo).await.unwrap();
14439 state.enabled_worktree_config = true;
14440 state.status = RunStatus::Ready;
14441 state.candidates.push(Candidate {
14442 index: 0,
14443 label: 'A',
14444 agent: "implement".into(),
14445 branch: branch.clone(),
14446 worktree: candidate.clone(),
14447 summary: String::new(),
14448 stat: String::new(),
14449 files: 0,
14450 commits: 0,
14451 empty: false,
14452 failed: None,
14453 verified_noop: None,
14454 duration_ms: 0,
14455 folded: false,
14456 });
14457 state.save_under(&home).unwrap();
14458 let config_before = std::fs::read(common.join("config")).unwrap();
14459 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14460 .await
14461 .unwrap();
14462 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14465 assert_eq!(
14466 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14467 0
14468 );
14469 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14470 assert!(
14471 tokio::time::timeout(Duration::from_millis(200), &mut fold)
14472 .await
14473 .is_err()
14474 );
14475 assert!(candidate.exists());
14476 assert!(reviewer.exists());
14477 assert!(git::branch_exists(&repo, &branch).await.unwrap());
14478 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14479 assert_eq!(
14480 git::git(&repo, &["worktree", "list", "--porcelain"])
14481 .await
14482 .unwrap(),
14483 worktrees_before
14484 );
14485 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14486 .await
14487 .unwrap()
14488 .unwrap_err();
14489 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14490 drop(fold);
14491 assert!(candidate.exists() && reviewer.exists());
14492 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14493 assert_eq!(
14494 state.resource_block.as_ref().unwrap().kind,
14495 "repository coordination"
14496 );
14497 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14498 drop(external);
14499 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14500 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14501 .await
14502 .unwrap()
14503 .unwrap();
14504 drop(fold);
14505 assert!(removed.contains(&branch));
14506 assert!(!root.exists());
14507 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14508 assert!(state.candidates[0].folded);
14509 assert!(!state.enabled_worktree_config);
14510 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14511 let enabled = git::git_raw(
14512 &repo,
14513 &["config", "--local", "--get", "extensions.worktreeConfig"],
14514 )
14515 .await
14516 .unwrap();
14517 assert!(!enabled.ok(), "the last config reference was released");
14518 }
14519
14520 #[tokio::test]
14523 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14524 let tmp = tempfile::tempdir().expect("tempdir");
14525 let repo = tmp.path().join("repo");
14526 std::fs::create_dir_all(&repo).unwrap();
14527 init_repo(&repo);
14528 let home = tmp.path().join("home");
14529
14530 let mut config = Config::default();
14531 config.graph.worktree_root = Some(tmp.path().join("wt"));
14532 let mut state = RunState::new(
14533 repo.clone(),
14534 "main".to_owned(),
14535 "deadbeef".to_owned(),
14536 "task".to_owned(),
14537 config,
14538 );
14539 git::git(&repo, &["branch", "magi/x/A", "main"])
14541 .await
14542 .expect("branch");
14543 state.candidates = vec![Candidate {
14544 index: 0,
14545 label: 'A',
14546 agent: "alpha".to_owned(),
14547 branch: "magi/x/A".to_owned(),
14548 worktree: state.worktree_root().join("cand-A"),
14549 summary: String::new(),
14550 stat: String::new(),
14551 files: 0,
14552 commits: 0,
14553 empty: false,
14554 failed: None,
14555 verified_noop: None,
14556 duration_ms: 0,
14557 folded: true,
14558 }];
14559 state.released_to = Some("20260901-000000-new1".to_owned());
14560 state.released_branches = vec!["magi/x/A".to_owned()];
14561
14562 fold_run(&mut state, true, &home).await.expect("fold_run");
14563
14564 assert!(
14565 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14566 "the handed-over branch survives a fold"
14567 );
14568 }
14569
14570 #[tokio::test]
14574 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14575 let tmp = tempfile::tempdir().expect("tempdir");
14576 let repo = tmp.path().join("repo");
14577 std::fs::create_dir_all(&repo).unwrap();
14578 init_repo(&repo);
14579 let run = |args: &[&str]| {
14580 let out = std::process::Command::new("git")
14581 .quiet()
14582 .args(args)
14583 .current_dir(&repo)
14584 .output()
14585 .expect("spawn git");
14586 assert!(out.status.success(), "git {args:?}");
14587 };
14588 run(&["branch", "magi/x/A"]);
14589 run(&["checkout", "-q", "-b", "magi/x/B"]);
14590 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14591 run(&["add", "-A"]);
14592 run(&["commit", "-q", "-m", "work"]);
14593 run(&["checkout", "-q", "main"]);
14594
14595 let probe = RunState::new(
14599 repo.clone(),
14600 "main".to_owned(),
14601 "deadbeef".to_owned(),
14602 "task".to_owned(),
14603 Config::default(),
14604 );
14605 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14606 let bare = tmp.path().join("origin.git");
14607 let out = std::process::Command::new("git")
14608 .quiet()
14609 .args(["init", "-q", "--bare"])
14610 .arg(&bare)
14611 .output()
14612 .expect("spawn git");
14613 assert!(out.status.success(), "git init --bare");
14614 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14615 run(&["push", "-q", "origin", "main"]);
14616
14617 let mut state = RunState::new(
14618 repo.clone(),
14619 "main".to_owned(),
14620 "deadbeef".to_owned(),
14621 "task".to_owned(),
14622 Config::default(),
14623 );
14624 state.seeds = vec![refs::Seed {
14625 token: "magi/27b2/A".to_owned(),
14626 kind: refs::SeedKind::Unresolved,
14627 sha: String::new(),
14628 branch: true,
14629 detail: "no branch or commit named magi/27b2/A".to_owned(),
14630 }];
14631
14632 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14633 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14634 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14635 let detail = empty_candidate_detail(&state, "main");
14636 assert!(detail.starts_with("empty candidate"), "{detail}");
14637 assert!(detail.contains("magi/27b2/A"), "{detail}");
14638 }
14639
14640 #[tokio::test]
14649 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14650 let tmp = tempfile::tempdir().expect("tempdir");
14651 let repo = tmp.path().join("repo");
14652 std::fs::create_dir_all(&repo).unwrap();
14653 init_repo(&repo);
14654
14655 let mut config = Config::default();
14656 config.merge.mode = MergeMode::Local;
14657
14658 let mut state = RunState::new(
14659 repo.clone(),
14660 "main".to_owned(),
14661 "deadbeef".to_owned(),
14662 "task".to_owned(),
14663 config,
14664 );
14665 state.candidates = vec![Candidate {
14666 index: 0,
14667 label: 'A',
14668 agent: "alpha".to_owned(),
14669 branch: "does-not-exist".to_owned(),
14670 worktree: repo.clone(),
14671 summary: String::new(),
14672 stat: String::new(),
14673 files: 0,
14674 commits: 0,
14675 empty: false,
14676 failed: None,
14677 verified_noop: None,
14678 duration_ms: 0,
14679 folded: false,
14680 }];
14681 state.tally = Some(Tally {
14682 first_choice: BTreeMap::from([('A', 1)]),
14683 borda: BTreeMap::new(),
14684 winner: 'A',
14685 rankings: 1,
14686 unanimous_initial: true,
14687 deliberated: false,
14688 changed_votes: 0,
14689 unanimous_final: true,
14690 tie_break: None,
14691 judges: 0,
14692 present: 0,
14693 quorum: 0,
14694 met_quorum: true,
14695 uncontested: Some("only candidate A produced a change".to_owned()),
14696 });
14697 state.reviews = vec![ReviewRound {
14698 round: 1,
14699 head: "deadbeef".to_owned(),
14700 verified_head: None,
14701 verified_at: None,
14702 reviews: Vec::new(),
14703 e2e: Vec::new(),
14704 fix: None,
14705 blocking: 0,
14706 answered: 0,
14707 expected: 0,
14708 clean: true,
14709 verify_retried: false,
14710 e2e_deferred: false,
14711 e2e_defer_reason: None,
14712 progressed: false,
14713 vote_split: false,
14714 reconsideration: Vec::new(),
14715 verdict: None,
14716 }];
14717 state.gate = vec![CommandOutcome {
14718 command: "test".to_owned(),
14719 code: Some(0),
14720 output_tail: String::new(),
14721 duration_ms: 0,
14722 resource_blocked: false,
14723 }];
14724 state.gate_ran = true;
14725 state.status = RunStatus::Ready;
14730 state.merge = Some(MergeOutcome {
14731 mode: MergeMode::Local,
14732 ok: false,
14733 detail: "already concluded".to_owned(),
14734 empty: false,
14735 });
14736
14737 let mut runner = Runner {
14738 state,
14739 roles: ResolvedRoles {
14740 implementers: Vec::new(),
14741 judges: Vec::new(),
14742 reviewers: Vec::new(),
14743 fixer: None,
14744 conductor: conductor(),
14745 implementer_roster: Vec::new(),
14746 judge_roster: Vec::new(),
14747 reviewer_roster: Vec::new(),
14748 },
14749 sem: Arc::new(Semaphore::new(1)),
14750 pause: Pause::new(),
14751 interrupt: Pause::new(),
14752 };
14753
14754 runner.merge().await.expect("merge");
14755
14756 assert_eq!(
14757 runner.state.status,
14758 RunStatus::Ready,
14759 "a concluded run's status must not change on reentry"
14760 );
14761 assert_eq!(
14762 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14763 Some("already concluded"),
14764 "merge must not run again once the node already recorded an outcome"
14765 );
14766 }
14767
14768 #[tokio::test]
14777 async fn merge_refuses_a_gate_that_has_not_actually_run() {
14778 let tmp = tempfile::tempdir().expect("tempdir");
14779 let repo = tmp.path().join("repo");
14780 std::fs::create_dir_all(&repo).unwrap();
14781 init_repo(&repo);
14782
14783 let mut config = Config::default();
14784 config.merge.mode = MergeMode::Local;
14785
14786 let mut state = RunState::new(
14787 repo.clone(),
14788 "main".to_owned(),
14789 "deadbeef".to_owned(),
14790 "task".to_owned(),
14791 config,
14792 );
14793 state.candidates = vec![Candidate {
14794 index: 0,
14795 label: 'A',
14796 agent: "alpha".to_owned(),
14797 branch: "does-not-exist".to_owned(),
14798 worktree: repo.clone(),
14799 summary: String::new(),
14800 stat: String::new(),
14801 files: 0,
14802 commits: 0,
14803 empty: false,
14804 failed: None,
14805 verified_noop: None,
14806 duration_ms: 0,
14807 folded: false,
14808 }];
14809 state.tally = Some(Tally {
14810 first_choice: BTreeMap::from([('A', 1)]),
14811 borda: BTreeMap::new(),
14812 winner: 'A',
14813 rankings: 1,
14814 unanimous_initial: true,
14815 deliberated: false,
14816 changed_votes: 0,
14817 unanimous_final: true,
14818 tie_break: None,
14819 judges: 0,
14820 present: 0,
14821 quorum: 0,
14822 met_quorum: true,
14823 uncontested: Some("only candidate A produced a change".to_owned()),
14824 });
14825 state.reviews = vec![ReviewRound {
14826 round: 1,
14827 head: "deadbeef".to_owned(),
14828 verified_head: None,
14829 verified_at: None,
14830 reviews: Vec::new(),
14831 e2e: Vec::new(),
14832 fix: None,
14833 blocking: 0,
14834 answered: 0,
14835 expected: 0,
14836 clean: true,
14837 verify_retried: false,
14838 e2e_deferred: false,
14839 e2e_defer_reason: None,
14840 progressed: false,
14841 vote_split: false,
14842 reconsideration: Vec::new(),
14843 verdict: None,
14844 }];
14845 state.gate = Vec::new();
14847 state.gate_ran = false;
14848 state.status = RunStatus::Gating;
14849
14850 let mut runner = Runner {
14851 state,
14852 roles: ResolvedRoles {
14853 implementers: Vec::new(),
14854 judges: Vec::new(),
14855 reviewers: Vec::new(),
14856 fixer: None,
14857 conductor: conductor(),
14858 implementer_roster: Vec::new(),
14859 judge_roster: Vec::new(),
14860 reviewer_roster: Vec::new(),
14861 },
14862 sem: Arc::new(Semaphore::new(1)),
14863 pause: Pause::new(),
14864 interrupt: Pause::new(),
14865 };
14866
14867 runner.merge().await.expect("merge");
14868
14869 assert!(
14870 runner.state.merge.is_none(),
14871 "an empty gate must never be read as a passing one: {:?}",
14872 runner.state.merge
14873 );
14874 }
14875
14876 #[tokio::test]
14883 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14884 let tmp = tempfile::tempdir().expect("tempdir");
14885 let repo = tmp.path().join("repo");
14886 std::fs::create_dir_all(&repo).unwrap();
14887 init_repo(&repo);
14888
14889 let config = Config::default();
14891
14892 let mut state = RunState::new(
14893 repo.clone(),
14894 "main".to_owned(),
14895 "deadbeef".to_owned(),
14896 "task".to_owned(),
14897 config,
14898 );
14899 state.candidates = vec![Candidate {
14900 index: 0,
14901 label: 'A',
14902 agent: "alpha".to_owned(),
14903 branch: "does-not-exist".to_owned(),
14904 worktree: repo.clone(),
14905 summary: String::new(),
14906 stat: String::new(),
14907 files: 0,
14908 commits: 0,
14909 empty: false,
14910 failed: None,
14911 verified_noop: None,
14912 duration_ms: 0,
14913 folded: false,
14914 }];
14915 state.tally = Some(Tally {
14916 first_choice: BTreeMap::from([('A', 1)]),
14917 borda: BTreeMap::new(),
14918 winner: 'A',
14919 rankings: 1,
14920 unanimous_initial: true,
14921 deliberated: false,
14922 changed_votes: 0,
14923 unanimous_final: true,
14924 tie_break: None,
14925 judges: 0,
14926 present: 0,
14927 quorum: 0,
14928 met_quorum: true,
14929 uncontested: Some("only candidate A produced a change".to_owned()),
14930 });
14931 state.reviews = vec![ReviewRound {
14932 round: 1,
14933 head: "deadbeef".to_owned(),
14934 verified_head: None,
14935 verified_at: None,
14936 reviews: Vec::new(),
14937 e2e: Vec::new(),
14938 fix: None,
14939 blocking: 0,
14940 answered: 0,
14941 expected: 0,
14942 clean: true,
14943 verify_retried: false,
14944 e2e_deferred: false,
14945 e2e_defer_reason: None,
14946 progressed: false,
14947 vote_split: false,
14948 reconsideration: Vec::new(),
14949 verdict: None,
14950 }];
14951
14952 let mut runner = Runner {
14953 state,
14954 roles: ResolvedRoles {
14955 implementers: Vec::new(),
14956 judges: Vec::new(),
14957 reviewers: Vec::new(),
14958 fixer: None,
14959 conductor: conductor(),
14960 implementer_roster: Vec::new(),
14961 judge_roster: Vec::new(),
14962 reviewer_roster: Vec::new(),
14963 },
14964 sem: Arc::new(Semaphore::new(1)),
14965 pause: Pause::new(),
14966 interrupt: Pause::new(),
14967 };
14968
14969 runner.gate().await.expect("gate");
14970 assert!(
14971 runner.state.gate_ran,
14972 "zero configured commands is still a real attempt, not an unrun gate"
14973 );
14974 assert!(runner.state.gate.is_empty());
14975 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
14976 assert_ne!(
14977 runner.state.status,
14978 RunStatus::Blocked,
14979 "a gate with nothing to check must not read as failed"
14980 );
14981
14982 runner.merge().await.expect("merge");
14983 assert_eq!(
14984 runner.state.status,
14985 RunStatus::Ready,
14986 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14987 );
14988 }
14989
14990 #[tokio::test]
15001 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15002 crate::run::pin_test_home();
15003 let home = crate::run::home();
15004
15005 let tmp = tempfile::tempdir().expect("tempdir");
15006 let repo = tmp.path().join("repo");
15007 std::fs::create_dir_all(&repo).unwrap();
15008 init_repo(&repo);
15009 let cache_dir = tmp.path().join("target");
15012
15013 let mut config = Config::default();
15014 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15015 config.graph.timeout_verify = Some(2);
15018 config.graph.timeout_resource_wait = 2;
15019
15020 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15021 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15022 .expect("no io error acquiring directly")
15023 {
15024 crate::cache::AcquireOutcome::Acquired(g) => g,
15025 crate::cache::AcquireOutcome::Busy(b) => {
15026 panic!("expected the direct acquire to win the lease first: {b:?}")
15027 }
15028 };
15029
15030 let mut state = RunState::new(
15031 repo.clone(),
15032 "main".to_owned(),
15033 "deadbeef".to_owned(),
15034 "task".to_owned(),
15035 config,
15036 );
15037 state.candidates = vec![Candidate {
15038 index: 0,
15039 label: 'A',
15040 agent: "alpha".to_owned(),
15041 branch: "does-not-exist".to_owned(),
15042 worktree: repo.clone(),
15043 summary: String::new(),
15044 stat: String::new(),
15045 files: 0,
15046 commits: 0,
15047 empty: false,
15048 failed: None,
15049 verified_noop: None,
15050 duration_ms: 0,
15051 folded: false,
15052 }];
15053 state.tally = Some(Tally {
15054 first_choice: BTreeMap::from([('A', 1)]),
15055 borda: BTreeMap::new(),
15056 winner: 'A',
15057 rankings: 1,
15058 unanimous_initial: true,
15059 deliberated: false,
15060 changed_votes: 0,
15061 unanimous_final: true,
15062 tie_break: None,
15063 judges: 0,
15064 present: 0,
15065 quorum: 0,
15066 met_quorum: true,
15067 uncontested: Some("only candidate A produced a change".to_owned()),
15068 });
15069 state.reviews = vec![ReviewRound {
15070 round: 1,
15071 head: "deadbeef".to_owned(),
15072 verified_head: None,
15073 verified_at: None,
15074 reviews: Vec::new(),
15075 e2e: Vec::new(),
15076 fix: None,
15077 blocking: 0,
15078 answered: 0,
15079 expected: 0,
15080 clean: true,
15081 verify_retried: false,
15082 e2e_deferred: false,
15083 e2e_defer_reason: None,
15084 progressed: false,
15085 vote_split: false,
15086 reconsideration: Vec::new(),
15087 verdict: None,
15088 }];
15089
15090 let mut runner = Runner {
15091 state,
15092 roles: ResolvedRoles {
15093 implementers: Vec::new(),
15094 judges: Vec::new(),
15095 reviewers: Vec::new(),
15096 fixer: None,
15097 conductor: conductor(),
15098 implementer_roster: Vec::new(),
15099 judge_roster: Vec::new(),
15100 reviewer_roster: Vec::new(),
15101 },
15102 sem: Arc::new(Semaphore::new(1)),
15103 pause: Pause::new(),
15104 interrupt: Pause::new(),
15105 };
15106
15107 let started = std::time::Instant::now();
15108 runner.gate().await.expect("gate");
15109 assert!(
15110 started.elapsed() < Duration::from_secs(1),
15111 "a gate with nothing to run must never wait on a lease it never needed"
15112 );
15113 assert!(
15114 runner.state.gate_ran,
15115 "zero commands is still a real, immediate attempt"
15116 );
15117 assert!(runner.state.gate.is_empty());
15118 assert_ne!(
15119 runner.state.status,
15120 RunStatus::Blocked,
15121 "must not read as resource-blocked on a lease it never asked for"
15122 );
15123 }
15124
15125 #[tokio::test]
15135 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15136 crate::run::pin_test_home();
15137
15138 let tmp = tempfile::tempdir().expect("tempdir");
15139 let repo = tmp.path().join("repo");
15140 std::fs::create_dir_all(&repo).unwrap();
15141 init_repo(&repo);
15142
15143 let mut config = Config::default();
15144 config.verify.gate = vec![
15145 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15146 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15147 .to_owned(),
15148 ];
15149
15150 let mut state = RunState::new(
15151 repo.clone(),
15152 "main".to_owned(),
15153 "deadbeef".to_owned(),
15154 "task".to_owned(),
15155 config,
15156 );
15157 let run_id = state.id.clone();
15158 state.candidates = vec![Candidate {
15159 index: 0,
15160 label: 'A',
15161 agent: "alpha".to_owned(),
15162 branch: "does-not-exist".to_owned(),
15163 worktree: repo.clone(),
15164 summary: String::new(),
15165 stat: String::new(),
15166 files: 0,
15167 commits: 0,
15168 empty: false,
15169 failed: None,
15170 verified_noop: None,
15171 duration_ms: 0,
15172 folded: false,
15173 }];
15174 state.tally = Some(Tally {
15175 first_choice: BTreeMap::from([('A', 1)]),
15176 borda: BTreeMap::new(),
15177 winner: 'A',
15178 rankings: 1,
15179 unanimous_initial: true,
15180 deliberated: false,
15181 changed_votes: 0,
15182 unanimous_final: true,
15183 tie_break: None,
15184 judges: 0,
15185 present: 0,
15186 quorum: 0,
15187 met_quorum: true,
15188 uncontested: Some("only candidate A produced a change".to_owned()),
15189 });
15190 state.reviews = vec![ReviewRound {
15191 round: 1,
15192 head: "deadbeef".to_owned(),
15193 verified_head: None,
15194 verified_at: None,
15195 reviews: Vec::new(),
15196 e2e: Vec::new(),
15197 fix: None,
15198 blocking: 0,
15199 answered: 0,
15200 expected: 0,
15201 clean: true,
15202 verify_retried: false,
15203 e2e_deferred: false,
15204 e2e_defer_reason: None,
15205 progressed: false,
15206 vote_split: false,
15207 reconsideration: Vec::new(),
15208 verdict: None,
15209 }];
15210
15211 let mut runner = Runner {
15212 state,
15213 roles: ResolvedRoles {
15214 implementers: Vec::new(),
15215 judges: Vec::new(),
15216 reviewers: Vec::new(),
15217 fixer: None,
15218 conductor: conductor(),
15219 implementer_roster: Vec::new(),
15220 judge_roster: Vec::new(),
15221 reviewer_roster: Vec::new(),
15222 },
15223 sem: Arc::new(Semaphore::new(1)),
15224 pause: Pause::new(),
15225 interrupt: Pause::new(),
15226 };
15227
15228 let started_marker = repo.join("started.marker");
15229 let release_marker = repo.join("release.marker");
15230 let poller = tokio::spawn(async move {
15231 for _ in 0..100 {
15236 if started_marker.exists()
15237 && let Ok(s) = crate::run::RunState::load(&run_id)
15238 && let Some(a) = s.active.get("gate")
15239 {
15240 std::fs::write(&release_marker, b"go").expect("release marker");
15241 return Some(a.clone());
15242 }
15243 tokio::time::sleep(Duration::from_millis(50)).await;
15244 }
15245 None
15246 });
15247
15248 runner.gate().await.expect("gate");
15249 let captured = poller.await.expect("poller task");
15250 let captured = captured.expect(
15251 "the poller never saw a `gate` task entry in run.json while the command was \
15252 still blocked on its own release marker",
15253 );
15254
15255 assert_eq!(captured.task.as_deref(), Some("gate"));
15256 assert_eq!(captured.node, "gate");
15257 assert_eq!(captured.index, Some(1));
15258 assert_eq!(captured.total, Some(1));
15259 assert!(
15260 captured
15261 .command
15262 .as_deref()
15263 .is_some_and(|c| c.contains("started.marker")),
15264 "{captured:?}"
15265 );
15266
15267 assert!(
15268 runner.state.active.is_empty(),
15269 "the entry must be cleared once the command actually finished: {:?}",
15270 runner.state.active
15271 );
15272 assert!(runner.state.gate_ran);
15273 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15274 }
15275
15276 #[tokio::test]
15279 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15280 use crate::verdict::{Finding, ReviewVote, Severity};
15281 crate::run::pin_test_home();
15282 let tmp = tempfile::tempdir().expect("tempdir");
15283 let repo = tmp.path().join("repo");
15284 std::fs::create_dir_all(&repo).unwrap();
15285 init_repo(&repo);
15286
15287 for (severity, vote, expect) in [
15288 (Severity::Major, ReviewVote::Reject, true),
15289 (Severity::Minor, ReviewVote::Reject, false),
15290 (Severity::Major, ReviewVote::Approve, false),
15291 ] {
15292 let mut round = review_round(false, 1, 1, 1, false, true);
15293 round.reviews = vec![ReviewRecord {
15294 reviewer: 1,
15295 agent: "alpha".to_owned(),
15296 summary: String::new(),
15297 findings: vec![Finding {
15298 id: "R1-1-1".to_owned(),
15299 severity,
15300 file: None,
15301 line: None,
15302 title: "t".to_owned(),
15303 detail: String::new(),
15304 }],
15305 vote: Some(vote),
15306 failed: None,
15307 duration_ms: 0,
15308 attempts: 0,
15309 }];
15310 let mut state = RunState::new(
15311 repo.clone(),
15312 "main".to_owned(),
15313 "deadbeef".to_owned(),
15314 "task".to_owned(),
15315 Config::default(),
15316 );
15317 state.reviews = vec![round];
15318 let mut runner = Runner {
15319 state,
15320 roles: ResolvedRoles {
15321 implementers: Vec::new(),
15322 judges: Vec::new(),
15323 reviewers: Vec::new(),
15324 fixer: None,
15325 conductor: conductor(),
15326 implementer_roster: Vec::new(),
15327 judge_roster: Vec::new(),
15328 reviewer_roster: Vec::new(),
15329 },
15330 sem: Arc::new(Semaphore::new(1)),
15331 pause: Pause::new(),
15332 interrupt: Pause::new(),
15333 };
15334 let shell = runner.state.config.shell();
15335 runner
15336 .stop_reviewing("round budget spent", &shell, &repo)
15337 .await
15338 .expect("stop_reviewing");
15339 assert_eq!(runner.state.status, RunStatus::Gating);
15340 assert_eq!(
15341 runner.state.contested_handoff.is_some(),
15342 expect,
15343 "{severity:?} + {vote:?}"
15344 );
15345 }
15346 }
15347
15348 #[tokio::test]
15361 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15362 crate::run::pin_test_home();
15363 let home = crate::run::home();
15364
15365 let tmp = tempfile::tempdir().expect("tempdir");
15366 let repo = tmp.path().join("repo");
15367 std::fs::create_dir_all(&repo).unwrap();
15368 init_repo(&repo);
15369 let head = crate::git::rev_parse(&repo, "HEAD")
15370 .await
15371 .expect("rev-parse");
15372 let cache_dir = tmp.path().join("target");
15375
15376 let mut config = Config::default();
15377 config.verify.e2e = vec![format!(
15378 "CARGO_TARGET_DIR='{}' test -f README.md",
15379 cache_dir.display()
15380 )];
15381 config.graph.review_rounds = 1;
15382 config.graph.timeout_verify = Some(2);
15385 config.graph.timeout_resource_wait = 2;
15386
15387 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15388 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15389 .expect("no io error acquiring directly")
15390 {
15391 crate::cache::AcquireOutcome::Acquired(g) => g,
15392 crate::cache::AcquireOutcome::Busy(b) => {
15393 panic!("expected the direct acquire to win the lease first: {b:?}")
15394 }
15395 };
15396
15397 let mut state = RunState::new(
15398 repo.clone(),
15399 "main".to_owned(),
15400 head.clone(),
15401 "task".to_owned(),
15402 config,
15403 );
15404 state.candidates = vec![Candidate {
15405 index: 0,
15406 label: 'A',
15407 agent: "alpha".to_owned(),
15408 branch: "does-not-exist".to_owned(),
15409 worktree: repo.clone(),
15410 summary: String::new(),
15411 stat: String::new(),
15412 files: 0,
15413 commits: 0,
15414 empty: false,
15415 failed: None,
15416 verified_noop: None,
15417 duration_ms: 0,
15418 folded: false,
15419 }];
15420 state.tally = Some(Tally {
15421 first_choice: BTreeMap::from([('A', 1)]),
15422 borda: BTreeMap::new(),
15423 winner: 'A',
15424 rankings: 1,
15425 unanimous_initial: true,
15426 deliberated: false,
15427 changed_votes: 0,
15428 unanimous_final: true,
15429 tie_break: None,
15430 judges: 0,
15431 present: 0,
15432 quorum: 0,
15433 met_quorum: true,
15434 uncontested: Some("only candidate A produced a change".to_owned()),
15435 });
15436 state.reviews = vec![ReviewRound {
15440 round: 1,
15441 head: head.clone(),
15442 verified_head: None,
15443 verified_at: None,
15444 reviews: Vec::new(),
15445 e2e: Vec::new(),
15446 fix: None,
15447 blocking: 1,
15448 answered: 1,
15449 expected: 1,
15450 clean: false,
15451 verify_retried: false,
15452 e2e_deferred: true,
15453 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15454 progressed: false,
15455 vote_split: false,
15456 reconsideration: Vec::new(),
15457 verdict: None,
15458 }];
15459
15460 let mut runner = Runner {
15461 state,
15462 roles: ResolvedRoles {
15463 implementers: Vec::new(),
15464 judges: Vec::new(),
15465 reviewers: Vec::new(),
15466 fixer: None,
15467 conductor: conductor(),
15468 implementer_roster: Vec::new(),
15469 judge_roster: Vec::new(),
15470 reviewer_roster: Vec::new(),
15471 },
15472 sem: Arc::new(Semaphore::new(1)),
15473 pause: Pause::new(),
15474 interrupt: Pause::new(),
15475 };
15476
15477 let shell = runner.state.config.shell();
15478 runner
15479 .stop_reviewing("round budget spent", &shell, &repo)
15480 .await
15481 .expect("stop_reviewing");
15482
15483 let last = runner.state.reviews.last().expect("round record");
15484 assert_eq!(
15485 last.e2e_status(),
15486 E2eStatus::ResourceBlocked,
15487 "the shared cache is still held; the attempt must read as blocked, not deferred or \
15488 failed: {last:?}"
15489 );
15490 assert_eq!(
15491 last.verified_head.as_deref(),
15492 Some(head.as_str()),
15493 "which commit this attempt targeted is known even though nothing finished checking \
15494 it"
15495 );
15496 let first_attempt_at = last
15497 .verified_at
15498 .expect("when this attempt ran is known too");
15499 assert_ne!(
15500 runner.state.status,
15501 RunStatus::Blocked,
15502 "contention is evidence about the machine, not the patch — it must not settle the \
15503 run as blocked: {:?}",
15504 runner.state.status
15505 );
15506 assert!(
15507 !runner
15508 .state
15509 .events
15510 .iter()
15511 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15512 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15513 runner.state.events
15514 );
15515
15516 runner
15521 .stop_reviewing("round budget spent", &shell, &repo)
15522 .await
15523 .expect("stop_reviewing retry");
15524 assert_eq!(
15525 runner.state.reviews.len(),
15526 1,
15527 "no new round was started: {:?}",
15528 runner.state.reviews
15529 );
15530 let last = runner.state.reviews.last().expect("round record");
15531 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15532 assert!(
15533 last.verified_at.expect("still known") > first_attempt_at,
15534 "a second reentry must be a fresh attempt, not a stale copy of the first"
15535 );
15536 assert_ne!(runner.state.status, RunStatus::Blocked);
15537
15538 held.release();
15539 }
15540
15541 #[tokio::test]
15553 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15554 crate::run::pin_test_home();
15555 let home = crate::run::home();
15556
15557 let tmp = tempfile::tempdir().expect("tempdir");
15558 let repo = tmp.path().join("repo");
15559 std::fs::create_dir_all(&repo).unwrap();
15560 init_repo(&repo);
15561 let head = crate::git::rev_parse(&repo, "HEAD")
15562 .await
15563 .expect("rev-parse");
15564 let cache_dir = tmp.path().join("target");
15565
15566 let mut config = Config::default();
15567 config.verify.e2e = vec![format!(
15568 "CARGO_TARGET_DIR='{}' test -f README.md",
15569 cache_dir.display()
15570 )];
15571 config.graph.review_rounds = 1;
15572 config.graph.timeout_verify = Some(2);
15573 config.graph.timeout_resource_wait = 2;
15574
15575 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15576 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15577 .expect("no io error acquiring directly")
15578 {
15579 crate::cache::AcquireOutcome::Acquired(g) => g,
15580 crate::cache::AcquireOutcome::Busy(b) => {
15581 panic!("expected the direct acquire to win the lease first: {b:?}")
15582 }
15583 };
15584
15585 let mut state = RunState::new(
15586 repo.clone(),
15587 "main".to_owned(),
15588 head.clone(),
15589 "task".to_owned(),
15590 config,
15591 );
15592 state.candidates = vec![Candidate {
15593 index: 0,
15594 label: 'A',
15595 agent: "alpha".to_owned(),
15596 branch: "does-not-exist".to_owned(),
15597 worktree: repo.clone(),
15598 summary: String::new(),
15599 stat: String::new(),
15600 files: 0,
15601 commits: 0,
15602 empty: false,
15603 failed: None,
15604 verified_noop: None,
15605 duration_ms: 0,
15606 folded: false,
15607 }];
15608 state.tally = Some(Tally {
15609 first_choice: BTreeMap::from([('A', 1)]),
15610 borda: BTreeMap::new(),
15611 winner: 'A',
15612 rankings: 1,
15613 unanimous_initial: true,
15614 deliberated: false,
15615 changed_votes: 0,
15616 unanimous_final: true,
15617 tie_break: None,
15618 judges: 0,
15619 present: 0,
15620 quorum: 0,
15621 met_quorum: true,
15622 uncontested: Some("only candidate A produced a change".to_owned()),
15623 });
15624 state.reviews = vec![ReviewRound {
15628 round: 1,
15629 head: head.clone(),
15630 verified_head: Some(head.clone()),
15631 verified_at: Some(jiff::Timestamp::now()),
15632 reviews: Vec::new(),
15633 e2e: vec![CommandOutcome {
15634 command: format!(
15635 "CARGO_TARGET_DIR='{}' test -f README.md",
15636 cache_dir.display()
15637 ),
15638 code: None,
15639 output_tail: "waiting for the shared build cache".to_owned(),
15640 duration_ms: 0,
15641 resource_blocked: true,
15642 }],
15643 fix: None,
15644 blocking: 1,
15645 answered: 1,
15646 expected: 1,
15647 clean: false,
15648 verify_retried: false,
15649 e2e_deferred: false,
15650 e2e_defer_reason: None,
15651 progressed: false,
15652 vote_split: false,
15653 reconsideration: Vec::new(),
15654 verdict: None,
15655 }];
15656
15657 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15658 let mut runner = Runner {
15659 state,
15660 roles: ResolvedRoles {
15661 implementers: Vec::new(),
15662 judges: Vec::new(),
15663 reviewers: Vec::new(),
15664 fixer: None,
15665 conductor: conductor(),
15666 implementer_roster: Vec::new(),
15667 judge_roster: Vec::new(),
15668 reviewer_roster: Vec::new(),
15669 },
15670 sem: Arc::new(Semaphore::new(1)),
15671 pause: Pause::new(),
15672 interrupt: Pause::new(),
15673 };
15674
15675 runner.review_loop().await.expect("review_loop");
15680
15681 assert_eq!(
15682 runner.state.reviews.len(),
15683 1,
15684 "no new round was started on top of the unresolved one: {:?}",
15685 runner.state.reviews
15686 );
15687 let last = &runner.state.reviews[0];
15688 assert_eq!(
15689 last.e2e_status(),
15690 E2eStatus::ResourceBlocked,
15691 "still contended: {last:?}"
15692 );
15693 assert!(
15694 last.verified_at.expect("still known") > first_attempt_at,
15695 "review_loop must have actually retried the check, not left it exactly as found"
15696 );
15697 assert_ne!(
15698 runner.state.status,
15699 RunStatus::Blocked,
15700 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15701 runner.state.status
15702 );
15703
15704 held.release();
15705 }
15706
15707 #[tokio::test]
15708 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15709 crate::run::pin_test_home();
15710 let tmp = tempfile::tempdir().expect("tempdir");
15711 let repo = tmp.path().join("repo");
15712 std::fs::create_dir_all(&repo).unwrap();
15713 init_repo(&repo);
15714
15715 let mut config = Config::default();
15716 config.merge.mode = MergeMode::Pr;
15717 config.graph.land = true;
15718 config.graph.land_approval = false;
15719
15720 let mut state = RunState::new(
15721 repo.clone(),
15722 "main".to_owned(),
15723 "deadbeef".to_owned(),
15724 "task".to_owned(),
15725 config,
15726 );
15727 state.candidates = vec![Candidate {
15728 index: 0,
15729 label: 'A',
15730 agent: "alpha".to_owned(),
15731 branch: "does-not-exist".to_owned(),
15732 worktree: repo.clone(),
15733 summary: String::new(),
15734 stat: String::new(),
15735 files: 0,
15736 commits: 0,
15737 empty: false,
15738 failed: None,
15739 verified_noop: None,
15740 duration_ms: 0,
15741 folded: false,
15742 }];
15743 state.tally = Some(Tally {
15744 first_choice: BTreeMap::from([('A', 1)]),
15745 borda: BTreeMap::new(),
15746 winner: 'A',
15747 rankings: 1,
15748 unanimous_initial: true,
15749 deliberated: false,
15750 changed_votes: 0,
15751 unanimous_final: true,
15752 tie_break: None,
15753 judges: 0,
15754 present: 0,
15755 quorum: 0,
15756 met_quorum: true,
15757 uncontested: Some("only candidate A produced a change".to_owned()),
15758 });
15759 state.reviews = vec![ReviewRound {
15760 round: 1,
15761 head: "deadbeef".to_owned(),
15762 verified_head: None,
15763 verified_at: None,
15764 reviews: Vec::new(),
15765 e2e: Vec::new(),
15766 fix: None,
15767 blocking: 0,
15768 answered: 0,
15769 expected: 0,
15770 clean: true,
15771 verify_retried: false,
15772 e2e_deferred: false,
15773 e2e_defer_reason: None,
15774 progressed: false,
15775 vote_split: false,
15776 reconsideration: Vec::new(),
15777 verdict: None,
15778 }];
15779 state.gate = vec![CommandOutcome {
15780 command: "test".to_owned(),
15781 code: Some(0),
15782 output_tail: String::new(),
15783 duration_ms: 0,
15784 resource_blocked: false,
15785 }];
15786 state.gate_ran = true;
15787 state.status = RunStatus::Landing;
15791 state.merge = Some(MergeOutcome {
15792 mode: MergeMode::Pr,
15793 ok: true,
15794 detail: "https://example.invalid/x/y/pull/1".to_owned(),
15795 empty: false,
15796 });
15797
15798 ask_test_home();
15802 state.save().expect("save landing run");
15805 let store = ask::Questions::open();
15806 let q = ask_open_question(&store, &state.id);
15807
15808 let mut runner = Runner {
15809 state,
15810 roles: ResolvedRoles {
15811 implementers: Vec::new(),
15812 judges: Vec::new(),
15813 reviewers: Vec::new(),
15814 fixer: None,
15815 conductor: conductor(),
15816 implementer_roster: Vec::new(),
15817 judge_roster: Vec::new(),
15818 reviewer_roster: Vec::new(),
15819 },
15820 sem: Arc::new(Semaphore::new(1)),
15821 pause: Pause::new(),
15822 interrupt: Pause::new(),
15823 };
15824
15825 runner.execute().await.expect("execute");
15830
15831 assert_eq!(
15832 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15833 Some("https://example.invalid/x/y/pull/1"),
15834 "reentry must not push again or open a second pull request over the \
15835 one `land` is already watching"
15836 );
15837 assert_ne!(
15838 runner.state.status,
15839 RunStatus::Landing,
15840 "land could not actually reach the fake pull request, so it must \
15841 have given up rather than left the run silently parked forever"
15842 );
15843 assert_eq!(runner.state.status, RunStatus::Blocked);
15847 assert!(
15848 store.get(&q.id).unwrap().status.open(),
15849 "Blocked is still alive; settle_questions must have been a no-op here"
15850 );
15851 }
15852
15853 fn state_with_round(round: ReviewRound) -> RunState {
15854 let mut s = RunState::new(
15855 PathBuf::from("/repo"),
15856 "main".to_owned(),
15857 "abc1234".to_owned(),
15858 "add retries".to_owned(),
15859 Config::default(),
15860 );
15861 s.reviews = vec![round];
15862 s
15863 }
15864
15865 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15866 crate::verdict::Finding {
15867 id: id.to_owned(),
15868 severity,
15869 file: None,
15870 line: None,
15871 title: title.to_owned(),
15872 detail: String::new(),
15873 }
15874 }
15875
15876 #[test]
15877 fn pr_body_names_open_findings_and_declined_ones() {
15878 let round = ReviewRound {
15879 round: 2,
15880 head: "deadbee".to_owned(),
15881 verified_head: None,
15882 verified_at: None,
15883 reviews: vec![ReviewRecord {
15884 attempts: 0,
15885 reviewer: 1,
15886 agent: "alpha".to_owned(),
15887 summary: String::new(),
15888 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15889 vote: None,
15890 failed: None,
15891 duration_ms: 0,
15892 }],
15893 e2e: vec![CommandOutcome {
15894 command: "cargo test".to_owned(),
15895 code: Some(0),
15896 output_tail: String::new(),
15897 duration_ms: 0,
15898 resource_blocked: false,
15899 }],
15900 verify_retried: false,
15901 e2e_deferred: false,
15902 e2e_defer_reason: None,
15903 fix: Some(FixRecord {
15904 agent: "alpha".to_owned(),
15905 addressed: Vec::new(),
15906 rejected: vec![crate::verdict::Rejection {
15907 id: "R1-1-1".to_owned(),
15908 why: "not reachable from any caller".to_owned(),
15909 }],
15910 notes: String::new(),
15911 committed: true,
15912 failed: None,
15913 duration_ms: 0,
15914 continuation: None,
15915 }),
15916 blocking: 0,
15917 answered: 1,
15918 expected: 1,
15919 clean: false,
15920 progressed: true,
15921 vote_split: false,
15922 reconsideration: Vec::new(),
15923 verdict: None,
15924 };
15925 let state = state_with_round(round);
15926 let body = pr_message(&state, 'A').body;
15927
15928 assert!(body.contains("add retries"), "the task must still be there");
15929 assert!(body.contains("R2-1-1"), "{body}");
15930 assert!(!body.contains("unused import"), "{body}");
15931 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15932 assert!(
15933 !body.contains("not reachable from any caller"),
15934 "the reason it was declined: {body}"
15935 );
15936 }
15937
15938 #[test]
15939 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15940 let round = ReviewRound {
15941 round: 1,
15942 head: "deadbee".to_owned(),
15943 verified_head: None,
15944 verified_at: None,
15945 reviews: vec![ReviewRecord {
15946 attempts: 0,
15947 reviewer: 1,
15948 agent: "alpha".to_owned(),
15949 summary: String::new(),
15950 findings: Vec::new(),
15951 vote: None,
15952 failed: None,
15953 duration_ms: 0,
15954 }],
15955 e2e: Vec::new(),
15956 verify_retried: false,
15957 e2e_deferred: false,
15958 e2e_defer_reason: None,
15959 fix: None,
15960 blocking: 0,
15961 answered: 1,
15962 expected: 1,
15963 clean: true,
15964 progressed: false,
15965 vote_split: false,
15966 reconsideration: Vec::new(),
15967 verdict: None,
15968 };
15969 let state = state_with_round(round);
15970 let body = pr_message(&state, 'A').body;
15971 assert!(!body.contains("Open review findings"), "{body}");
15972 assert!(!body.contains("Declined"), "{body}");
15973 }
15974
15975 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15976 let mut state = RunState::new(
15977 PathBuf::from("/repo"),
15978 "main".to_owned(),
15979 "abc1234".to_owned(),
15980 instruction.to_owned(),
15981 Config::default(),
15982 );
15983 state.candidates.push(Candidate {
15984 index: 0,
15985 label: 'A',
15986 agent: "alpha".to_owned(),
15987 branch: "magi/x/A".to_owned(),
15988 worktree: PathBuf::from("/wt"),
15989 summary: summary.to_owned(),
15990 stat: String::new(),
15991 files: 1,
15992 commits: 1,
15993 empty: false,
15994 failed: None,
15995 verified_noop: None,
15996 folded: false,
15997 duration_ms: 0,
15998 });
15999 state
16000 }
16001
16002 fn review_state(subjects: &[&str]) -> RunState {
16003 let mut state = state_with_summary(
16004 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16005 "",
16006 );
16007 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16008 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16009 state
16010 }
16011
16012 #[tokio::test]
16015 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16016 ask_test_home();
16017 let tmp = tempfile::tempdir().unwrap();
16018 let repo = tmp.path().join("repo");
16019 std::fs::create_dir_all(&repo).unwrap();
16020 init_repo(&repo);
16021 let origin = tmp.path().join("origin.git");
16022 let g = |dir: &Path, args: &[&str]| {
16023 let out = std::process::Command::new("git")
16024 .args(args)
16025 .current_dir(dir)
16026 .quiet()
16027 .output()
16028 .expect("spawn git");
16029 assert!(
16030 out.status.success(),
16031 "git {args:?}: {}",
16032 String::from_utf8_lossy(&out.stderr)
16033 );
16034 };
16035 g(
16036 tmp.path(),
16037 &[
16038 "clone",
16039 "--bare",
16040 "-q",
16041 repo.to_str().unwrap(),
16042 origin.to_str().unwrap(),
16043 ],
16044 );
16045 g(
16046 &repo,
16047 &["remote", "add", "origin", origin.to_str().unwrap()],
16048 );
16049 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16050
16051 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16053 g(&repo, &["add", "-A"]);
16054 g(
16055 &repo,
16056 &["commit", "-q", "-m", "chore(deps): update a crate"],
16057 );
16058 g(&repo, &["push", "-q", "origin", "main"]);
16059 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16060 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16061 g(&repo, &["add", "-A"]);
16062 g(
16063 &repo,
16064 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16065 );
16066 g(&repo, &["checkout", "-q", "main"]);
16067
16068 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16069 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16070 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16072 assert_eq!(fallback, c1);
16073
16074 let mut state = review_state(&[
16075 "chore(deps): update a crate",
16076 "fix(daemon): apply a chosen action",
16077 ]);
16078 state.repo = repo.clone();
16079 state.base_branch = "main".to_owned();
16080 state.base_commit = c1;
16081 refresh_reviewed_commits(&mut state, "feat/own").await;
16082 assert_eq!(
16083 state.reviewed_commits,
16084 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16085 );
16086 assert_eq!(
16087 review_title(&state).as_deref(),
16088 Some("fix(daemon): apply a chosen action")
16089 );
16090 assert_eq!(
16091 leaked_subjects(&state, "feat/own").await,
16092 Some(vec!["chore(deps): update a crate".to_owned()])
16093 );
16094 state.config.merge.remote = "nowhere".to_owned();
16097 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16098 }
16099
16100 struct SummaryFixture {
16104 _tmp: tempfile::TempDir,
16105 repo: PathBuf,
16106 reply: PathBuf,
16107 runner: Runner,
16108 }
16109
16110 fn summary_fixture() -> SummaryFixture {
16111 ask_test_home();
16112 let tmp = tempfile::tempdir().unwrap();
16113 let repo = tmp.path().join("repo");
16114 std::fs::create_dir_all(&repo).unwrap();
16115 init_repo(&repo);
16116 let origin = tmp.path().join("origin.git");
16117 let g = |args: &[&str]| {
16118 let out = std::process::Command::new("git")
16119 .args(args)
16120 .current_dir(&repo)
16121 .quiet()
16122 .output()
16123 .expect("spawn git");
16124 assert!(
16125 out.status.success(),
16126 "git {args:?}: {}",
16127 String::from_utf8_lossy(&out.stderr)
16128 );
16129 };
16130 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16131 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16132 g(&["push", "-q", "origin", "main"]);
16133 g(&["checkout", "-q", "-b", "feat/own"]);
16134 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16135 g(&["add", "-A"]);
16136 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16137 let reply = tmp.path().join("reply.txt");
16138 std::fs::write(
16139 &reply,
16140 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16141 )
16142 .unwrap();
16143 let mut runner = runner_at(RunStatus::Gating);
16144 let mut state = review_state(&["feat(own): add the own file"]);
16145 state.repo = repo.clone();
16146 state.base_branch = "main".to_owned();
16147 state.candidates[0].branch = "feat/own".to_owned();
16148 state.candidates[0].worktree = repo.clone();
16149 let mut author = spec("alpha");
16150 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16151 author
16152 .env
16153 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16154 state.config.agents = vec![author];
16155 runner.state = state;
16156 SummaryFixture {
16157 _tmp: tmp,
16158 repo,
16159 reply,
16160 runner,
16161 }
16162 }
16163
16164 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16165 let winner = f.runner.state.candidates[0].clone();
16166 f.runner
16167 .guarded_pr_message(&winner, None, true)
16168 .await
16169 .unwrap()
16170 .expect("an English description never parks")
16171 }
16172
16173 fn clean_round_with_minor_findings() -> ReviewRound {
16174 let mut round = passing_round(2, None);
16175 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16176 round.reviews = vec![ReviewRecord {
16177 attempts: 0,
16178 reviewer: 1,
16179 agent: "alpha".into(),
16180 summary: "内部レビュー".into(),
16181 findings: vec![
16182 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16183 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16184 ],
16185 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16186 failed: None,
16187 duration_ms: 0,
16188 }];
16189 round
16190 }
16191
16192 #[tokio::test]
16193 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16194 for review_only in [false, true] {
16195 let mut f = summary_fixture();
16196 if !review_only {
16197 f.runner.state.candidates[0].agent = "alpha".into();
16198 }
16199 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16200 assert!(f.runner.state.reviews[0].clean);
16201 assert!(f.runner.state.reviews[0].fix.is_none());
16202 assert!(f.runner.state.open_findings().is_empty());
16203 assert_eq!(
16204 crate::pr_outcome::required_finding_ids(&f.runner.state),
16205 ["R2-1-1", "R2-1-2"]
16206 );
16207 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16208 let evidence = crate::pr_outcome::summary_evidence(
16209 &f.runner.state,
16210 crate::pr_outcome::FinalHead::At(&head),
16211 );
16212 assert!(
16213 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16214 );
16215 let omitted = posted(&mut f).await;
16217 assert!(omitted.body.contains("No description"));
16218 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16219 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16220 assert!(f.runner.state.change_summary.is_none());
16221 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16222 let described = posted(&mut f).await;
16223 assert!(described.body.contains("improve documentation for callers"));
16224 assert!(described.body.contains("remaining edge case"));
16225 assert!(described.body.contains("## Open review findings"));
16226 assert!(!described.body.contains("未修正"));
16227 }
16228 }
16229
16230 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16231 let mut f = summary_fixture();
16232 if !review_only {
16233 f.runner.state.candidates[0].agent = "alpha".into();
16234 }
16235 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16236 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16237 fixed
16238 .fix
16239 .as_mut()
16240 .unwrap()
16241 .rejected
16242 .push(crate::verdict::Rejection {
16243 id: "R1-1-1".into(),
16244 why: "呼び出されないコードのため辞退".into(),
16245 });
16246 let mut clean = clean_round_with_minor_findings();
16247 clean.verified_head = Some(head.clone());
16248 f.runner.state.reviews = vec![fixed, clean];
16249 ok_gate(&mut f.runner.state, Some(&head));
16250 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16251 f.runner.state.config.agents.push(scripted("beta", reply));
16252 f.runner.state.config.roles.language_judge =
16253 Some(crate::config::AgentChoice::One("beta".into()));
16254 f
16255 }
16256
16257 #[tokio::test]
16258 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16259 for review_only in [false, true] {
16260 let mut f = rewrite_fixture_with_findings(review_only,
16261 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16262 let m = posted(&mut f).await;
16263 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16264 let expected =
16265 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16266 assert!(m.body.contains(&expected));
16267 assert!(m.body.contains("## Review rounds"));
16268 assert!(
16269 m.body
16270 .contains("Gate (`verify.gate`): passed on the final commit")
16271 );
16272 assert!(
16273 m.body
16274 .contains("Tests (`verify.e2e`): passed on the final commit")
16275 );
16276 let c = f.runner.state.change_summary.as_ref().unwrap();
16277 assert!(
16278 m.body
16279 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16280 );
16281 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16282 assert!(crate::pr_outcome::covers_required_findings(
16283 &f.runner.state,
16284 &m.body
16285 ));
16286 assert!(m.body.contains("documentation remains incomplete"));
16287 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16288 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16289 assert!(f.runner.state.github_text.is_none());
16290 }
16291 }
16292
16293 #[tokio::test]
16294 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16295 for review_only in [false, true] {
16296 for reply in [
16297 "TITLE: fix: public description\nA short English description.",
16298 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16299 ] {
16300 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16301 let winner = f.runner.state.candidates[0].clone();
16302 let posted = f
16303 .runner
16304 .guarded_pr_message(&winner, None, true)
16305 .await
16306 .unwrap();
16307 assert!(posted.is_none());
16308 assert!(f.runner.state.parked);
16309 let withheld = f.runner.state.github_text.as_ref().unwrap();
16310 assert!(
16311 withheld.rewrite.is_none(),
16312 "an incomplete rewrite must not be selectable"
16313 );
16314 let q = ask::Questions::open()
16315 .get(withheld.question.as_ref().unwrap())
16316 .unwrap();
16317 assert!(
16318 !q.choices
16319 .contains(&crate::github_text::USE_REWRITE.to_owned())
16320 );
16321 assert!(
16322 q.detail.contains("R1-1-1")
16323 && q.detail.contains("R2-1-1")
16324 && q.detail.contains("R2-1-2")
16325 );
16326 }
16327 }
16328 }
16329
16330 #[tokio::test]
16331 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16332 for review_only in [false, true] {
16333 let mut f = summary_fixture();
16334 if !review_only {
16335 f.runner.state.candidates[0].agent = "alpha".into();
16336 }
16337 f.runner.state.instruction = "内部タスクの説明".into();
16338 f.runner.state.candidates[0].summary = "内部の候補報告".into();
16339 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16340 f.runner.state.reviews = vec![
16341 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16342 round_with_notes(2, Some("二回目の内部修正報告です。")),
16343 passing_round(3, Some(&head)),
16344 ];
16345 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16346 ok_gate(&mut f.runner.state, Some(&head));
16347 let captured = f.repo.join("public-prompt.txt");
16348 let author = &mut f.runner.state.config.agents[0];
16349 author.command = vec![
16350 "sh".into(),
16351 "-c".into(),
16352 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16353 ];
16354 author.env.insert(
16355 "PROMPT_FILE".into(),
16356 captured.to_string_lossy().into_owned(),
16357 );
16358 let m = posted(&mut f).await;
16359 assert!(m.body.contains("Adds `own.txt`"));
16360 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16361 assert!(m.body.contains("R1-1-1"));
16362 assert!(
16363 m.body
16364 .contains("Gate (`verify.gate`): passed on the final commit")
16365 );
16366 assert!(
16367 m.body
16368 .contains("Tests (`verify.e2e`): passed on the final commit")
16369 );
16370 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16371 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16372 let prompt = std::fs::read_to_string(captured).unwrap();
16373 assert!(prompt.contains(&format!(
16374 "git diff {} {head}",
16375 f.runner.state.change_summary.as_ref().unwrap().base
16376 )));
16377 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16378 assert_eq!(
16379 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16380 "二回目の内部修正報告です。"
16381 );
16382 }
16383 }
16384
16385 #[tokio::test]
16386 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16387 for review_only in [false, true] {
16388 let mut f = summary_fixture();
16389 if !review_only {
16390 f.runner.state.candidates[0].agent = "alpha".into();
16391 }
16392 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16393 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16394 std::fs::write(&f.reply, "no usable description").unwrap();
16395 let failed = posted(&mut f).await;
16396 assert!(
16397 failed
16398 .body
16399 .contains("No description of what this change does could be generated")
16400 );
16401 assert!(failed.body.is_ascii());
16402 assert!(f.runner.state.github_text.is_none());
16403 assert!(f.runner.state.change_summary.is_none());
16404 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16405 let evidence = crate::pr_outcome::summary_evidence(
16406 &f.runner.state,
16407 crate::pr_outcome::FinalHead::At(&head),
16408 );
16409 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16410 assert!(
16411 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16412 .exists()
16413 );
16414 f.runner.state =
16415 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16416 std::fs::write(
16417 &f.reply,
16418 "TITLE: feat(own): add the own file\nAdds the committed own file.",
16419 )
16420 .unwrap();
16421 let retried = posted(&mut f).await;
16422 assert!(retried.body.contains("Adds the committed own file."));
16423 assert_eq!(
16424 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16425 "日本語の修正報告"
16426 );
16427 }
16428 }
16429
16430 #[tokio::test]
16431 async fn same_head_verification_and_findings_invalidate_public_prose() {
16432 let mut f = summary_fixture();
16433 posted(&mut f).await;
16434 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16435 let original = f
16436 .runner
16437 .state
16438 .change_summary
16439 .as_ref()
16440 .unwrap()
16441 .evidence
16442 .clone();
16443 ok_gate(&mut f.runner.state, Some(&head));
16444 std::fs::write(
16445 &f.reply,
16446 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16447 )
16448 .unwrap();
16449 let verified = posted(&mut f).await;
16450 assert!(verified.body.contains("now verified"));
16451 assert_ne!(
16452 f.runner.state.change_summary.as_ref().unwrap().evidence,
16453 original
16454 );
16455 let mut round = round_with_notes(1, Some("内部報告"));
16456 round.reviews.push(ReviewRecord {
16457 attempts: 0,
16458 reviewer: 1,
16459 agent: "alpha".into(),
16460 summary: "内部レビュー".into(),
16461 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16462 vote: None,
16463 failed: None,
16464 duration_ms: 0,
16465 });
16466 round
16467 .fix
16468 .as_mut()
16469 .unwrap()
16470 .rejected
16471 .push(crate::verdict::Rejection {
16472 id: "R1-1-1".into(),
16473 why: "呼び出し元から到達できない".into(),
16474 });
16475 f.runner.state.reviews = vec![round];
16476 let omitted = posted(&mut f).await;
16478 assert!(omitted.body.contains("No description"));
16479 assert!(omitted.body.contains("R1-1-1"));
16480 assert!(!omitted.body.contains("到達"));
16481 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16482 let described = posted(&mut f).await;
16483 assert!(described.body.contains("unreachable from callers"));
16484 assert!(described.body.contains("R1-1-1"));
16485 assert!(!described.body.contains("到達"));
16486 let evidence = f
16487 .runner
16488 .state
16489 .change_summary
16490 .as_ref()
16491 .unwrap()
16492 .evidence
16493 .clone();
16494 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16495 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16496 assert!(posted(&mut f).await.body.contains("rationale updated"));
16497 assert_ne!(
16498 f.runner.state.change_summary.as_ref().unwrap().evidence,
16499 evidence
16500 );
16501 f.runner
16503 .state
16504 .change_summary
16505 .as_mut()
16506 .unwrap()
16507 .evidence
16508 .clear();
16509 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16510 assert!(posted(&mut f).await.body.contains("current specification"));
16511 }
16512
16513 #[tokio::test]
16514 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16515 let mut f = summary_fixture();
16516 let m = posted(&mut f).await;
16517 assert_eq!(m.title, "feat(own): add the own file");
16518 assert!(
16519 m.body
16520 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16521 "{}",
16522 m.body
16523 );
16524 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16525 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16526 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16527 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16528 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16529 let kept = f.runner.state.change_summary.as_ref().unwrap();
16530 assert_eq!(
16531 (kept.head.as_str(), kept.base.as_str()),
16532 (head.as_str(), base.as_str())
16533 );
16534 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16535 }
16536
16537 #[tokio::test]
16538 async fn review_only_summary_receives_the_configured_github_language() {
16539 let mut f = summary_fixture();
16540 f.runner.state.config.graph.github_language = "ja".into();
16541 let captured = f.repo.join("summary-prompt.txt");
16542 let author = &mut f.runner.state.config.agents[0];
16543 author.command = vec![
16544 "sh".into(),
16545 "-c".into(),
16546 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16547 ];
16548 author.env.insert(
16549 "PROMPT_FILE".into(),
16550 captured.to_string_lossy().into_owned(),
16551 );
16552 std::fs::write(
16553 &f.reply,
16554 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16555 )
16556 .unwrap();
16557 let m = posted(&mut f).await;
16558 let prompt = std::fs::read_to_string(captured).unwrap();
16559 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16560 assert!(!prompt.contains("in English"));
16561 assert_eq!(m.title, "feat(own): ファイルを追加");
16562 assert!(m.body.contains("own.txt を追加します。"));
16563 }
16564
16565 #[tokio::test]
16566 async fn review_fixes_come_after_the_whole_change_summary() {
16567 let mut f = summary_fixture();
16568 f.runner.state.reviews = vec![
16569 round_with_notes(1, Some("handled the PATH case")),
16570 round_with_notes(2, None),
16571 ];
16572 let m = posted(&mut f).await;
16573 let summary = m.body.find("## Summary").expect("summary");
16574 let fixes = m.body.find("## Review rounds").expect("review rounds");
16575 assert!(summary < fixes, "{}", m.body);
16576 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16577 assert!(m.body.contains("## Final verification"), "{}", m.body);
16578 }
16579
16580 #[tokio::test]
16581 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16582 let mut f = summary_fixture();
16583 let first = posted(&mut f).await;
16584 let first_head = f.runner.state.change_summary.clone().unwrap().head;
16585 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16587 let again = posted(&mut f).await;
16588 assert_eq!((again.title, again.body), (first.title, first.body));
16589
16590 let g = |args: &[&str]| {
16591 let out = std::process::Command::new("git")
16592 .args(args)
16593 .current_dir(&f.repo)
16594 .quiet()
16595 .output()
16596 .expect("spawn git");
16597 assert!(out.status.success(), "git {args:?}");
16598 };
16599 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16600 g(&["add", "-A"]);
16601 g(&["commit", "-q", "-m", "fix(own): handle more"]);
16602 let second = posted(&mut f).await;
16603 let kept = f.runner.state.change_summary.clone().unwrap();
16604 assert_ne!(kept.head, first_head);
16605 assert!(second.body.contains("Not read yet."), "{}", second.body);
16606 assert!(!second.body.contains("so the first summary is read"));
16607 assert_eq!(second.title, "feat(own): other");
16608 }
16609
16610 #[tokio::test]
16611 async fn an_unavailable_summary_is_stated_and_never_invented() {
16612 let mut f = summary_fixture();
16613 std::fs::write(&f.reply, "no title line here").unwrap();
16614 let m = posted(&mut f).await;
16615 assert!(
16616 m.body
16617 .contains("No description of what this change does could be generated"),
16618 "{}",
16619 m.body
16620 );
16621 assert!(m.body.contains("internal reports remain in local records"));
16622 assert!(f.runner.state.change_summary.is_none());
16623 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16624 let winner = f.runner.state.candidates[0].clone();
16626 let preview = f
16627 .runner
16628 .guarded_pr_message(&winner, None, false)
16629 .await
16630 .unwrap()
16631 .unwrap();
16632 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16633 }
16634
16635 #[test]
16636 fn pr_message_review_single_commit_uses_its_subject() {
16637 let state = review_state(&["feat(nats): per-role user"]);
16638 let m = pr_message(&state, 'A');
16639 assert_eq!(m.title, "feat(nats): per-role user");
16640 assert!(!m.body.contains("Review the work already"), "{}", m.body);
16641 assert!(m.body.contains("## Commits under review"), "{}", m.body);
16642 }
16643
16644 #[test]
16645 fn pr_message_review_multi_commit_takes_the_oldest() {
16646 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16647 let m = pr_message(&state, 'A');
16648 assert_eq!(m.title, "feat: the change");
16649 for s in ["feat: the change", "fix: typo", "fix: again"] {
16650 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16651 }
16652 }
16653
16654 #[test]
16655 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16656 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16657 let mut state = review_state(&[first, "fix: later fixup"]);
16658 state.candidates[0].branch = "機能/ログイン".to_owned();
16659 let m = pr_message(&state, 'A');
16660 assert!(
16661 m.title.starts_with("chore: land candidate A of run"),
16662 "{}",
16663 m.title
16664 );
16665 }
16666 }
16667
16668 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16669 BranchFacts {
16670 commits: commits
16671 .iter()
16672 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16673 .collect(),
16674 stat: stat.to_owned(),
16675 }
16676 }
16677
16678 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16679 let mut r = review_round(true, 0, 1, 1, true, true);
16680 r.round = round;
16681 r.fix = notes.map(|n| FixRecord {
16682 agent: "fixer".to_owned(),
16683 addressed: Vec::new(),
16684 rejected: Vec::new(),
16685 notes: n.to_owned(),
16686 committed: true,
16687 failed: None,
16688 duration_ms: 0,
16689 continuation: None,
16690 });
16691 r
16692 }
16693
16694 #[test]
16695 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16696 let state = review_state(&["ignored"]);
16697 let f = facts(
16698 &[
16699 (
16700 "fix(login): resolve PATH on macOS",
16701 "Login shells skip rc files.",
16702 ),
16703 ("fix: address review", ""),
16704 ],
16705 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16706 );
16707 let m = pr_message_with(&state, 'A', Some(&f));
16708 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16709 assert!(
16710 m.body
16711 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
16712 "{}",
16713 m.body
16714 );
16715 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16716 assert!(m.body.contains("## Diff stat"), "{}", m.body);
16717 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16718 for banned in [
16719 "Review the work already",
16720 "no task statement",
16721 "Original task",
16722 ] {
16723 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16724 }
16725 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16726 }
16727
16728 #[test]
16729 fn pr_message_review_truncates_a_huge_first_commit_body() {
16730 let state = review_state(&["ignored"]);
16731 let f = facts(
16732 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16733 "s",
16734 );
16735 let m = pr_message_with(&state, 'A', Some(&f));
16736 assert!(m.body.len() < 30_000, "{}", m.body.len());
16737 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16738 assert!(
16739 m.body.contains("- ... 1 more commit(s)"),
16740 "{}",
16741 m.body.len()
16742 );
16743 assert!(m.body.ends_with("magi:candidate-a\n"));
16744 }
16745
16746 #[test]
16747 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16748 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16749 assert_eq!(m.title, "feat: x");
16750 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16751 assert!(!m.body.contains("Diff stat"), "{}", m.body);
16752 assert!(!m.body.contains("no task statement"), "{}", m.body);
16753 }
16754
16755 #[test]
16756 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16757 let mut state = review_state(&["日本語の件名"]);
16758 state.candidates[0].branch = "fix/macos-login-path".to_owned();
16759 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16760 state.candidates[0].branch = "機能/ログイン".to_owned();
16761 assert!(
16762 pr_message(&state, 'A')
16763 .title
16764 .starts_with("chore: land candidate A")
16765 );
16766 }
16767
16768 #[test]
16769 fn pr_message_review_fixes_survive_a_clean_final_round() {
16770 let mut state = review_state(&["feat: x"]);
16771 state.reviews = vec![
16772 round_with_notes(1, Some("handled the PATH case")),
16773 round_with_notes(2, None),
16774 ];
16775 let body = pr_message(&state, 'A').body;
16776 assert!(
16777 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16778 "{body}"
16779 );
16780 assert!(body.contains("### Round 1"), "{body}");
16781
16782 state.reviews = vec![
16783 round_with_notes(1, Some("first fix")),
16784 round_with_notes(2, Some("")),
16785 round_with_notes(3, Some("second fix")),
16786 round_with_notes(4, None),
16787 ];
16788 let body = pr_message(&state, 'A').body;
16789 assert!(
16790 body.contains("### Round 1") && !body.contains("first fix"),
16791 "{body}"
16792 );
16793 assert!(
16794 body.contains("### Round 3") && !body.contains("second fix"),
16795 "{body}"
16796 );
16797 assert!(body.contains("### Round 2"), "{body}");
16798 }
16799
16800 const FINAL: &str = "abcdef1234567890";
16801
16802 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16803 let mut r = round_with_notes(round, None);
16804 r.verified_head = verified.map(str::to_owned);
16805 r
16806 }
16807
16808 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16809 state.gate = vec![CommandOutcome {
16810 command: "gate".to_owned(),
16811 code: Some(0),
16812 output_tail: String::new(),
16813 duration_ms: 0,
16814 resource_blocked: false,
16815 }];
16816 state.gate_ran = true;
16817 state.gate_head = head.map(str::to_owned);
16818 }
16819
16820 fn body_at(state: &RunState) -> String {
16821 pr_message_raw(
16822 state,
16823 'A',
16824 None,
16825 crate::pr_outcome::FinalHead::At(FINAL),
16826 ChangeNote::NotAsked,
16827 )
16828 .body
16829 }
16830
16831 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16832
16833 #[test]
16834 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16835 let mut state = review_state(&["feat: x"]);
16836 state.reviews = vec![
16837 round_with_notes(1, Some(SANDBOXED)),
16838 passing_round(2, Some(FINAL)),
16839 ];
16840 ok_gate(&mut state, Some(FINAL));
16841 let body = body_at(&state);
16842 assert!(!body.contains("Handled the PATH case."), "{body}");
16843 assert!(!body.contains("could not commit"), "{body}");
16844 assert!(!body.contains("could not run"), "{body}");
16845 assert!(
16846 body.contains("The fix is committed on the branch"),
16847 "{body}"
16848 );
16849 assert!(!body.contains("sandbox"), "{body}");
16850 assert!(
16851 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16852 "{body}"
16853 );
16854 assert!(
16855 body.contains("Gate (`verify.gate`): passed on the final commit"),
16856 "{body}"
16857 );
16858 assert!(
16860 state.reviews[0]
16861 .fix
16862 .as_ref()
16863 .unwrap()
16864 .notes
16865 .contains("could not commit")
16866 );
16867 }
16868
16869 #[test]
16870 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16871 let mut state = review_state(&["feat: x"]);
16872 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16873 r1.fix.as_mut().unwrap().committed = false;
16874 let mut r2 = passing_round(2, Some(FINAL));
16875 r2.e2e[0].code = Some(1);
16876 state.reviews = vec![r1, r2];
16877 let body = body_at(&state);
16878 assert!(!body.contains("could not commit"), "{body}");
16879 assert!(!body.contains("could not run"), "{body}");
16880 assert!(body.contains("This round produced no commit"), "{body}");
16881 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16882 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16883 assert!(!body.contains("passed"), "{body}");
16884 }
16885
16886 #[test]
16887 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16888 let mut state = review_state(&["feat: x"]);
16889 state.reviews = vec![
16890 round_with_notes(1, Some(SANDBOXED)),
16891 passing_round(2, Some("0123456789abcdef")),
16892 ];
16893 ok_gate(&mut state, Some("0123456789abcdef"));
16894 let body = body_at(&state);
16895 assert!(!body.contains("HTTP tests could not run"), "{body}");
16896 assert!(!body.contains("could not commit"), "{body}");
16898 assert!(
16899 body.contains("passed on an earlier commit (`0123456`)"),
16900 "{body}"
16901 );
16902 assert!(!body.contains("passed on the final commit"), "{body}");
16903
16904 ok_gate(&mut state, None);
16906 let body = body_at(&state);
16907 assert!(
16908 body.contains("passed on a commit that was not recorded"),
16909 "{body}"
16910 );
16911 }
16912
16913 #[test]
16914 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16915 let state = state_with_summary(
16916 "add retries to the client",
16917 "TITLE: feat: retries\n\nDid it.",
16918 );
16919 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16920 assert_eq!(m.title, "feat: retries");
16921 assert!(
16922 m.body.contains("<summary>Original task</summary>"),
16923 "{}",
16924 m.body
16925 );
16926 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16927 assert!(
16928 m.body
16929 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16930 "{}",
16931 m.body
16932 );
16933 }
16934
16935 #[test]
16936 fn pr_message_review_bounds_a_long_english_subject() {
16937 let long = format!("feat: {}", "word ".repeat(100));
16938 let m = pr_message(&review_state(&[&long]), 'A');
16939 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16940 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16941 }
16942
16943 #[test]
16944 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16945 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16946 let m = pr_message(&state, 'A');
16947 assert_eq!(m.title, "add retries");
16948 assert!(m.body.contains("<summary>Original task</summary>"));
16949 assert!(!m.body.contains("Commits under review"));
16950 assert_eq!(landing_subject_source(&state), state.instruction);
16951 }
16952
16953 #[test]
16954 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16955 let state = review_state(&["feat: the change", "fix: typo"]);
16956 let source = landing_subject_source(&state);
16957 assert_eq!(land::merge_subject("", &source), "feat: the change");
16958 assert_eq!(
16959 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16960 "feat: the change"
16961 );
16962 assert_eq!(
16964 land::merge_subject("feat: renamed by hand", &source),
16965 "feat: renamed by hand"
16966 );
16967 let blank = review_state(&["日本語"]);
16968 assert!(
16969 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16970 );
16971 }
16972
16973 #[test]
16974 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16975 let state = review_state(&["feat: the change"]);
16976 let source = landing_subject_source(&state);
16977 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16978 assert_eq!(
16979 land::merge_subject(landing_title(&state, old), &source),
16980 "feat: the change"
16981 );
16982 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16983 let task = state_with_summary("add retries", "");
16984 assert_eq!(landing_title(&task, old), old);
16985 }
16986
16987 #[test]
16988 fn pr_message_describes_the_change_not_the_task() {
16989 let state = state_with_summary(
16990 "今回やってほしいこと: results projector を直す",
16991 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
16992 );
16993 let m = pr_message(&state, 'A');
16994 assert_eq!(m.title, "fix(web): batch the runs list reads");
16995 assert!(
16996 m.body.starts_with("## Summary\n\n- reads run.json once"),
16997 "{}",
16998 m.body
16999 );
17000 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17001 let task_at = m.body.find("今回やってほしいこと").unwrap();
17002 let details_at = m.body.find("<details>").unwrap();
17003 assert!(
17004 details_at < task_at,
17005 "the task lives inside <details>: {}",
17006 m.body
17007 );
17008 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17009 assert!(m.body.contains("magi:candidate-a"));
17010 }
17011
17012 #[test]
17013 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17014 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17015 let m = pr_message(&state, 'A');
17016 assert_eq!(m.title, "add retries");
17017 assert!(
17018 m.body.contains("## Summary\n\n- did some things"),
17019 "{}",
17020 m.body
17021 );
17022
17023 let none = RunState::new(
17024 PathBuf::from("/repo"),
17025 "main".to_owned(),
17026 "abc1234".to_owned(),
17027 "add retries".to_owned(),
17028 Config::default(),
17029 );
17030 let m = pr_message(&none, 'A');
17031 assert_eq!(m.title, "add retries");
17032 assert!(!m.body.contains("## Summary"), "{}", m.body);
17033 }
17034
17035 #[test]
17036 fn pr_message_refuses_the_candidate_commit_subject() {
17037 for bad in [
17038 "TITLE: magi: candidate A (uncommitted work)",
17039 "TITLE: chore: stuff (uncommitted work)",
17040 "TITLE: ",
17041 ] {
17042 let state = state_with_summary("add retries", bad);
17043 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17044 }
17045 }
17046
17047 #[test]
17048 fn pr_message_bounds_a_very_long_task_and_title() {
17049 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17050 let state = state_with_summary(&long, "- nothing");
17051 let m = pr_message(&state, 'A');
17052 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17053 assert!(!m.title.contains('\n'));
17054
17055 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17056 let m = pr_message(&state, 'A');
17057 assert!(m.title.starts_with("feat: "));
17058 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17059 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17060 }
17061
17062 fn long_title_of(instruction: &str) -> String {
17063 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17064 }
17065
17066 #[test]
17067 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17068 let first = "Make the landing path keep a readable title for long tasks";
17069 let line = format!(
17070 "{first}. {}",
17071 "And then keep going with more words ".repeat(20)
17072 );
17073 let t = long_title_of(&line);
17074 assert_eq!(t, first);
17075 assert!(!t.starts_with("chore: land"));
17076 }
17077
17078 #[test]
17079 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17080 let line = "word ".repeat(200);
17081 let t = long_title_of(&line);
17082 assert!(t.ends_with("word..."), "{t}");
17083 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17084 }
17085
17086 #[test]
17087 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17088 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17089 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17090 }
17091 }
17092
17093 #[test]
17094 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17095 let mut state = state_with_summary("再試行を追加する", "- nothing");
17096 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17097 state.config.graph.github_language = "ja".into();
17098 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17099 }
17100
17101 #[test]
17102 fn pr_message_title_limit_is_exact() {
17103 let at = "a".repeat(PR_TITLE_MAX);
17104 assert_eq!(long_title_of(&at), at);
17105 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17106 assert!(over.ends_with("..."), "{over}");
17107 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17108 }
17109
17110 #[test]
17111 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17112 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17113 let t = long_title_of(&line);
17114 assert!(t.ends_with("..."), "{t}");
17115 assert!(t.is_ascii(), "{t}");
17116 }
17117
17118 #[test]
17119 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17120 let line = format!(
17121 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17122 "plus more filler words ".repeat(20)
17123 );
17124 let t = long_title_of(&line);
17125 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17126 }
17127
17128 #[test]
17129 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17130 let line = format!("\n\n# {}", "title words ".repeat(40));
17131 let state = state_with_summary(&line, "- nothing");
17132 let m = pr_message(&state, 'A');
17133 assert!(m.title.starts_with("title words"), "{}", m.title);
17134 assert_eq!(
17135 land::merge_subject(&m.title, &landing_subject_source(&state)),
17136 m.title
17137 );
17138 assert_eq!(
17140 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17141 "feat: renamed by hand"
17142 );
17143 }
17144
17145 #[tokio::test]
17146 async fn github_text_rewrite_is_bounded_and_falls_back() {
17147 crate::run::pin_test_home();
17148 for (reply, accepted) in [
17149 (
17150 "TITLE: fix: retries\nAdd retries for failed requests.",
17151 true,
17152 ),
17153 (
17154 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17155 false,
17156 ),
17157 (
17158 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17159 true,
17160 ),
17161 (
17162 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17163 true,
17164 ),
17165 (
17166 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17167 true,
17168 ),
17169 (
17170 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17171 false,
17172 ),
17173 ("TITLE: fix: retries\n", false),
17174 ] {
17175 let dir = tempfile::tempdir().unwrap();
17176 let mut runner = runner_at(RunStatus::Gating);
17177 runner.state = state_with_summary(
17178 "add retries",
17179 "TITLE: fix: retries\n日本語の説明を書きます。",
17180 );
17181 runner.state.repo = dir.path().to_owned();
17182 runner.state.candidates[0].worktree = dir.path().to_owned();
17183 let mut author = spec("alpha");
17184 author.command = vec![
17185 "sh".into(),
17186 "-c".into(),
17187 "printf '%s' \"$REWRITE_REPLY\"".into(),
17188 ];
17189 author.env.insert("REWRITE_REPLY".into(), reply.into());
17190 runner.state.config.agents = vec![author];
17191 seed_public_summary(&mut runner);
17192 let winner = runner.state.candidates[0].clone();
17193 let first = runner
17194 .guarded_pr_message(&winner, None, true)
17195 .await
17196 .unwrap();
17197 let message = if accepted {
17198 first.expect("a passing rewrite never asks")
17199 } else {
17200 assert!(first.is_none());
17202 assert!(runner.state.parked);
17203 let store = ask::Questions::open();
17204 let asked: Vec<_> = store
17205 .list()
17206 .into_iter()
17207 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17208 .collect();
17209 assert_eq!(asked.len(), 1);
17210 assert!(!asked[0].detail.contains("secret"));
17211 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17212 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17213 assert_eq!(
17214 asked[0]
17215 .choices
17216 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17217 rewrite.is_some()
17218 );
17219 store
17222 .update(&asked[0].id, |q| {
17223 q.abandon("test");
17224 Ok(())
17225 })
17226 .unwrap();
17227 let again = runner
17228 .guarded_pr_message(&winner, None, true)
17229 .await
17230 .unwrap()
17231 .expect("an abandoned question falls back");
17232 let n = store
17233 .list()
17234 .into_iter()
17235 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17236 .count();
17237 assert_eq!(n, 1, "asked once per rejected text");
17238 again
17239 };
17240 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17241 assert!(!message.body.contains("secret"));
17242 assert!(!message.body.contains("prefix"));
17243 assert!(!message.title.contains("secret"));
17244 assert_eq!(
17245 message.body.contains("Add retries for failed requests."),
17246 accepted
17247 );
17248 assert_eq!(
17249 runner.state.seats["github-text-rewrite"].turns, 1,
17250 "only one rewrite invocation"
17251 );
17252 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17253 if accepted {
17254 assert!(runner.state.github_text.is_none());
17255 assert!(!runner.state.parked);
17256 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17257 assert!(message.body.contains("## Final verification"));
17258 assert!(message.body.contains("Evidence: final diff"));
17259 assert!(
17260 message
17261 .body
17262 .contains(&format!("magi:run/{}", runner.state.id))
17263 );
17264 }
17265 if !accepted {
17266 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17267 assert!(
17268 message
17269 .body
17270 .contains(&format!("magi:run/{}", runner.state.id))
17271 );
17272 }
17273 }
17274 }
17275
17276 fn scripted(id: &str, reply: &str) -> AgentSpec {
17277 let mut a = spec(id);
17278 a.command = vec![
17279 "sh".into(),
17280 "-c".into(),
17281 "printf '%s' \"$REWRITE_REPLY\"".into(),
17282 ];
17283 a.env.insert("REWRITE_REPLY".into(), reply.into());
17284 a
17285 }
17286
17287 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17288 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17289
17290 fn seed_public_summary(runner: &mut Runner) {
17292 let repo = &runner.state.repo;
17293 let g = |args: &[&str]| {
17294 let out = std::process::Command::new("git")
17295 .args(args)
17296 .current_dir(repo)
17297 .quiet()
17298 .output()
17299 .unwrap();
17300 assert!(out.status.success(), "git {args:?}");
17301 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17302 };
17303 g(&["init", "-q", "-b", "main"]);
17304 g(&[
17305 "-c",
17306 "user.name=Test",
17307 "-c",
17308 "user.email=test@example.invalid",
17309 "commit",
17310 "-q",
17311 "--allow-empty",
17312 "-m",
17313 "feat: fixture",
17314 ]);
17315 let head = g(&["rev-parse", "HEAD"]);
17316 g(&["update-ref", "refs/remotes/origin/main", &head]);
17317 let evidence = crate::pr_outcome::summary_evidence(
17318 &runner.state,
17319 crate::pr_outcome::FinalHead::At(&head),
17320 );
17321 let generated = &runner.state.candidates[0].summary;
17322 runner.state.change_summary = Some(crate::run::ChangeSummary {
17323 base: head.clone(),
17324 head,
17325 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17326 title: summary_title(generated).unwrap(),
17327 body: summary_without_title(generated),
17328 });
17329 }
17330
17331 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17332 let mut runner = runner_at(RunStatus::Gating);
17333 runner.state = state_with_summary("add retries", JA_SUMMARY);
17334 if review_only {
17335 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17336 }
17337 runner.state.repo = dir.to_owned();
17338 runner.state.candidates[0].worktree = dir.to_owned();
17339 runner.state.config.agents = agents;
17340 seed_public_summary(&mut runner);
17341 runner
17342 }
17343
17344 fn gate_questions(runner: &Runner) -> usize {
17345 ask::Questions::open()
17346 .list()
17347 .into_iter()
17348 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17349 .count()
17350 }
17351
17352 #[tokio::test]
17353 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17354 crate::run::pin_test_home();
17355 let dir = tempfile::tempdir().unwrap();
17356 let mut runner = rewrite_runner(
17357 dir.path(),
17358 true,
17359 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17360 );
17361 runner.state.config.roles.language_judge =
17362 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17363 let winner = runner.state.candidates[0].clone();
17364 let msg = runner
17365 .guarded_pr_message(&winner, None, true)
17366 .await
17367 .unwrap()
17368 .expect("an English rewrite posts without asking");
17369 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17370 assert!(msg.body.contains("<details>kept</details>"));
17371 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17372 assert_eq!(gate_questions(&runner), 0);
17373 assert!(runner.state.github_text.is_none());
17374 assert!(!runner.state.seats.contains_key("impl-A"));
17375 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17376 }
17377
17378 #[tokio::test]
17379 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17380 crate::run::pin_test_home();
17381 let dir = tempfile::tempdir().unwrap();
17382 let mut runner = rewrite_runner(
17383 dir.path(),
17384 false,
17385 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17386 );
17387 runner.state.config.roles.language_judge =
17388 Some(crate::config::AgentChoice::One("beta".into()));
17389 runner.state.seats.insert(
17390 "impl-A".into(),
17391 SeatState::new("impl-A", "alpha", runner.state.seed),
17392 );
17393 let winner = runner.state.candidates[0].clone();
17394 let msg = runner
17395 .guarded_pr_message(&winner, None, true)
17396 .await
17397 .unwrap()
17398 .expect("rewritten");
17399 assert!(msg.body.contains("<details>kept</details>"));
17400 assert_eq!(runner.state.seats["impl-A"].turns, 0);
17401 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17402 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17403 }
17404
17405 #[tokio::test]
17406 async fn rewrite_reuses_a_resumable_implementer_session() {
17407 crate::run::pin_test_home();
17408 let dir = tempfile::tempdir().unwrap();
17409 let mut runner = rewrite_runner(
17410 dir.path(),
17411 false,
17412 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17413 );
17414 runner.state.config.roles.language_judge =
17415 Some(crate::config::AgentChoice::One("beta".into()));
17416 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17417 seat.turns = 1;
17418 runner.state.seats.insert("impl-A".into(), seat);
17419 let winner = runner.state.candidates[0].clone();
17420 runner
17421 .guarded_pr_message(&winner, None, true)
17422 .await
17423 .unwrap()
17424 .expect("rewritten");
17425 assert_eq!(runner.state.seats["impl-A"].turns, 2);
17426 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17427 }
17428
17429 #[tokio::test]
17430 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17431 crate::run::pin_test_home();
17432 let dir = tempfile::tempdir().unwrap();
17433 let mut broken = spec("bad");
17434 broken.command = vec!["false".into()];
17435 let mut runner = rewrite_runner(
17436 dir.path(),
17437 true,
17438 vec![
17439 broken,
17440 scripted("noformat", "Add retries."),
17441 scripted("good", EN_REWRITE),
17442 ],
17443 );
17444 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17445 "bad".into(),
17446 "noformat".into(),
17447 "good".into(),
17448 ]));
17449 let winner = runner.state.candidates[0].clone();
17450 let msg = runner
17451 .guarded_pr_message(&winner, None, true)
17452 .await
17453 .unwrap()
17454 .expect("the third agent rewrites");
17455 assert!(msg.body.contains("`src/net.rs`"));
17456 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17457 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17458 }
17459
17460 #[tokio::test]
17461 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17462 crate::run::pin_test_home();
17463 for (agents, role, saved) in [
17465 (
17466 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17467 Some("beta"),
17468 true,
17469 ),
17470 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17471 (Vec::new(), None, false),
17472 ] {
17473 let dir = tempfile::tempdir().unwrap();
17474 let mut runner = rewrite_runner(dir.path(), true, agents);
17475 runner.state.config.roles.language_judge =
17476 role.map(|r| crate::config::AgentChoice::One(r.into()));
17477 let winner = runner.state.candidates[0].clone();
17478 let first = runner
17479 .guarded_pr_message(&winner, None, true)
17480 .await
17481 .unwrap();
17482 assert!(first.is_none(), "withheld text parks the run");
17483 assert!(runner.state.parked);
17484 assert_eq!(gate_questions(&runner), 1);
17485 let g = runner.state.github_text.as_ref().unwrap();
17486 assert_eq!(g.rewrite.is_some(), saved);
17487 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17488 }
17489 }
17490
17491 #[tokio::test]
17492 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17493 crate::run::pin_test_home();
17494 let dir = tempfile::tempdir().unwrap();
17495 let mut runner = runner_at(RunStatus::Gating);
17496 runner.state =
17497 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17498 runner.state.repo = dir.path().to_owned();
17499 runner.state.candidates[0].worktree = dir.path().to_owned();
17500 let mut author = spec("alpha");
17501 author.command = vec![
17502 "sh".into(),
17503 "-c".into(),
17504 "printf '%s' \"$REWRITE_REPLY\"".into(),
17505 ];
17506 author.env.insert(
17507 "REWRITE_REPLY".into(),
17508 format!(
17509 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17510 "English sentence. ".repeat(600)
17511 ),
17512 );
17513 runner.state.config.agents = vec![author];
17514 seed_public_summary(&mut runner);
17515 let winner = runner.state.candidates[0].clone();
17516 assert!(
17517 runner
17518 .guarded_pr_message(&winner, None, true)
17519 .await
17520 .unwrap()
17521 .is_none()
17522 );
17523 let record = runner.state.github_text.clone().unwrap();
17524 let rewrite = record.rewrite.clone().unwrap();
17525 assert_eq!(rewrite.failed, ["body-language"]);
17526 assert!(
17527 rewrite
17528 .body
17529 .ends_with(&format!("magi:run/{}", runner.state.id))
17530 );
17531 let store = ask::Questions::open();
17532 let q = store.get(record.question.as_ref().unwrap()).unwrap();
17533 assert!(
17534 q.choices
17535 .contains(&crate::github_text::USE_REWRITE.to_owned())
17536 );
17537 for label in [
17538 "ORIGINAL",
17539 "FALLBACK",
17540 "REWRITE",
17541 "Rules still failed: body-language",
17542 "truncated",
17543 ] {
17544 assert!(q.detail.contains(label), "{label}");
17545 }
17546 let artifact = std::fs::read_to_string(crate::run::artifact_path(
17547 &runner.state,
17548 &crate::github_text::artifact_name(&record.fingerprint),
17549 ))
17550 .unwrap();
17551 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17552 runner.state =
17554 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17555 store
17556 .update(&q.id, |q| {
17557 q.say("まだ日本語")?;
17558 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17559 })
17560 .unwrap();
17561 assert!(
17562 runner
17563 .guarded_pr_message(&winner, None, true)
17564 .await
17565 .unwrap()
17566 .is_none()
17567 );
17568 let retry = runner.state.github_text.as_ref().unwrap();
17569 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17570 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17571 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17572 assert!(
17573 q.posting_comparison
17574 .as_ref()
17575 .unwrap()
17576 .matches_detail(&q.detail)
17577 );
17578 store
17579 .update(&q.id, |q| {
17580 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17581 })
17582 .unwrap();
17583 let message = runner
17584 .guarded_pr_message(&winner, None, true)
17585 .await
17586 .unwrap()
17587 .unwrap();
17588 assert_eq!(message.title, rewrite.title);
17589 assert_eq!(message.body, rewrite.body);
17590 assert_eq!(
17591 runner.state.github_text.as_ref().unwrap().choice,
17592 Some(GithubTextChoice::Rewrite)
17593 );
17594 let again = runner
17595 .guarded_pr_message(&winner, None, true)
17596 .await
17597 .unwrap()
17598 .unwrap();
17599 assert_eq!(again.title, message.title);
17600 assert_eq!(again.body, message.body);
17601 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17602 assert!(
17603 !runner
17604 .state
17605 .events
17606 .iter()
17607 .any(|e| e.message.contains(&rewrite.title)
17608 || e.message.contains("English sentence."))
17609 );
17610 }
17611
17612 #[tokio::test]
17613 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17614 crate::run::pin_test_home();
17615 let dir = tempfile::tempdir().unwrap();
17616 let mut runner = runner_at(RunStatus::Gating);
17617 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17618 runner.state.repo = dir.path().to_owned();
17619 runner.state.candidates[0].worktree = dir.path().to_owned();
17620 seed_public_summary(&mut runner);
17621 let winner = runner.state.candidates[0].clone();
17622 assert!(
17623 runner
17624 .guarded_pr_message(&winner, None, true)
17625 .await
17626 .unwrap()
17627 .is_none()
17628 );
17629 let record = runner.state.github_text.as_ref().unwrap();
17630 assert!(record.rewrite.is_none());
17631 let mut old = serde_json::to_value(record).unwrap();
17633 old.as_object_mut().unwrap().remove("rewrite");
17634 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17635 assert!(old.rewrite.is_none());
17636 let id = record.question.clone().unwrap();
17637 ask::Questions::open()
17638 .update(&id, |q| {
17639 q.choices.push(crate::github_text::USE_REWRITE.into());
17640 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17641 })
17642 .unwrap();
17643 let message = runner
17644 .guarded_pr_message(&winner, None, true)
17645 .await
17646 .unwrap()
17647 .unwrap();
17648 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17649 assert_eq!(
17650 runner.state.github_text.as_ref().unwrap().choice,
17651 Some(GithubTextChoice::Fallback)
17652 );
17653 }
17654
17655 #[tokio::test]
17656 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17657 crate::run::pin_test_home();
17658 let dir = tempfile::tempdir().unwrap();
17659 let mut runner = runner_at(RunStatus::Gating);
17660 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17661 runner.state.repo = dir.path().to_owned();
17662 runner.state.candidates[0].worktree = dir.path().to_owned();
17663 seed_public_summary(&mut runner);
17664 let winner = runner.state.candidates[0].clone();
17665 assert!(
17666 runner
17667 .guarded_pr_message(&winner, None, true)
17668 .await
17669 .unwrap()
17670 .is_none()
17671 );
17672 let store = ask::Questions::open();
17673 let find = |run: &str| {
17674 store
17675 .list()
17676 .into_iter()
17677 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17678 .unwrap()
17679 };
17680 let run = runner.state.id.clone();
17681 let q = find(&run);
17682 let fp = runner
17683 .state
17684 .github_text
17685 .as_ref()
17686 .unwrap()
17687 .fingerprint
17688 .clone();
17689 let name = crate::github_text::artifact_name(&fp);
17690 let path = crate::run::artifact_path(&runner.state, &name);
17691 assert!(q.detail.contains(&path.display().to_string()));
17692 assert!(q.detail.contains("title-language: title"));
17693 let saved = std::fs::read_to_string(&path).expect("artifact written");
17694 assert!(saved.contains("日本語のタイトル"));
17695 assert!(q.cwd.is_none());
17696 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17697 let say = |id: &str, text: &str, choice: &str| {
17699 store
17700 .update(id, |q| {
17701 q.thread.push(ask::Turn {
17702 who: ask::Who::Operator,
17703 body: text.to_owned(),
17704 at: jiff::Timestamp::now(),
17705 note: None,
17706 });
17707 q.answer(ask::Answer::Choice(choice.to_owned()))
17708 })
17709 .unwrap();
17710 };
17711 say(&q.id, "まだ日本語", "use my text");
17712 assert!(
17713 runner
17714 .guarded_pr_message(&winner, None, true)
17715 .await
17716 .unwrap()
17717 .is_none()
17718 );
17719 let second = find(&run);
17720 assert_ne!(second.id, q.id);
17721 assert!(second.detail.contains("did not pass"));
17722 assert!(second.detail.contains("日本語のタイトル"));
17725 assert!(!second.detail.contains("まだ日本語"));
17726 assert!(second.detail.contains(&path.display().to_string()));
17727 say(&second.id, "fix: retry failed requests", "use my text");
17728 let message = runner
17729 .guarded_pr_message(&winner, None, true)
17730 .await
17731 .unwrap()
17732 .expect("a vetted replacement is posted");
17733 assert_eq!(message.title, "fix: retry failed requests");
17734 assert_eq!(
17735 runner
17736 .state
17737 .github_text
17738 .as_ref()
17739 .unwrap()
17740 .chosen_title
17741 .as_deref(),
17742 Some("fix: retry failed requests")
17743 );
17744 assert!(runner.state.github_text.as_ref().unwrap().resolved);
17745 let again = runner
17747 .guarded_pr_message(&winner, None, true)
17748 .await
17749 .unwrap()
17750 .unwrap();
17751 assert_eq!(again.title, message.title);
17752 assert_eq!(
17753 store
17754 .list()
17755 .into_iter()
17756 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17757 .count(),
17758 2
17759 );
17760 }
17761
17762 #[test]
17763 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17764 let mut state = state_with_summary(
17768 "add retries",
17769 "TITLE: fix(web): batch reads\n- reads run.json once",
17770 );
17771 state.config.graph.language = "ja".to_owned();
17772 let m = pr_message(&state, 'A');
17773 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17774 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17775
17776 let task = "今回やってほしいこと: results projector を直す";
17779 let mut state = state_with_summary(task, "- no title line");
17780 state.config.graph.language = "ja".to_owned();
17781 let m = pr_message(&state, 'A');
17782 assert_eq!(
17783 m.title,
17784 format!("chore: land candidate A of run {}", state.id)
17785 );
17786 assert!(
17787 m.body.contains(&format!(
17788 "<summary>Original task</summary>\n\n{task}\n\n</details>"
17789 )),
17790 "{}",
17791 m.body
17792 );
17793 }
17794
17795 #[test]
17796 fn pr_message_scrubs_home_paths_and_addresses() {
17797 let state = state_with_summary(
17798 "fix it in /Users/someone/src/x",
17799 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17800 );
17801 let m = pr_message(&state, 'A');
17802 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17803 assert!(!m.body.contains(leak), "{}", m.body);
17804 }
17805 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17806 }
17807
17808 #[test]
17809 fn pr_message_survives_a_task_that_closes_details() {
17810 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17811 let m = pr_message(&state, 'A');
17812 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17813 }
17814
17815 #[test]
17816 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17817 let cmd = manual_merge_command(
17818 MergeStyle::Squash,
17819 Path::new("/repo"),
17820 "b",
17821 "fix: \"quoted\" $(x) `y`\n\nbody",
17822 );
17823 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17824 }
17825
17826 #[test]
17827 fn manual_merge_command_matches_the_configured_style() {
17828 let repo = Path::new("/repo");
17829 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17830
17831 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17832 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17833
17834 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17835 assert_eq!(
17836 squash,
17837 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17838 \"Merge magi run 0832 (candidate A)\""
17839 );
17840
17841 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17842 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17843 }
17844
17845 #[test]
17846 fn a_nudge_gets_a_quarter_of_the_budget() {
17847 assert_eq!(retry_budget(secs(1200), true), secs(300));
17849 assert_eq!(retry_budget(secs(3600), true), secs(900));
17850 }
17851
17852 #[test]
17853 fn a_resent_prompt_keeps_the_whole_budget() {
17854 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17857 assert_eq!(retry_budget(secs(60), false), secs(60));
17858 }
17859
17860 #[test]
17861 fn the_floor_never_exceeds_the_original_budget() {
17862 assert_eq!(retry_budget(secs(60), true), secs(60));
17866 assert_eq!(retry_budget(secs(480), true), secs(120));
17867 assert_eq!(retry_budget(secs(0), true), secs(0));
17868 }
17869
17870 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17871 agent::CommandEvidence {
17872 id: "item1".to_owned(),
17873 description: "cargo test".to_owned(),
17874 exit_code,
17875 result_summary: String::new(),
17876 source: "codex".to_owned(),
17877 }
17878 }
17879
17880 #[test]
17881 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17882 assert!(!has_unconfirmed_command(&[]));
17886 }
17887
17888 #[test]
17889 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17890 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17894 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17895 assert!(!has_unconfirmed_command(&[
17896 evidence(Some(0)),
17897 evidence(Some(101))
17898 ]));
17899 }
17900
17901 #[test]
17902 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17903 assert!(has_unconfirmed_command(&[
17904 evidence(Some(0)),
17905 evidence(None)
17906 ]));
17907 }
17908
17909 #[test]
17910 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17911 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17912 assert_eq!(
17913 verified_noop_claim(true, &[], text).as_deref(),
17914 Some("already fixed by b32cfc4, on main.")
17915 );
17916 }
17917
17918 #[test]
17919 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17920 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17923 assert!(verified_noop_claim(false, &[], text).is_none());
17924 }
17925
17926 #[test]
17927 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17928 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17929 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17930 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17932 }
17933
17934 #[test]
17935 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17936 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17937 }
17938
17939 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17942 runner.state.candidates = shape
17943 .iter()
17944 .enumerate()
17945 .map(|(i, &(empty, verified))| Candidate {
17946 index: i,
17947 label: (b'A' + i as u8) as char,
17948 agent: "sonnet".to_owned(),
17949 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17950 worktree: PathBuf::from(format!("/wt/{i}")),
17951 summary: String::new(),
17952 stat: String::new(),
17953 files: 0,
17954 commits: 0,
17955 empty,
17956 failed: None,
17957 verified_noop: verified.map(str::to_owned),
17958 duration_ms: 0,
17959 folded: false,
17960 })
17961 .collect();
17962 }
17963
17964 #[test]
17965 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17966 ask_test_home();
17967 let mut runner = runner_at(RunStatus::Implementing);
17968 set_candidates(
17969 &mut runner,
17970 &[
17971 (true, Some("already on main at b32cfc4")),
17972 (true, Some("same fix, see the existing test")),
17973 ],
17974 );
17975
17976 runner
17977 .after_implement()
17978 .expect("a verified no-op is not an error");
17979
17980 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17981 }
17982
17983 #[test]
17984 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17985 ask_test_home();
17986 let mut runner = runner_at(RunStatus::Implementing);
17987 set_candidates(
17991 &mut runner,
17992 &[(true, Some("already on main at b32cfc4")), (true, None)],
17993 );
17994
17995 let err = runner
17996 .after_implement()
17997 .expect_err("an unverified empty candidate must still fail the run");
17998
17999 assert!(
18000 err.to_string().contains("no candidate produced a change"),
18001 "{err}"
18002 );
18003 assert_eq!(runner.state.status, RunStatus::Failed);
18004 }
18005
18006 #[test]
18007 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18008 ask_test_home();
18009 let mut runner = runner_at(RunStatus::Implementing);
18010 set_candidates(&mut runner, &[(true, None), (true, None)]);
18011
18012 let err = runner
18013 .after_implement()
18014 .expect_err("no candidate declared anything; this is an ordinary failure");
18015
18016 assert!(
18017 err.to_string().contains("no candidate produced a change"),
18018 "{err}"
18019 );
18020 assert_eq!(runner.state.status, RunStatus::Failed);
18021 }
18022}
18023
18024#[derive(Debug)]
18027struct GraftRestoreFailed(String);
18028
18029impl std::fmt::Display for GraftRestoreFailed {
18030 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18031 write!(
18032 f,
18033 "could not restore the winner to its pre-graft commit: {}",
18034 self.0
18035 )
18036 }
18037}
18038
18039impl std::error::Error for GraftRestoreFailed {}
18040
18041#[cfg(test)]
18045mod resource_wait_tests {
18046 use super::*;
18047 use std::collections::BTreeMap;
18048
18049 fn sh(id: &str, script: &str) -> AgentSpec {
18050 AgentSpec {
18051 id: id.to_owned(),
18052 kind: crate::config::AgentKind::Command,
18053 model: None,
18054 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18055 extra_args: Vec::new(),
18056 env: BTreeMap::new(),
18057 prompt_delivery: None,
18058 }
18059 }
18060
18061 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18062 SeatJob {
18063 seat: SeatState::new("impl-A", &spec.id, 1),
18064 spec,
18065 cwd: dir.to_path_buf(),
18066 prompt: "do it".to_owned(),
18067 timeout,
18068 allow_write: true,
18069 sessions: false,
18070 artifacts: dir.join("art"),
18071 stem: "impl-A".to_owned(),
18072 handover: None,
18073 }
18074 }
18075
18076 fn state(resource_wait_secs: u64) -> RunState {
18077 let mut config = Config::default();
18078 config.graph.timeout_resource_wait = resource_wait_secs;
18079 RunState::new(
18080 PathBuf::from("/repo"),
18081 "main".to_owned(),
18082 "abc1234".to_owned(),
18083 "task".to_owned(),
18084 config,
18085 )
18086 }
18087
18088 fn hold(cache: &Path) -> crate::cache::Guard {
18089 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18090 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18091 crate::cache::AcquireOutcome::Acquired(g) => g,
18092 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18093 }
18094 }
18095
18096 async fn run(
18097 jobs: Vec<SeatJob>,
18098 sem: Arc<Semaphore>,
18099 cache: Option<&Path>,
18100 st: &mut RunState,
18101 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18102 let prompts = Prompts::default();
18103 let ctx = WaveCtx {
18104 run: "r",
18105 node: "implement",
18106 prompts: &prompts,
18107 cache,
18108 round: None,
18109 carry_seats: false,
18110 pauses: None,
18111 };
18112 wave(jobs, sem, &ctx, st, 0).await
18113 }
18114
18115 #[tokio::test]
18116 async fn verification_cache_wait_preserves_the_command_budget() {
18117 crate::run::pin_test_home();
18118 let tmp = tempfile::tempdir().unwrap();
18119 let cache = tmp.path().join("target");
18120 let guard = hold(&cache);
18121 let release = tokio::spawn(async move {
18122 tokio::time::sleep(Duration::from_millis(100)).await;
18123 guard.release();
18124 });
18125 let mut st = state(10);
18126 let budget = Duration::from_millis(50);
18127 let wait = repository_wait(&st, None);
18128 let (outcomes, _) = with_cache_lease(
18129 &mut st,
18130 Some(&cache),
18131 "gate",
18132 "gate",
18133 tmp.path(),
18134 "abc1234",
18135 budget,
18136 "test gate",
18137 &wait,
18138 |_, available| async move {
18139 assert_eq!(
18140 available, budget,
18141 "contention must not shorten verification"
18142 );
18143 (Vec::new(), false, Vec::new())
18144 },
18145 )
18146 .await
18147 .unwrap();
18148 release.await.unwrap();
18149 assert!(outcomes.is_empty(), "{outcomes:?}");
18150 assert!(
18151 st.events
18152 .iter()
18153 .any(|event| event.message.contains("waiting for"))
18154 );
18155 assert!(st.resource_wait.is_none());
18156 hold(&cache).release();
18158 }
18159
18160 #[tokio::test]
18161 async fn verification_cache_wait_expires_without_starting_commands() {
18162 crate::run::pin_test_home();
18163 let tmp = tempfile::tempdir().unwrap();
18164 let cache = tmp.path().join("target");
18165 let _held = hold(&cache);
18166 let mut st = state(0);
18167 let wait = repository_wait(&st, None);
18168 let (outcomes, _) = with_cache_lease(
18169 &mut st,
18170 Some(&cache),
18171 "gate",
18172 "gate",
18173 tmp.path(),
18174 "abc1234",
18175 Duration::from_secs(60),
18176 "test gate",
18177 &wait,
18178 |_, _| async { panic!("commands must not start without the cache lease") },
18179 )
18180 .await
18181 .unwrap();
18182 assert_eq!(outcomes.len(), 1);
18183 assert!(outcomes[0].resource_blocked);
18184 assert!(st.resource_wait.is_none());
18185 }
18186
18187 #[tokio::test]
18188 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18189 crate::run::pin_test_home();
18190 let tmp = tempfile::tempdir().unwrap();
18191 let cache = tmp.path().join("target-a");
18192 let guard = hold(&cache);
18193 tokio::spawn(async move {
18195 tokio::time::sleep(Duration::from_millis(1500)).await;
18196 guard.release();
18197 });
18198 let mut st = state(60);
18199 let results = run(
18200 vec![job(
18201 sh("alpha", "echo ok"),
18202 tmp.path(),
18203 Duration::from_secs(1),
18204 )],
18205 Arc::new(Semaphore::new(1)),
18206 Some(&cache),
18207 &mut st,
18208 )
18209 .await;
18210 assert!(
18211 matches!(results[0].2, AgentOutcome::Ok(_)),
18212 "the wait must not be charged to the agent"
18213 );
18214 assert!(st.resource_block.is_none());
18215 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18216 assert!(st.resource_wait.is_none() && st.active.is_empty());
18217 }
18218
18219 #[tokio::test]
18220 #[cfg(unix)]
18221 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18222 use std::os::fd::AsRawFd as _;
18223 crate::run::pin_test_home();
18224 let tmp = tempfile::tempdir().unwrap();
18225 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18226 let common = tmp.path().join(".git");
18227 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18228 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18229 assert_eq!(
18230 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18231 0
18232 );
18233 let mut st = state(1);
18234 st.config.coordination = crate::config::test_coordination();
18235 let mut reader = job(
18236 sh("reader", "echo must-not-run"),
18237 tmp.path(),
18238 Duration::from_secs(30),
18239 );
18240 reader.allow_write = false;
18241 let results = run(
18242 vec![reader.clone()],
18243 Arc::new(Semaphore::new(1)),
18244 None,
18245 &mut st,
18246 )
18247 .await;
18248 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18249 assert_eq!(results[0].1.turns, 0);
18250 assert!(FailClass::of(&results[0].2).is_none());
18251 assert_eq!(
18252 st.resource_block.as_ref().unwrap().kind,
18253 "repository coordination"
18254 );
18255 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18256 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18257 drop(external);
18258 let editor = crate::repo_lock::SeatLease::for_checkout(
18259 tmp.path(),
18260 true,
18261 crate::config::test_coordination().as_ref(),
18262 )
18263 .await
18264 .unwrap();
18265 let pause = Pause::new();
18266 let interrupt = Pause::new();
18267 let prompts = Prompts::default();
18268 let ctx = WaveCtx {
18269 run: "r",
18270 node: "review",
18271 prompts: &prompts,
18272 cache: None,
18273 round: None,
18274 carry_seats: false,
18275 pauses: Some((&pause, &interrupt)),
18276 };
18277 let (results, ()) = tokio::join!(
18278 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18279 async {
18280 tokio::time::sleep(Duration::from_millis(50)).await;
18281 pause.park();
18282 }
18283 );
18284 assert!(
18285 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18286 );
18287 assert_eq!(results[0].1.turns, 0);
18288 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18289 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18290 drop(editor);
18291 }
18292
18293 #[tokio::test]
18294 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18295 crate::run::pin_test_home();
18296 let tmp = tempfile::tempdir().unwrap();
18297 let cache = tmp.path().join("target-b");
18298 let _held = hold(&cache);
18299 let mut st = state(1);
18300 let results = run(
18301 vec![job(
18302 sh("alpha", "echo ok"),
18303 tmp.path(),
18304 Duration::from_secs(1),
18305 )],
18306 Arc::new(Semaphore::new(1)),
18307 Some(&cache),
18308 &mut st,
18309 )
18310 .await;
18311 let (_, seat, out) = &results[0];
18312 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18313 assert!(wave_blocked(&results));
18314 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18315 assert_eq!(seat.turns, 0, "nothing was asked");
18316 assert!(
18317 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18318 "no agent was spawned"
18319 );
18320 assert!(FailClass::of(out).is_none(), "not a failure class");
18321 let block = st.resource_block.as_ref().expect("recorded for the resume");
18322 assert_eq!(block.kind, "shared build cache");
18323 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18324 assert!(st.active.is_empty());
18325 }
18326
18327 #[tokio::test]
18328 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18329 crate::run::pin_test_home();
18330 let tmp = tempfile::tempdir().unwrap();
18331 let mut st = state(1);
18332 let results = run(
18333 vec![job(
18334 sh("alpha", "echo ok"),
18335 tmp.path(),
18336 Duration::from_secs(1),
18337 )],
18338 Arc::new(Semaphore::new(0)),
18339 None,
18340 &mut st,
18341 )
18342 .await;
18343 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18344 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18345 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18346 }
18347
18348 #[tokio::test]
18349 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18350 crate::run::pin_test_home();
18351 let tmp = tempfile::tempdir().unwrap();
18352 let mut st = state(60);
18353 let results = run(
18354 vec![job(
18355 sh("alpha", "sleep 30"),
18356 tmp.path(),
18357 Duration::from_secs(1),
18358 )],
18359 Arc::new(Semaphore::new(1)),
18360 None,
18361 &mut st,
18362 )
18363 .await;
18364 let out = &results[0].2;
18365 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18366 let class = FailClass::of(out).expect("a failure");
18367 assert_eq!(class, FailClass::Timeout);
18368 assert!(should_hand_over(None, &class), "existing handover policy");
18369 assert!(st.resource_block.is_none());
18370 }
18371}