1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(
488 repo: &Path,
489 base_branch: &str,
490 remote: &str,
491 coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493 let _lease = crate::repo_lock::SeatLease::operation_with(
494 repo,
495 coordination,
496 &crate::repo_lock::WaitPolicy::current(),
497 )
498 .await?;
499 let tracking = format!("{remote}/{base_branch}");
500 let fetched = git::fetch(repo, remote, base_branch).await;
501 if let Ok(out) = &fetched
502 && out.ok()
503 && git::rev_exists(repo, &tracking).await
504 {
505 return Ok(crate::run::FetchedBase {
506 remote: remote.to_owned(),
507 branch: base_branch.to_owned(),
508 commit: git::rev_parse(repo, &tracking).await?,
509 fetched_at: jiff::Timestamp::now(),
510 });
511 }
512 let why = match &fetched {
513 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514 Ok(_) => format!("{remote} has no {base_branch}"),
515 Err(e) => e.to_string(),
516 };
517 bail!(
520 "cannot read {tracking} ({why}); refusing to branch off the local \
521 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522 base / remote in magi.toml. This is launcher preparation: check the \
523 launcher's Git administration access, network and repository coordination \
524 before retrying; an unattended seat cannot approve those permissions"
525 )
526}
527
528struct FixClaim {
545 path: PathBuf,
546}
547
548impl FixClaim {
549 fn acquire(dir: &Path) -> Result<Self> {
550 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551 let path = dir.join("fix.lock");
552 match Self::create(&path) {
553 Ok(claim) => Ok(claim),
554 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555 if Self::reclaim_if_dead(&path) {
556 Self::create(&path).with_context(|| format!("lock {}", path.display()))
557 } else {
558 bail!(
559 "another repair or `magi fix` is already running for this run ({} exists)",
560 path.display()
561 )
562 }
563 }
564 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565 }
566 }
567
568 fn create(path: &Path) -> std::io::Result<Self> {
569 let mut f = std::fs::OpenOptions::new()
570 .write(true)
571 .create_new(true)
572 .open(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
627 repo: &Path,
628 instruction: String,
629 also_scan: &str,
630 config: Config,
631 origin: Origin,
632 ) -> Result<Self> {
633 let repo = git::toplevel(repo).await?;
634 let missing = agent::missing_programs(&config.agents);
635 if !missing.is_empty() {
636 bail!(
637 "these agent programs are not on PATH: {}. Fix the roster in \
638 magi.toml or install them.",
639 missing.join(", ")
640 );
641 }
642 let base_branch =
643 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644 .await?;
645 let fetched_base = resolve_base(
646 &repo,
647 &base_branch,
648 &config.merge.remote,
649 config.coordination.as_ref(),
650 )
651 .await?;
652 let base_commit = fetched_base.commit.clone();
653 let roles = config.resolve_roles()?;
654 let max_parallel = config.graph.max_parallel.max(1);
655 let seeds = refs::resolve(
658 &repo,
659 &base_commit,
660 &config.merge.remote,
661 &format!("{also_scan}\n{instruction}"),
662 )
663 .await;
664 refs::plan(&repo, &seeds).await?;
665 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666 state.fetched_base = Some(fetched_base);
670 state.origin = Some(origin);
671 for seed in &seeds {
672 state.event(
673 "seed",
674 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675 );
676 }
677 state.seeds = seeds;
678 state.event("start", format!("run {} created", state.id));
679 state.save()?;
680 Ok(Self {
681 state,
682 roles,
683 sem: Arc::new(Semaphore::new(max_parallel)),
684 pause: Pause::new(),
685 interrupt: Pause::new(),
686 })
687 }
688
689 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703 Self::review_taking_over(repo, branch, config, None, origin).await
704 }
705
706 pub async fn review_taking_over(
713 repo: &Path,
714 branch: &str,
715 config: Config,
716 takeover: Option<crate::handover::Takeover>,
717 origin: Origin,
718 ) -> Result<Self> {
719 let repo = git::toplevel(repo).await?;
720 let missing = agent::missing_programs(&config.agents);
721 if !missing.is_empty() {
722 bail!(
723 "these agent programs are not on PATH: {}. Fix the roster in \
724 magi.toml or install them.",
725 missing.join(", ")
726 );
727 }
728 let base_branch =
729 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730 .await?;
731 if base_branch == branch {
732 bail!("`{branch}` is the base branch; there is nothing to review against");
733 }
734 let fetched_base = resolve_base(
735 &repo,
736 &base_branch,
737 &config.merge.remote,
738 config.coordination.as_ref(),
739 )
740 .await?;
741 let base_commit = fetched_base.commit.clone();
742
743 let roles = config.resolve_roles()?;
744 let max_parallel = config.graph.max_parallel.max(1);
745 let mut state = RunState::new(
746 repo.clone(),
747 base_branch,
748 base_commit.clone(),
749 String::new(),
750 config,
751 );
752 state.fetched_base = Some(fetched_base);
753 state.origin = Some(origin);
754
755 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762 earlier: Vec::new(),
763 home: crate::run::home(),
764 choice: None,
765 task_instruction: None,
766 });
767 let wait = repository_wait(&state, None);
768 let coordination_config = state.config.coordination.clone();
769 let coordination =
770 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771 .await?;
772 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773 if let Some(released) = &released {
774 state.event(
775 "release",
776 format!(
777 "took `{branch}` over from run {}: its worktree was released: {}",
778 crate::run::short_of(&released.old_id),
779 released.audit
780 ),
781 );
782 }
783 if let Some(choice) = takeover.choice.as_ref()
787 && let Err(e) =
788 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789 .await
790 {
791 if let Some(released) = &released {
792 released.restore(&repo, branch).await;
793 }
794 return Err(e.context("applying the owner's answer about the diverged branch"));
795 }
796 for id in takeover.earlier.iter().rev() {
802 if let Ok(old) = RunState::load_under(id, &takeover.home)
803 && let Some(head) = old.pr_head.clone()
804 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805 {
806 state.pr_head = Some(head);
807 break;
808 }
809 }
810 drop(coordination);
811 let opened = Self::open_review(
812 &repo,
813 branch,
814 state,
815 roles,
816 max_parallel,
817 base_commit,
818 &takeover,
819 )
820 .await;
821 if opened.is_err()
822 && let Some(released) = &released
823 {
824 let _coordination = crate::repo_lock::SeatLease::operation_with(
825 &repo,
826 coordination_config.as_ref(),
827 &wait,
828 )
829 .await?;
830 released.restore(&repo, branch).await;
831 }
832 opened
833 }
834
835 pub async fn recover_pr(
837 repo: &Path,
838 task: &str,
839 selection: crate::pr_recovery::Selection,
840 config: Config,
841 ) -> Result<Self> {
842 selection.check_task(task, &selection.instruction)?;
843 let base_branch =
844 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
845 .await?;
846 let fetched_base = resolve_base(
847 repo,
848 &base_branch,
849 &config.merge.remote,
850 config.coordination.as_ref(),
851 )
852 .await?;
853 let base_commit = fetched_base.commit.clone();
854 selection
855 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
856 .await?;
857 let roles = config.resolve_roles()?;
858 let max_parallel = config.graph.max_parallel.max(1);
859 let mut state = RunState::new(
860 repo.to_path_buf(),
861 base_branch,
862 base_commit.clone(),
863 selection.instruction.clone(),
864 config,
865 );
866 state.fetched_base = Some(fetched_base);
867 state.origin = Some(crate::run::Origin::queue(task));
868 state.pr_head = Some(selection.branch.clone());
869 state.pr_recovery = Some(selection.clone());
870 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
871 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872 }
873
874 async fn open_selected(
875 repo: &Path,
876 state: RunState,
877 roles: ResolvedRoles,
878 max_parallel: usize,
879 base_commit: String,
880 selection: crate::pr_recovery::Selection,
881 ) -> Result<Self> {
882 let mut state = state;
883 let branch = state.branch_for('A');
884 let wait = repository_wait(&state, None);
885 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
886 git::git(repo, &["branch", &branch, &selection.head]).await?;
887 drop(coordination);
888 state.event(
889 "recovery",
890 format!(
891 "selected public implementation {} from {}",
892 selection.head, selection.url
893 ),
894 );
895 let takeover = crate::handover::Takeover {
896 earlier: Vec::new(),
897 home: crate::run::home(),
898 choice: None,
899 task_instruction: Some(selection.instruction.clone()),
900 };
901 state.save()?;
902 let id = state.id.clone();
903 let opened = Self::open_review(
904 repo,
905 &branch,
906 state,
907 roles,
908 max_parallel,
909 base_commit,
910 &takeover,
911 )
912 .await;
913 match opened {
914 Ok(runner) => Ok(runner),
915 Err(e) => {
916 if let Ok(mut partial) = RunState::load(&id) {
917 partial.status = RunStatus::Blocked;
918 partial.driver_exited = true;
919 partial.event(
920 "recovery",
921 format!("startup failed; retained source and candidate {branch}: {e:#}"),
922 );
923 partial.save()?;
924 }
925 Err(crate::pr_recovery::StartupFailed {
926 run: id,
927 cause: format!("{e:#}"),
928 }
929 .into())
930 }
931 }
932 }
933
934 async fn inherit(
939 repo: &Path,
940 branch: &str,
941 base_commit: &str,
942 takeover: &crate::handover::Takeover,
943 ) -> Option<crate::recovery::RecoveryRecord> {
944 use crate::recovery::{Relation, assess};
945 let newest = takeover.earlier.last()?;
946 let mut chosen = None;
947 for id in takeover.earlier.iter().rev() {
948 if let Ok(old) = RunState::load_under(id, &takeover.home)
949 && !old.reviews.is_empty()
950 && (old.winner().is_some_and(|w| w.branch == branch)
951 || old.pr_head.as_deref() == Some(branch))
952 {
953 chosen = Some(old);
954 break;
955 }
956 }
957 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
958 .await
959 .unwrap_or_default();
960 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
961 Some(last) if last.head == tip => Relation::Same,
962 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
963 _ => Relation::Diverged,
964 };
965 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
966 Some(assess(from, chosen.as_ref(), base_commit, relation))
967 }
968
969 async fn open_review(
972 repo: &Path,
973 branch: &str,
974 mut state: RunState,
975 roles: ResolvedRoles,
976 max_parallel: usize,
977 base_commit: String,
978 takeover: &crate::handover::Takeover,
979 ) -> Result<Self> {
980 let wait = repository_wait(&state, None);
981 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
982 if state.pr_recovery.is_none() {
983 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
984 }
985 let start = review_base(
988 repo,
989 &state.config.merge.remote,
990 &state.base_branch,
991 &base_commit,
992 branch,
993 )
994 .await;
995 let log = git::log_oneline(repo, &start, branch)
996 .await
997 .unwrap_or_default();
998 let log_text = if log.trim().is_empty() {
999 "(no commit messages)"
1000 } else {
1001 log.trim()
1002 };
1003 let mut instruction = format!(
1004 "Review the work already on branch `{branch}`. There is no task \
1005 statement: what the change claims to do is whatever its commits \
1006 say.\n\n{log_text}"
1007 );
1008 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1012 if let Some(task) = takeover
1013 .task_instruction
1014 .as_deref()
1015 .filter(|t| !t.trim().is_empty())
1016 && rec.reviewed_head.is_some()
1017 {
1018 instruction = format!(
1019 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1020 task.trim()
1021 );
1022 rec.criteria_inherited = true;
1023 }
1024 state.event("recovery", rec.summary());
1025 state.recovery = Some(rec);
1026 }
1027 if let Some(selected) = &state.pr_recovery {
1028 instruction = selected.instruction.clone();
1029 }
1030 state.task_instruction = takeover.task_instruction.clone();
1031 state.instruction = instruction;
1032 state.reviewed_commits = Some(
1033 git::subjects(repo, &start, branch)
1034 .await
1035 .unwrap_or_default(),
1036 );
1037
1038 let worktree = state.worktree_root().join("under-review");
1041 if let Some(parent) = worktree.parent() {
1042 tokio::fs::create_dir_all(parent).await.ok();
1043 }
1044 let path = worktree.to_string_lossy().to_string();
1045 git::git(repo, &["worktree", "add", &path, branch])
1046 .await
1047 .with_context(|| {
1048 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1049 })?;
1050
1051 let setup = match ensure_setup_config(&mut state, repo).await {
1052 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1053 Err(e) => Err(e),
1054 };
1055 if let Err(e) = setup {
1056 git::worktree_remove_clean(repo, &worktree).await.ok();
1057 return Err(e);
1058 }
1059 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1060 .await
1061 .unwrap_or(0);
1062 if commits == 0 {
1063 git::worktree_remove_clean(repo, &worktree).await.ok();
1064 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1065 }
1066 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1067 .await
1068 .map(|f| f.len())
1069 .unwrap_or(0);
1070 if files == 0
1071 && let (Ok(head_tree), Ok(base_tree)) = (
1072 git::tree_of(&worktree, "HEAD").await,
1073 git::tree_of(&worktree, &base_commit).await,
1074 )
1075 && head_tree == base_tree
1076 {
1077 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1078 git::worktree_remove_clean(repo, &worktree).await.ok();
1079 bail!(
1080 "`{branch}` at {} has a tree identical to base {}; this usually means \
1081 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1082 against `{}/{branch}`) rather than an empty change",
1083 short(&head),
1084 short(&base_commit),
1085 state.config.merge.remote
1086 );
1087 }
1088 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1089 .await
1090 .unwrap_or_default();
1091
1092 state.candidates.push(Candidate {
1093 index: 0,
1094 label: 'A',
1095 agent: EXISTING_BRANCH.to_owned(),
1098 branch: branch.to_owned(),
1099 worktree,
1100 summary: String::new(),
1101 stat,
1102 files,
1103 commits,
1104 empty: false,
1105 failed: None,
1106 verified_noop: None,
1107 duration_ms: 0,
1108 folded: false,
1109 });
1110 state.tally = Some(Tally {
1111 first_choice: BTreeMap::from([('A', 0)]),
1112 borda: BTreeMap::new(),
1113 winner: 'A',
1114 rankings: 0,
1115 unanimous_initial: false,
1116 deliberated: false,
1117 changed_votes: 0,
1118 unanimous_final: false,
1119 tie_break: None,
1120 judges: 0,
1124 present: 0,
1125 quorum: 0,
1126 met_quorum: true,
1127 uncontested: Some("review-only run: nothing competed".to_owned()),
1128 });
1129 state.status = RunStatus::Reviewing;
1130 state.prep_complete = Some(true);
1131 state.event(
1132 "start",
1133 format!(
1134 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1135 state.id
1136 ),
1137 );
1138 state.save()?;
1139 Ok(Self {
1140 state,
1141 roles,
1142 sem: Arc::new(Semaphore::new(max_parallel)),
1143 pause: Pause::new(),
1144 interrupt: Pause::new(),
1145 })
1146 }
1147
1148 pub fn resume(id: &str) -> Result<Self> {
1150 let state = RunState::load(id)?;
1151 crate::pr_head::ensure_owner(&state)?;
1152 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1153 bail!(
1154 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1155 );
1156 }
1157 if let Some(to) = &state.released_to {
1158 bail!(
1159 "run {} cannot be resumed: its worktree was released to run {}",
1160 state.short(),
1161 crate::run::short_of(to)
1162 );
1163 }
1164 let roles = state.config.resolve_roles()?;
1165 if let Some(id) = &state.config.graph.selected_implementer
1166 && state.candidates.iter().any(|c| c.agent != *id)
1167 {
1168 bail!(
1169 "replacement implementer pin does not match the saved candidates; task must stay held"
1170 );
1171 }
1172 let max_parallel = state.config.graph.max_parallel.max(1);
1173 Ok(Self {
1174 state,
1175 roles,
1176 sem: Arc::new(Semaphore::new(max_parallel)),
1177 pause: Pause::new(),
1178 interrupt: Pause::new(),
1179 })
1180 }
1181
1182 pub async fn execute(&mut self) -> Result<()> {
1189 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1192 || self.state.land_approval.is_some()
1193 || self.state.merge_decision.is_some()
1194 || !self.state.merge_repairs.is_empty()
1195 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1196 {
1197 let claim = FixClaim::acquire(&self.state.dir())?;
1198 let disk = RunState::load(&self.state.id)?;
1199 let checkpoint = |s: &RunState| {
1200 (
1201 s.merge_repairs.len(),
1202 s.merge_repairs
1203 .last()
1204 .map(|r| (r.question.clone(), r.phase)),
1205 s.land_approval
1206 .as_ref()
1207 .map(|a| (a.question.clone(), a.head.clone())),
1208 )
1209 };
1210 if checkpoint(&disk) != checkpoint(&self.state) {
1211 bail!(
1212 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1213 );
1214 }
1215 Some(claim)
1216 } else {
1217 None
1218 };
1219 let mut execution = self.claim_driver()?;
1220 let runner = &mut *execution.runner;
1221 let wait = runner.repository_wait();
1222 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1225 let result = match result {
1226 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1227 record_repository_block(&mut runner.state, &e);
1228 runner.park_here()?;
1229 Ok(())
1230 }
1231 other => other,
1232 };
1233 execution.driver.finish(&mut runner.state)?;
1234 let ended = if result.is_err() {
1235 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1236 } else {
1237 crate::notices::run_ended(&runner.state)
1238 };
1239 if let Some(notice) = ended {
1240 crate::notices::raise(notice);
1241 }
1242 result
1243 }
1244
1245 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1246 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1247 }
1248
1249 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1250 let wait = self.repository_wait();
1251 coordinate_operation(&mut self.state, cwd, &wait).await
1252 }
1253
1254 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1256 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1257 let disk = RunState::load(&self.state.id)?;
1258 crate::pr_head::ensure_owner(&disk)?;
1259 if disk.liveness(false) == crate::run::Liveness::Live {
1260 bail!("run already has a live driver; wait for its execution to end");
1261 }
1262 if let Some(to) = &disk.released_to {
1263 bail!(
1264 "run {} cannot continue: its worktree was released to run {}",
1265 self.state.short(),
1266 crate::run::short_of(to)
1267 );
1268 }
1269 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1270 let mut execution = DriverExecution {
1271 runner: self,
1272 driver,
1273 };
1274 let _owner = owner;
1277 let state = &mut execution.runner.state;
1278 state.parked = false;
1283 state.resource_wait = None;
1284 state.resource_block = None;
1285 state.clear_active();
1292 let pid = std::process::id();
1307 state.driver_pid = Some(pid);
1308 state.driver_started_at = crate::proc::process_started_at(pid);
1309 state.driver_exited = false;
1310 state.begin_pass();
1311 execution.driver.bind(state);
1312 state.save()?;
1313 execution.driver.record(state)?;
1314 Ok(execution)
1315 }
1316
1317 async fn execute_graph(&mut self) -> Result<()> {
1318 self.persist_candidates().await?;
1321 if self
1322 .state
1323 .merge_decision
1324 .as_ref()
1325 .is_some_and(|decision| !decision.closed)
1326 && self.state.status != RunStatus::Landing
1327 {
1328 if crate::merge_decision::resumable_hold(&self.state) {
1329 self.state.merge = Some(MergeOutcome {
1330 mode: MergeMode::Pr,
1331 ok: false,
1332 detail: self
1333 .state
1334 .merge_decision
1335 .as_ref()
1336 .unwrap()
1337 .target
1338 .pr
1339 .clone(),
1340 empty: false,
1341 });
1342 self.run_land().await?;
1343 self.settle_questions();
1344 return Ok(());
1345 }
1346 bail!(
1347 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1348 );
1349 }
1350 if self.state.config.graph.selected_implementer.is_some()
1351 && self.state.status == RunStatus::Blocked
1352 && self.state.tally.is_none()
1353 && self.state.candidate_preservation.is_empty()
1354 {
1355 bail!(
1356 "selected implementation is held; explicitly release the owning task to retry it"
1357 );
1358 }
1359
1360 if self.state.merge_repairs.last().is_some_and(|r| {
1361 matches!(
1362 r.phase,
1363 crate::run::MergeRepairPhase::Requested
1364 | crate::run::MergeRepairPhase::Fixing
1365 | crate::run::MergeRepairPhase::Reviewing
1366 )
1367 }) {
1368 if self.repair_merge_review().await? {
1369 self.run_land().await?;
1370 }
1371 self.settle_questions();
1372 return Ok(());
1373 }
1374 if self.state.pr_publication_pending {
1377 self.merge().await?;
1378 if self.resource_halt()? {
1379 return Ok(());
1380 }
1381 self.state.save()?;
1382 self.settle_questions();
1383 return Ok(());
1384 }
1385 if self.state.status == RunStatus::Stalled {
1398 if self.recover_stall().await? {
1399 self.finish_after_tally().await?;
1400 } else {
1401 self.state.save()?;
1403 }
1404 return Ok(());
1405 }
1406 if self.state.status == RunStatus::Landing {
1416 self.run_land().await?;
1417 self.settle_questions();
1422 return Ok(());
1423 }
1424 if self.state.github_text.is_some()
1429 && self.state.merge.is_none()
1430 && self.state.status == RunStatus::Gating
1431 {
1432 self.merge().await?;
1433 if self.resource_halt()? {
1434 return Ok(());
1435 }
1436 self.state.save()?;
1437 return Ok(());
1438 }
1439 if let Err(e) = self.prep().await {
1440 if e.is::<crate::repo_lock::ResourceBlocked>() {
1441 return Err(e);
1442 }
1443 self.state.status = RunStatus::Blocked;
1444 self.state
1445 .prep_failure
1446 .get_or_insert_with(|| format!("{e:#}"));
1447 self.state
1448 .event("prep", format!("preparation stopped: {e:#}"));
1449 self.state.save()?;
1450 return Err(e);
1451 }
1452 if self.park_here()? {
1453 return Ok(());
1454 }
1455 self.advise().await?;
1456 if self.park_here()? {
1457 return Ok(());
1458 }
1459 self.implement().await?;
1460 if self.park_here()? {
1461 return Ok(());
1462 }
1463 if self.state.status == RunStatus::VerifiedNoop {
1467 return Ok(());
1468 }
1469 self.judge().await?;
1470 if self.park_here()? {
1471 return Ok(());
1472 }
1473 self.deliberate().await?;
1474 if self.park_here()? {
1475 return Ok(());
1476 }
1477 self.vote().await?;
1478 if self.park_here()? {
1479 return Ok(());
1480 }
1481 self.tally()?;
1482 if self.state.status == RunStatus::Stalled {
1487 self.state.save()?;
1491 return Ok(());
1492 }
1493 self.finish_after_tally().await?;
1494 Ok(())
1495 }
1496
1497 fn stop_for_resource(&mut self) -> Result<()> {
1502 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1504 {
1505 self.state.status = RunStatus::Blocked;
1506 }
1507 self.state.save()
1508 }
1509
1510 fn resource_halt(&mut self) -> Result<bool> {
1512 if self.state.resource_block.is_none() {
1513 return Ok(false);
1514 }
1515 self.stop_for_resource()?;
1516 Ok(true)
1517 }
1518
1519 fn park_here(&mut self) -> Result<bool> {
1526 let parking = self.pause.parked() || self.interrupt.parked();
1527 if !parking && self.state.resource_block.is_some() {
1528 self.stop_for_resource()?;
1529 return Ok(true);
1530 }
1531 if !self.pause.parked() && !self.interrupt.parked() {
1536 return Ok(false);
1537 }
1538 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1539 let why = match cause.as_ref().map(ParkCause::label) {
1540 Some(reason) => format!(
1541 "parked after `{}` ({reason}) — resume to carry on from here",
1542 self.state.status.as_str()
1543 ),
1544 None => format!(
1545 "parked after `{}` — resume to carry on from here",
1546 self.state.status.as_str()
1547 ),
1548 };
1549 self.state.event("park", why);
1550 self.state
1551 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1552 self.state.parked = true;
1553 self.state.save()?;
1554 Ok(true)
1555 }
1556
1557 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1559 if self.state.prep_failure.is_none() {
1560 return Ok(());
1561 }
1562 let failed_at = self
1563 .state
1564 .events
1565 .iter()
1566 .rev()
1567 .find(|event| event.node == "prep")
1568 .map_or(self.state.updated_at, |event| event.at);
1569 anyhow::ensure!(
1570 approval_at > failed_at,
1571 "preparation retry requires a new explicit release after the failure"
1572 );
1573 self.state.prep_failure = None;
1574 self.state.prep_complete = Some(false);
1575 self.state
1576 .event("prep-retry", "operator approved retry of preparation");
1577 Ok(())
1578 }
1579
1580 pub fn on_pause(&mut self, pause: Pause) {
1582 self.pause = pause;
1583 }
1584
1585 pub fn watch_interrupt(&mut self, pause: Pause) {
1591 self.interrupt = pause;
1592 }
1593
1594 fn settle_questions(&mut self) {
1614 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1615 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1616 }
1617 }
1618
1619 async fn finish_after_tally(&mut self) -> Result<()> {
1622 self.graft().await?;
1625 if self.resource_halt()? {
1626 return Ok(());
1627 }
1628 self.fold_losers().await?;
1629 self.sync_to_base().await?;
1634 if self.state.status == RunStatus::AlreadyInBase {
1635 return Ok(());
1636 }
1637 self.review_loop().await?;
1638 if self.resource_halt()? {
1639 return Ok(());
1640 }
1641 self.sync_to_base().await?;
1642 if self.state.status == RunStatus::AlreadyInBase {
1643 return Ok(());
1644 }
1645 self.gate().await?;
1646 self.merge().await?;
1647 if self.resource_halt()? {
1648 return Ok(());
1649 }
1650 self.state.save()?;
1651 Ok(())
1652 }
1653
1654 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1661 let result = match ensure_setup_config(&mut self.state, repo).await {
1662 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1663 Err(e) => Err(e),
1664 };
1665 if let Err(e) = result {
1666 git::worktree_remove_clean(repo, wt).await.ok();
1667 if in_prep {
1668 self.rollback_prep(repo).await;
1669 }
1670 self.state.status = RunStatus::Blocked;
1671 self.state.event(
1672 "setup",
1673 format!("worktree setup failed in {}: {e:#}", wt.display()),
1674 );
1675 self.state.save()?;
1676 return Err(e);
1677 }
1678 Ok(())
1679 }
1680
1681 async fn rollback_prep(&mut self, repo: &Path) {
1683 let root = self.state.worktree_root();
1684 for c in std::mem::take(&mut self.state.candidates) {
1685 if matches!(
1686 git::worktree_remove_clean(repo, &c.worktree).await,
1687 Ok(true)
1688 ) {
1689 git::branch_delete(repo, &c.branch).await.ok();
1690 } else {
1691 self.state.candidates.push(c);
1692 }
1693 }
1694 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1695 for j in 1..=seats {
1696 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1697 .await
1698 .ok();
1699 }
1700 for k in 1..=self.state.config.graph.advisors {
1701 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1702 .await
1703 .ok();
1704 }
1705 }
1706
1707 async fn prep(&mut self) -> Result<()> {
1708 if let Some(reason) = &self.state.prep_failure {
1709 bail!(
1710 "candidate preparation previously failed: {reason}; correct the source before retrying"
1711 );
1712 }
1713 if self
1714 .state
1715 .prep_complete
1716 .unwrap_or(!self.state.candidates.is_empty())
1717 {
1718 return Ok(());
1719 }
1720 let repo = self.state.repo.clone();
1721 let _coordination = self.coordinate(&repo).await?;
1724 self.rollback_prep(&repo).await;
1725 if !self.state.candidates.is_empty() {
1726 bail!(
1727 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1728 );
1729 }
1730 self.state.prep_complete = Some(false);
1731 self.state.status = RunStatus::Prep;
1732 self.state.save()?;
1733 let repo = self.state.repo.clone();
1734 let base = self.state.base_commit.clone();
1735 let plan = refs::plan(&repo, &self.state.seeds).await?;
1736 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1737 let root = self.state.worktree_root();
1738 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1739
1740 let hooks_dir = self.state.dir().join("hooks");
1743 if self.state.config.blind.commit_msg_hook {
1744 std::fs::create_dir_all(&hooks_dir)
1745 .with_context(|| format!("create {}", hooks_dir.display()))?;
1746 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1747 let path = hooks_dir.join("commit-msg");
1748 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1749 make_executable(&path)?;
1750 git::acquire_worktree_config(&repo).await?;
1758 self.state.enabled_worktree_config = true;
1759 }
1760
1761 ensure_setup_config(&mut self.state, &repo).await?;
1764
1765 for (index, (spec, label)) in self
1766 .roles
1767 .implementers
1768 .clone()
1769 .into_iter()
1770 .zip(labels)
1771 .enumerate()
1772 {
1773 let branch = self.state.branch_for(label);
1774 let worktree = root.join(format!("cand-{label}"));
1775 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1776 if self.state.config.blind.commit_msg_hook {
1777 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1778 }
1779 git::local_exclude(&worktree, "/.magi/").await?;
1780 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1781 git::worktree_remove_clean(&repo, &worktree).await.ok();
1784 git::branch_delete(&repo, &branch).await.ok();
1785 self.rollback_prep(&repo).await;
1786 self.state.status = RunStatus::Blocked;
1787 self.state
1788 .event("prep", format!("worktree setup failed: {e:#}"));
1789 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1790 self.state.save()?;
1791 return Err(e);
1792 }
1793 self.state.candidates.push(Candidate {
1794 index,
1795 label,
1796 agent: spec.id.clone(),
1797 branch,
1798 worktree,
1799 summary: String::new(),
1800 stat: String::new(),
1801 files: 0,
1802 commits: 0,
1803 empty: false,
1804 failed: None,
1805 verified_noop: None,
1806 duration_ms: 0,
1807 folded: false,
1808 });
1809 self.state.save()?;
1810 for pick in &plan.picks {
1811 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1812 {
1813 self.state.status = RunStatus::Blocked;
1814 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1815 self.state
1816 .event("prep", format!("cannot apply referenced commit: {e}"));
1817 self.state.save()?;
1818 return Err(e);
1819 }
1820 }
1821 }
1822
1823 for j in 1..=self.roles.judges.len() {
1824 let wt = root.join(format!("judge-{j}"));
1825 if !wt.exists() {
1826 git::worktree_add_detached(&repo, &wt, &base).await?;
1827 self.setup_seat_worktree(&repo, &wt, true).await?;
1828 }
1829 }
1830
1831 if self.state.config.graph.advise {
1839 for k in 1..=self.state.config.graph.advisors {
1840 let wt = root.join(format!("advisor-{k}"));
1841 if !wt.exists() {
1842 git::worktree_add_detached(&repo, &wt, &base).await?;
1843 self.setup_seat_worktree(&repo, &wt, true).await?;
1844 }
1845 }
1846 }
1847
1848 let authors: Vec<&str> = self
1853 .roles
1854 .implementers
1855 .iter()
1856 .map(|a| a.id.as_str())
1857 .collect();
1858 let overlap: Vec<String> = self
1859 .roles
1860 .judges
1861 .iter()
1862 .enumerate()
1863 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1864 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1865 .collect();
1866 if !overlap.is_empty() {
1867 let note = format!(
1868 "{} also authored a candidate; blind, but the panel is less \
1869 independent than {} distinct agents would be",
1870 overlap.join(", "),
1871 self.roles.judges.len()
1872 );
1873 self.state.event("prep", note);
1874 }
1875
1876 self.state.event(
1877 "prep",
1878 format!(
1879 "{} candidates, {} judges, base {} ({})",
1880 self.state.candidates.len(),
1881 self.roles.judges.len(),
1882 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1883 self.state.base_branch
1884 ),
1885 );
1886 self.state.prep_complete = Some(true);
1887 self.state.status = RunStatus::Implementing;
1888 self.state.save()?;
1889 Ok(())
1890 }
1891
1892 async fn advise(&mut self) -> Result<()> {
1925 let implement_untouched = self
1926 .state
1927 .candidates
1928 .iter()
1929 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1930 if !self.state.config.graph.advise || self.state.advise_attempted {
1931 return Ok(());
1932 }
1933 if !implement_untouched {
1934 self.state.event(
1935 "advise",
1936 "skipping the design-deliberation stage: at least one \
1937 candidate already shows implementation progress, so this \
1938 run is past the point the stage exists to run before"
1939 .to_owned(),
1940 );
1941 self.state.advise_attempted = true;
1942 self.state.save()?;
1943 return Ok(());
1944 }
1945 let run_id = self.state.id.clone();
1946 let prompts = self.state.config.prompts.clone();
1947 let instruction = self.state.instruction.clone();
1948 let language = self.state.config.graph.language.clone();
1949 let root = self.state.worktree_root();
1950 let n = self.state.config.graph.advisors;
1951 let where_recorded = self.state.dir().join("run.json");
1952
1953 let seats = match self.state.config.advisors() {
1954 Ok(seats) if !seats.is_empty() => seats,
1955 Ok(_) => {
1956 self.state.event(
1957 "advise",
1958 format!(
1959 "[graph] advisors is 0; skipping the design-deliberation \
1960 stage and continuing without a synthesis brief (see {})",
1961 where_recorded.display()
1962 ),
1963 );
1964 self.state.advise_attempted = true;
1965 self.state.save()?;
1966 return Ok(());
1967 }
1968 Err(e) => {
1969 self.state.event(
1970 "advise",
1971 format!(
1972 "could not resolve advisor seats ({e:#}); continuing \
1973 without a design-deliberation brief (see {})",
1974 where_recorded.display()
1975 ),
1976 );
1977 self.state.advise_attempted = true;
1978 self.state.save()?;
1979 return Ok(());
1980 }
1981 };
1982
1983 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1984 let artifacts = agent::artifacts_dir(&self.state.dir());
1985 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1986
1987 let mut jobs = Vec::new();
1988 for (i, spec) in seats.iter().cloned().enumerate() {
1989 let seat_key = format!("advisor-{}", i + 1);
1990 let seat = self.seat(&seat_key, &spec.id);
1991 jobs.push(SeatJob {
1992 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1993 spec,
1994 seat,
1995 cwd: worktrees[i % worktrees.len()].clone(),
1996 timeout,
1997 allow_write: false,
1998 sessions: false,
1999 artifacts: artifacts.clone(),
2000 stem: seat_key,
2001 handover: None,
2002 });
2003 }
2004
2005 self.state.event(
2006 "advise",
2007 format!(
2008 "{} advisor seat(s) sketching a design in parallel",
2009 jobs.len()
2010 ),
2011 );
2012 let mut quota_losses = Vec::new();
2013 let cache = self.state.config.cache_dir();
2014 let ctx = WaveCtx {
2015 carry_seats: false,
2016 pauses: Some((&self.pause, &self.interrupt)),
2017 run: &run_id,
2018 node: "advise",
2019 prompts: &prompts,
2020 cache: cache.as_deref(),
2021 round: None,
2022 };
2023 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2024 let results = ask_json_wave::<Proposal>(
2025 jobs,
2026 Arc::clone(&self.sem),
2027 self.state.config.graph.retries,
2028 &advisor_roster,
2029 &ctx,
2030 &mut quota_losses,
2031 &mut self.state,
2032 &|p: &Proposal| p.validate(),
2033 )
2034 .await;
2035 if self.state.resource_block.is_some() {
2036 return self.stop_for_resource();
2037 }
2038 self.state.quota.extend(quota_losses);
2039
2040 let mut records = Vec::with_capacity(results.len());
2041 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2042 let agent_id = seat.agent.clone();
2043 self.state.seats.insert(seat.key.clone(), seat);
2044 match res {
2045 Ok((proposal, out)) => {
2046 self.state
2047 .event("advise", format!("advisor-{} proposed a design", i + 1));
2048 records.push(advise::AdvisorRecord::proposed(
2049 i + 1,
2050 agent_id,
2051 proposal,
2052 out.duration_ms,
2053 ));
2054 }
2055 Err(e) => {
2056 self.state.event(
2057 "advise",
2058 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2059 );
2060 records.push(advise::AdvisorRecord::failed(
2061 i + 1,
2062 agent_id,
2063 e.to_string(),
2064 ));
2065 }
2066 }
2067 }
2068
2069 let mut advice = advise::Advice {
2070 records,
2071 synthesis: None,
2072 };
2073 if advice.proposals().is_empty() {
2074 self.state.event(
2075 "advise",
2076 "no advisor produced a usable proposal; continuing without a \
2077 synthesis brief"
2078 .to_owned(),
2079 );
2080 } else {
2081 match self
2082 .synthesize_brief(
2083 &advice,
2084 &instruction,
2085 &language,
2086 &worktrees[0],
2087 &artifacts,
2088 &run_id,
2089 &prompts,
2090 cache.as_deref(),
2091 )
2092 .await
2093 {
2094 Ok(Some(text)) => {
2095 self.state.event(
2096 "advise",
2097 "synthesized a design brief for the implementer".to_owned(),
2098 );
2099 advice.synthesis = Some(text);
2100 }
2101 Ok(None) => {
2102 self.state.event(
2103 "advise",
2104 "the synthesis seat produced nothing usable; continuing \
2105 without a design brief"
2106 .to_owned(),
2107 );
2108 }
2109 Err(e) => {
2110 if e.is::<crate::repo_lock::ResourceBlocked>() {
2111 return Err(e);
2112 }
2113 self.state.event(
2114 "advise",
2115 format!("could not synthesize a design brief: {e:#}"),
2116 );
2117 }
2118 }
2119 }
2120 advise::apply_reflection(&mut advice);
2121
2122 self.state.advice = Some(advice);
2123 self.state.advise_attempted = true;
2124 self.state.save()?;
2125 Ok(())
2126 }
2127
2128 #[allow(clippy::too_many_arguments)]
2140 async fn synthesize_brief(
2141 &mut self,
2142 advice: &advise::Advice,
2143 instruction: &str,
2144 language: &str,
2145 cwd: &Path,
2146 artifacts: &Path,
2147 run_id: &str,
2148 prompts: &Prompts,
2149 cache: Option<&Path>,
2150 ) -> Result<Option<String>> {
2151 let chain = agent::pick_chain(
2152 &self.state.config.agents,
2153 self.state.config.roles.synthesizer.as_ref(),
2154 &agent::installed,
2155 "synthesizer",
2156 )?;
2157 let proposals = advice.proposals();
2158 let mut prompt = prompt::with_overlay(
2159 prompt::synthesize_brief(instruction, &proposals, language),
2160 prompts.overlay("advise"),
2161 );
2162 if cache.is_some() {
2163 prompt.push('\n');
2168 prompt.push_str(&prompt::build_cache_note("advise", false));
2169 }
2170 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2171 let mut last = None;
2176 for (n, spec) in chain.iter().enumerate() {
2177 if n > 0 {
2178 self.state
2179 .event("advise", format!("synthesis falling back to {}", spec.id));
2180 }
2181 let mut seat = self.seat("advise-synthesis", &spec.id);
2182 let wait = self.repository_wait();
2183 let outcome = wait
2184 .scope(agent::invoke(
2185 spec,
2186 &mut seat,
2187 &Invocation {
2188 scope: crate::agent::InvocationScope::Repository,
2189 coordination: self.state.config.coordination.as_ref(),
2190 fetched_base: None,
2191 cwd,
2192 prompt: &prompt,
2193 timeout,
2194 allow_write: false,
2195 unsandboxed: false,
2196 sessions: false,
2197 artifacts,
2198 stem: &if n == 0 {
2199 "advise-synthesis".to_owned()
2200 } else {
2201 format!("advise-synthesis-{}", spec.id)
2202 },
2203 run: run_id,
2204 node: "advise",
2205 cache_dir: None,
2206 attachments: &[],
2207 writable: &[],
2208 },
2209 ))
2210 .await;
2211 if outcome.is_ok() {
2212 self.state.seats.insert(seat.key.clone(), seat);
2213 }
2214 let advance = agent::chain_advances(&outcome);
2215 last = Some(outcome);
2216 if !advance {
2217 break;
2218 }
2219 }
2220 let out = last.expect("a chain holds at least one agent")?;
2223 if !out.usable() {
2224 return Ok(None);
2225 }
2226 let text =
2227 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2228 Ok((!text.trim().is_empty()).then_some(text))
2229 }
2230
2231 async fn implement(&mut self) -> Result<()> {
2234 self.persist_candidates().await?;
2235 let run_id = self.state.id.clone();
2240 let prompts = self.state.config.prompts.clone();
2241 let todo: Vec<usize> = self
2242 .state
2243 .candidates
2244 .iter()
2245 .enumerate()
2246 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2247 .map(|(i, _)| i)
2248 .collect();
2249 if todo.is_empty() {
2250 return self.after_implement();
2251 }
2252 self.state.status = RunStatus::Implementing;
2253
2254 let language = self.state.config.graph.language.clone();
2255 let github_language = self.state.config.graph.github_language.clone();
2256 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2257 let sessions = self.state.config.graph.sessions;
2258 let artifacts = agent::artifacts_dir(&self.state.dir());
2259 let brief = self
2263 .state
2264 .advice
2265 .as_ref()
2266 .and_then(|a| a.synthesis.as_deref())
2267 .map(str::to_owned);
2268 let attachments = self.state.attachments.clone();
2269
2270 let mut jobs = Vec::new();
2271 for &i in &todo {
2272 let (index, label, worktree) = {
2273 let c = &self.state.candidates[i];
2274 (c.index, c.label, c.worktree.clone())
2275 };
2276 let spec = self.roles.implementers[index].clone();
2277 let seat_key = format!("impl-{label}");
2278 let seat = self.seat(&seat_key, &spec.id);
2279 let instruction = seeded_instruction(&self.state);
2280 jobs.push(SeatJob {
2281 spec,
2282 seat,
2283 prompt: prompt::implement_in(
2284 &instruction,
2285 &worktree.to_string_lossy(),
2286 &language,
2287 &github_language,
2288 brief.as_deref(),
2289 &attachments,
2290 ),
2291 cwd: worktree,
2292 timeout,
2293 allow_write: true,
2294 sessions,
2295 artifacts: artifacts.clone(),
2296 stem: format!("impl-{label}"),
2297 handover: None,
2298 });
2299 }
2300
2301 self.state.event(
2302 "implement",
2303 format!("{} candidates in parallel", jobs.len()),
2304 );
2305 let mut sent = jobs.clone();
2311 let cache = self.state.config.cache_dir();
2312 let ctx = WaveCtx {
2313 carry_seats: false,
2314 pauses: Some((&self.pause, &self.interrupt)),
2315 run: &run_id,
2316 node: "implement",
2317 prompts: &prompts,
2318 cache: cache.as_deref(),
2319 round: None,
2320 };
2321 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2322 if !wave_blocked(&results) {
2323 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2324 .await;
2325 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2326 .await?;
2327 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2328 .await;
2329 }
2330 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2331
2332 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2333 && results
2334 .iter()
2335 .any(|(_, _, out)| FailClass::of(out).is_some());
2336 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2337 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2338 continue;
2339 }
2340 let seat_key = seat.key.clone();
2341 let agent = seat.agent.clone();
2350 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2351 self.state.seats.insert(seat.key.clone(), seat);
2352 let label = self.state.candidates[i].label;
2353
2354 let (summary, duration, failed, verified_claim) = match out {
2355 AgentOutcome::Ok(o) => {
2356 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2357 let failed = (!o.usable()).then(|| {
2358 if o.timed_out {
2359 "agent timed out".to_owned()
2360 } else {
2361 format!(
2362 "agent exited with {}",
2363 crate::notices::exit_status(o.exit_code)
2364 )
2365 }
2366 });
2367 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2368 (text, o.duration_ms, failed, verified_claim)
2369 }
2370 AgentOutcome::Dropped(o) => {
2376 let why = o
2377 .dropped
2378 .as_ref()
2379 .map(|d| d.why.as_str())
2380 .unwrap_or("the CLI ended the stream without delivering its answer");
2381 (
2382 String::new(),
2383 o.duration_ms,
2384 Some(format!("the CLI dropped the stream ({why})")),
2385 None,
2386 )
2387 }
2388 AgentOutcome::Quota(o) => {
2389 self.state.quota.push(QuotaLoss {
2390 seat: seat_key,
2391 node: "implement".to_owned(),
2392 at: Timestamp::now(),
2393 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2394 });
2395 (
2396 String::new(),
2397 o.duration_ms,
2398 Some("rate limited (quota); produced no change".to_owned()),
2399 None,
2400 )
2401 }
2402 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2403 (String::new(), 0, Some(e), None)
2404 }
2405 };
2406
2407 let c = &mut self.state.candidates[i];
2410 if !exhausted_the_fallback_chain {
2411 c.agent = agent;
2412 }
2413 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2414 c.duration_ms = duration;
2415 c.failed = failed;
2416 c.verified_noop = verified_claim;
2417 self.state
2418 .candidate_preservation
2419 .insert(label, "awaiting persistence".to_owned());
2420 }
2421 self.state.save()?;
2422 self.persist_candidates().await?;
2423
2424 if resource_blocked {
2425 return self.stop_for_resource();
2426 }
2427 if selected_failed {
2428 self.state.status = RunStatus::Blocked;
2429 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2430 self.state.save()?;
2431 bail!(
2432 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2433 );
2434 }
2435 self.after_implement()
2436 }
2437
2438 async fn persist_candidates(&mut self) -> Result<()> {
2440 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2441 for label in labels {
2442 let i = self
2443 .state
2444 .candidates
2445 .iter()
2446 .position(|c| c.label == label)
2447 .context("pending candidate is missing")?;
2448 let c = self.state.candidates[i].clone();
2449 let result = async {
2450 let wait = crate::repo_lock::WaitPolicy::new(
2451 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2452 || false,
2453 );
2454 let _coordination =
2455 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2456 if let Err(e) = git::snapshot_uncommitted(
2459 &c.worktree,
2460 &self
2461 .state
2462 .dir()
2463 .join("artifacts")
2464 .join(format!("candidate-{label}-recovery")),
2465 )
2466 .await
2467 {
2468 self.state.event(
2469 "preserve",
2470 format!("candidate {label} snapshot incomplete: {e:#}"),
2471 );
2472 }
2473 let rescued = git::rescue_commit(
2474 &c.worktree,
2475 &format!("magi: candidate {label} (uncommitted work)"),
2476 )
2477 .await?;
2478 self.state.note_withheld("implement", &rescued.withheld);
2479 let base = &self.state.base_commit;
2480 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2481 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2482 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2483 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2484 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2485 if c.empty && commits > 0 && !patch.trim().is_empty()
2486 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2487 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2488 || self.state.tally.is_some())
2489 {
2490 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2494 }
2495 let c = &mut self.state.candidates[i];
2496 c.commits = commits;
2497 c.stat = stat;
2498 c.files = files;
2499 c.empty = commits == 0 || patch.trim().is_empty();
2500 if !c.empty {
2501 c.failed = None;
2502 c.verified_noop = None;
2503 }
2504 self.state.event(
2505 "preserve",
2506 format!("candidate {label}: {files} files, {commits} commits"),
2507 );
2508 Ok::<_, anyhow::Error>(())
2509 }
2510 .await;
2511 match result {
2512 Ok(()) => {
2513 self.state.candidate_preservation.remove(&label);
2514 }
2515 Err(e) => {
2516 let reason = format!("candidate {label} preservation failed: {e:#}");
2517 self.state.event("preserve", &reason);
2518 self.state.candidate_preservation.insert(label, reason);
2519 }
2520 }
2521 if self.state.candidate_preservation.is_empty()
2522 && self.state.status == RunStatus::Blocked
2523 && self.state.candidates.iter().all(|c| c.failed.is_none())
2524 {
2525 self.state.status = RunStatus::Implementing;
2529 self.state.event(
2530 "preserve",
2531 "candidate persistence recovered; implementation can continue",
2532 );
2533 }
2534 self.state.save()?;
2535 }
2536 if !self.state.candidate_preservation.is_empty() {
2537 self.state.status = RunStatus::Blocked;
2538 self.state.save()?;
2539 bail!(
2540 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2541 );
2542 }
2543 Ok(())
2544 }
2545
2546 async fn resume_undelivered(
2574 &mut self,
2575 results: &mut [(usize, SeatState, AgentOutcome)],
2576 sent: &[SeatJob],
2577 prompts: &Prompts,
2578 run_id: &str,
2579 ) {
2580 for (wi, seat, out) in results.iter_mut() {
2581 let Some(dropped) = (match &*out {
2582 AgentOutcome::Dropped(o) => o.dropped.clone(),
2583 _ => None,
2584 }) else {
2585 continue;
2586 };
2587 let Some(job) = sent.get(*wi) else { continue };
2588 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2590 self.state.event(
2591 "implement",
2592 format!(
2593 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2594 work is in the tree",
2595 seat.key, dropped.output_tokens, dropped.why
2596 ),
2597 );
2598 continue;
2599 }
2600 if !has_context(&job.spec, seat, job.sessions) {
2608 self.state.event(
2609 "implement",
2610 format!(
2611 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2612 is no session left to resume",
2613 seat.key, dropped.output_tokens, dropped.why
2614 ),
2615 );
2616 continue;
2617 }
2618 self.state.event(
2619 "implement",
2620 format!(
2621 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2622 conversation",
2623 seat.key, dropped.output_tokens, dropped.why
2624 ),
2625 );
2626 let mut retry = job.clone();
2627 retry.seat = seat.clone();
2628 retry.prompt = prompt::resume_after_drop(&dropped.why);
2629 retry.timeout = retry_budget(job.timeout, true);
2630 retry.stem = format!("{}-resume", job.stem);
2631 let cache = self.state.config.cache_dir();
2632 let ctx = WaveCtx {
2633 carry_seats: false,
2634 pauses: Some((&self.pause, &self.interrupt)),
2635 run: run_id,
2636 node: "implement",
2637 prompts,
2638 cache: cache.as_deref(),
2639 round: None,
2640 };
2641 let (resumed_seat, resumed) =
2642 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2643 *seat = resumed_seat;
2644 *out = resumed;
2645 }
2646 }
2647
2648 async fn resume_seat_handovers(
2713 &mut self,
2714 results: &mut [(usize, SeatState, AgentOutcome)],
2715 sent: &mut [SeatJob],
2716 prompts: &Prompts,
2717 run_id: &str,
2718 ) -> Result<()> {
2719 let instruction = seeded_instruction(&self.state);
2720 let language = self.state.config.graph.language.clone();
2721 let github_language = self.state.config.graph.github_language.clone();
2722 let brief = self
2723 .state
2724 .advice
2725 .as_ref()
2726 .and_then(|a| a.synthesis.as_deref())
2727 .map(str::to_owned);
2728 let attachments = self.state.attachments.clone();
2729 for (wi, seat, out) in results.iter_mut() {
2730 let others: BTreeSet<String> = sent
2733 .iter()
2734 .enumerate()
2735 .filter(|(j, _)| j != wi)
2736 .map(|(_, j)| j.spec.id.clone())
2737 .collect();
2738 let Some(job) = sent.get_mut(*wi) else {
2739 continue;
2740 };
2741 let start = self
2746 .roles
2747 .implementer_roster
2748 .iter()
2749 .position(|s| s.id == job.spec.id)
2750 .unwrap_or(0);
2751 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2752 let mut fallback_attempt = 0usize;
2753 let mut prev: Option<FailClass> = None;
2754 while let Some(cur) = FailClass::of(&*out) {
2755 if !should_hand_over(prev.as_ref(), &cur) {
2756 break;
2757 }
2758 let Some(next) =
2759 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2760 .cloned()
2761 else {
2762 break;
2763 };
2764 tried.insert(next.id.clone());
2765 fallback_attempt += 1;
2766
2767 let coordination = self.coordinate(&job.cwd).await?;
2768 if let Ok(r) = git::rescue_commit(
2769 &job.cwd,
2770 &format!(
2771 "magi: candidate {} (uncommitted work before {} fallback)",
2772 seat.key,
2773 if cur == FailClass::Quota {
2774 "quota"
2775 } else {
2776 "handover"
2777 }
2778 ),
2779 )
2780 .await
2781 {
2782 self.state.note_withheld("implement", &r.withheld);
2783 }
2784 drop(coordination);
2785
2786 record_handover(
2787 &mut self.state,
2788 "implement",
2789 &seat.key,
2790 &seat.agent,
2791 &next.id,
2792 &cur,
2793 &fail_reason(&*out),
2794 );
2795 prev = Some(cur.clone());
2796
2797 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2798 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2799 job.spec = next.clone();
2807 let mut retry = job.clone();
2808 retry.seat = new_seat;
2809 retry.prompt = prompt::implement_in(
2810 &instruction,
2811 &job.cwd.to_string_lossy(),
2812 &language,
2813 &github_language,
2814 brief.as_deref(),
2815 &attachments,
2816 );
2817 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2818 let cache = self.state.config.cache_dir();
2819 let ctx = WaveCtx {
2820 carry_seats: false,
2821 pauses: Some((&self.pause, &self.interrupt)),
2822 run: run_id,
2823 node: "implement",
2824 prompts,
2825 cache: cache.as_deref(),
2826 round: None,
2827 };
2828 let (fallback_seat, fallback_out) = run_one(
2829 retry,
2830 Arc::clone(&self.sem),
2831 &ctx,
2832 &mut self.state,
2833 fallback_attempt,
2834 )
2835 .await;
2836 *seat = fallback_seat;
2837 *out = fallback_out;
2838 }
2839 }
2840 Ok(())
2841 }
2842
2843 async fn resume_unconfirmed_commands(
2867 &mut self,
2868 results: &mut [(usize, SeatState, AgentOutcome)],
2869 sent: &[SeatJob],
2870 prompts: &Prompts,
2871 run_id: &str,
2872 ) {
2873 for (wi, seat, out) in results.iter_mut() {
2874 let AgentOutcome::Ok(o) = &*out else {
2875 continue;
2876 };
2877 if !has_unconfirmed_command(&o.commands) {
2878 continue;
2879 }
2880 let Some(job) = sent.get(*wi) else { continue };
2881 if !has_context(&job.spec, seat, job.sessions) {
2882 self.state.event(
2883 "implement",
2884 format!(
2885 "{}: the reply named a command whose own CLI never confirmed the exit \
2886 status of, but there is no session left to resume",
2887 seat.key
2888 ),
2889 );
2890 continue;
2891 }
2892 self.state.event(
2893 "implement",
2894 format!(
2895 "{}: the reply named a command whose own CLI never confirmed the exit \
2896 status of; resuming the conversation",
2897 seat.key
2898 ),
2899 );
2900 let mut retry = job.clone();
2901 retry.seat = seat.clone();
2902 retry.prompt = prompt::resume_incomplete(
2903 "a command in your last reply had no confirmed exit status",
2904 );
2905 retry.timeout = retry_budget(job.timeout, true);
2906 retry.stem = format!("{}-confirm", job.stem);
2907 let cache = self.state.config.cache_dir();
2908 let ctx = WaveCtx {
2909 carry_seats: false,
2910 pauses: Some((&self.pause, &self.interrupt)),
2911 run: run_id,
2912 node: "implement",
2913 prompts,
2914 cache: cache.as_deref(),
2915 round: None,
2916 };
2917 let (resumed_seat, resumed) =
2918 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2919 *seat = resumed_seat;
2920 *out = resumed;
2921 }
2922 }
2923
2924 async fn continue_fix_report(
2945 &mut self,
2946 mut seat: SeatState,
2947 parse_err: String,
2948 job: &SeatJob,
2949 prompts: &Prompts,
2950 run_id: &str,
2951 round: usize,
2952 ) -> (
2953 SeatState,
2954 Option<FixReport>,
2955 Option<String>,
2956 ContinuationRecord,
2957 ) {
2958 let mut last_err = parse_err;
2959 let mut cumulative_wait_ms = 0u64;
2960 let mut attempts = 0usize;
2961 loop {
2962 if !has_context(&job.spec, &seat, job.sessions) {
2963 self.state.event(
2964 "fix",
2965 format!(
2966 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2967 session left to resume into"
2968 ),
2969 );
2970 let outcome = if attempts == 0 {
2971 ContinuationOutcome::NoSession
2972 } else {
2973 ContinuationOutcome::Exhausted
2974 };
2975 return (
2976 seat,
2977 None,
2978 Some(format!("unparsable fix report: {last_err}")),
2979 ContinuationRecord {
2980 attempts,
2981 cumulative_wait_ms,
2982 outcome,
2983 },
2984 );
2985 }
2986 if attempts >= MAX_FIX_CONTINUATIONS {
2987 self.state.event(
2988 "fix",
2989 format!(
2990 "round {round}: fixer's reply still had no adoption report after \
2991 {attempts} continuation(s) ({last_err}); giving up"
2992 ),
2993 );
2994 return (
2995 seat,
2996 None,
2997 Some(format!(
2998 "unparsable fix report after {attempts} continuation(s): {last_err}"
2999 )),
3000 ContinuationRecord {
3001 attempts,
3002 cumulative_wait_ms,
3003 outcome: ContinuationOutcome::Exhausted,
3004 },
3005 );
3006 }
3007 attempts += 1;
3008 self.state.event(
3009 "fix",
3010 format!(
3011 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3012 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3013 ),
3014 );
3015 let mut retry = job.clone();
3016 retry.seat = seat.clone();
3017 retry.prompt = prompt::resume_incomplete(&last_err);
3018 retry.timeout = retry_budget(job.timeout, true);
3019 retry.stem = format!("{}-continue{attempts}", job.stem);
3020 let cache = self.state.config.cache_dir();
3021 let ctx = WaveCtx {
3022 carry_seats: false,
3023 pauses: Some((&self.pause, &self.interrupt)),
3024 run: run_id,
3025 node: "fix",
3026 prompts,
3027 cache: cache.as_deref(),
3028 round: Some(round),
3029 };
3030 let (resumed_seat, resumed_out) = run_one(
3031 retry,
3032 Arc::clone(&self.sem),
3033 &ctx,
3034 &mut self.state,
3035 attempts,
3036 )
3037 .await;
3038 seat = resumed_seat;
3039 match resumed_out {
3040 AgentOutcome::Ok(o) => {
3041 cumulative_wait_ms += o.duration_ms;
3042 match verdict::extract_json::<FixReport>(&o.text) {
3043 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3044 self.state.event(
3045 "fix",
3046 format!(
3047 "round {round}: fixer's adoption report recovered after \
3048 {attempts} continuation(s)"
3049 ),
3050 );
3051 return (
3052 seat,
3053 Some(report),
3054 None,
3055 ContinuationRecord {
3056 attempts,
3057 cumulative_wait_ms,
3058 outcome: ContinuationOutcome::Resumed,
3059 },
3060 );
3061 }
3062 Ok(_) => {
3070 last_err = "the reply parsed, but it reported a command whose own CLI \
3071 never confirmed an exit status"
3072 .to_owned();
3073 }
3074 Err(e) => last_err = e.to_string(),
3075 }
3076 }
3077 AgentOutcome::Quota(o) => {
3078 cumulative_wait_ms += o.duration_ms;
3079 self.state.quota.push(QuotaLoss {
3080 seat: seat.key.clone(),
3081 node: "fix".to_owned(),
3082 at: Timestamp::now(),
3083 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3084 });
3085 self.state.event(
3086 "fix",
3087 format!(
3088 "round {round}: continuation rate limited (quota); not retrying now"
3089 ),
3090 );
3091 return (
3092 seat,
3093 None,
3094 Some("rate limited (quota) while recovering the fix report".to_owned()),
3095 ContinuationRecord {
3096 attempts,
3097 cumulative_wait_ms,
3098 outcome: ContinuationOutcome::QuotaLost,
3099 },
3100 );
3101 }
3102 AgentOutcome::Dropped(o) => {
3103 cumulative_wait_ms += o.duration_ms;
3104 let why = o
3105 .dropped
3106 .as_ref()
3107 .map(|d| d.why.as_str())
3108 .unwrap_or("the CLI ended the stream without delivering its answer");
3109 last_err = format!("the CLI dropped the stream ({why})");
3110 }
3111 AgentOutcome::Failed(e) => last_err = e,
3112 AgentOutcome::ResourceBlocked(e) => {
3115 return (
3116 seat,
3117 None,
3118 Some(format!("the continuation was not started: {e}")),
3119 ContinuationRecord {
3120 attempts: attempts.saturating_sub(1),
3121 cumulative_wait_ms,
3122 outcome: ContinuationOutcome::NoSession,
3123 },
3124 );
3125 }
3126 }
3127 }
3128 }
3129
3130 fn after_implement(&mut self) -> Result<()> {
3131 if !self.state.candidate_preservation.is_empty() {
3132 bail!("candidate preservation incomplete; judging is unsafe");
3133 }
3134 if self.state.leaks.is_empty() {
3136 let cfg = self.state.config.blind.clone();
3137 let mut leaks = Vec::new();
3138 for c in &self.state.candidates {
3139 let Some(patch) =
3140 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3141 else {
3142 continue;
3143 };
3144 leaks.extend(blind::scan(
3145 &format!("candidate {} patch", c.label),
3146 &patch,
3147 &cfg.vendor_tokens,
3148 ));
3149 }
3150 if !leaks.is_empty() {
3151 let summary = leaks
3152 .iter()
3153 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3154 .collect::<Vec<_>>()
3155 .join(", ");
3156 match cfg.on_leak {
3157 LeakPolicy::Fail => {
3158 self.state.status = RunStatus::Failed;
3159 self.state
3160 .event("blind", format!("vendor text in a patch: {summary}"));
3161 self.state.leaks = leaks;
3162 self.state.save()?;
3163 self.settle_questions();
3164 bail!(
3165 "blind.on_leak = \"fail\" and vendor text reached a \
3166 judged patch: {summary}"
3167 );
3168 }
3169 LeakPolicy::Redact => self.state.event(
3170 "blind",
3171 format!("redacting vendor text for judging: {summary}"),
3172 ),
3173 LeakPolicy::Warn => self.state.event(
3174 "blind",
3175 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3176 ),
3177 }
3178 self.state.leaks = leaks;
3179 }
3180 }
3181
3182 if self.state.viable().is_empty() {
3183 if self.state.all_candidates_verified_noop() {
3184 self.state.status = RunStatus::VerifiedNoop;
3195 self.state.save()?;
3196 self.settle_questions();
3197 return Ok(());
3198 }
3199 self.state.status = RunStatus::Failed;
3200 self.state.save()?;
3201 self.settle_questions();
3202 bail!("no candidate produced a change; nothing to judge");
3203 }
3204 self.state.status = RunStatus::Judging;
3205 self.state.save()?;
3206 Ok(())
3207 }
3208
3209 async fn judge(&mut self) -> Result<()> {
3212 let run_id = self.state.id.clone();
3217 let prompts = self.state.config.prompts.clone();
3218 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3219 return Ok(());
3220 }
3221 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3222 if viable.len() == 1 {
3223 self.state.judge_skipped = true;
3230 self.state.event(
3231 "judge",
3232 format!(
3233 "only candidate {} produced a change; judging skipped",
3234 viable[0].label
3235 ),
3236 );
3237 self.state.save()?;
3238 return Ok(());
3239 }
3240 self.state.status = RunStatus::Judging;
3241
3242 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3243 let language = self.state.config.graph.language.clone();
3244 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3245 let sessions = self.state.config.graph.sessions;
3246 let artifacts = agent::artifacts_dir(&self.state.dir());
3247 let root = self.state.worktree_root();
3248 let base_short = short(&self.state.base_commit);
3249
3250 let mut jobs = Vec::new();
3251 let mut orders = Vec::new();
3252 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3253 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3254 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3255 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3256 let seat_key = format!("judge-{}", j + 1);
3257 let seat = self.seat(&seat_key, &spec.id);
3258 jobs.push(SeatJob {
3259 prompt: prompt::judge(
3260 &self.state.instruction,
3261 &views,
3262 self.roles.judges.len(),
3263 &base_short,
3264 &language,
3265 ),
3266 spec,
3267 seat,
3268 cwd: root.join(format!("judge-{}", j + 1)),
3269 timeout,
3270 allow_write: false,
3271 sessions,
3272 artifacts: artifacts.clone(),
3273 stem: format!("judge-{}", j + 1),
3274 handover: None,
3275 });
3276 }
3277
3278 self.state.event(
3279 "judge",
3280 format!(
3281 "{} judges ranking {} candidates blind",
3282 jobs.len(),
3283 viable.len()
3284 ),
3285 );
3286 let labels_for_check = labels.clone();
3287 let mut quota_losses = Vec::new();
3288 let cache = self.state.config.cache_dir();
3289 let ctx = WaveCtx {
3290 carry_seats: false,
3291 pauses: Some((&self.pause, &self.interrupt)),
3292 run: &run_id,
3293 node: "judge",
3294 prompts: &prompts,
3295 cache: cache.as_deref(),
3296 round: None,
3297 };
3298 let results = ask_json_wave::<Ranking>(
3299 jobs,
3300 Arc::clone(&self.sem),
3301 self.state.config.graph.retries,
3302 &self.roles.judge_roster,
3303 &ctx,
3304 &mut quota_losses,
3305 &mut self.state,
3306 &move |r: &Ranking| r.validate(&labels_for_check),
3307 )
3308 .await;
3309 if self.state.resource_block.is_some() {
3310 return self.stop_for_resource();
3311 }
3312 self.state.quota.extend(quota_losses);
3313
3314 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3315 let agent_id = seat.agent.clone();
3316 self.state.seats.insert(seat.key.clone(), seat);
3317 let mut record = Judgement {
3318 judge: j + 1,
3319 seat: format!("judge-{}", j + 1),
3320 agent: agent_id,
3321 ranking: Vec::new(),
3322 reasons: BTreeMap::new(),
3323 confidence: None,
3324 strengths: BTreeMap::new(),
3325 order: orders[j].clone(),
3326 failed: None,
3327 duration_ms: 0,
3328 };
3329 match res {
3330 Ok((ranking, out)) => {
3331 record.ranking = ranking.normalized();
3332 record.reasons = ranking.reasons;
3333 record.strengths = ranking.strengths;
3334 record.confidence = ranking.confidence;
3335 record.duration_ms = out.duration_ms;
3336 self.state.event(
3337 "judge",
3338 format!(
3339 "judge {} ranked {}",
3340 j + 1,
3341 record.ranking.iter().collect::<String>()
3342 ),
3343 );
3344 }
3345 Err(e) => {
3346 record.failed = Some(e.to_string());
3347 self.state
3348 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3349 }
3350 }
3351 self.state.judgements.push(record);
3352 self.state.save()?;
3353 }
3354 Ok(())
3355 }
3356
3357 async fn deliberate(&mut self) -> Result<()> {
3360 let run_id = self.state.id.clone();
3365 let prompts = self.state.config.prompts.clone();
3366 if !self.state.deliberation.is_empty() {
3367 return Ok(());
3368 }
3369 let tops: Vec<char> = self
3370 .state
3371 .judgements
3372 .iter()
3373 .filter_map(|j| j.ranking.first().copied())
3374 .collect();
3375 let rounds = self.state.config.graph.deliberate_rounds;
3376 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3377 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3378 self.state.event(
3379 "deliberate",
3380 format!("judges agreed on {} outright; no deliberation", tops[0]),
3381 );
3382 }
3383 self.state.status = RunStatus::Voting;
3384 self.state.save()?;
3385 return Ok(());
3386 }
3387
3388 self.state.status = RunStatus::Deliberating;
3389 self.state.event(
3390 "deliberate",
3391 format!(
3392 "split: first choices were {} — opening {rounds} round(s)",
3393 tops.iter().collect::<String>()
3394 ),
3395 );
3396
3397 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3398 let language = self.state.config.graph.language.clone();
3399 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3400 let sessions = self.state.config.graph.sessions;
3401 let artifacts = agent::artifacts_dir(&self.state.dir());
3402 let root = self.state.worktree_root();
3403 let base_short = short(&self.state.base_commit);
3404
3405 for round in 1..=rounds {
3409 let mut turns: Vec<DeliberationTurn> = Vec::new();
3410 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3411 if self.state.judgements[j].failed.is_some() {
3412 continue;
3413 }
3414 let seat_key = format!("judge-{}", j + 1);
3415 let spec = self.occupant(&seat_key, spec);
3416 let mut seat = self.seat(&seat_key, &spec.id);
3417 let transcript = self.transcript(&turns, j);
3418 let build = |context: Option<&str>| {
3419 prompt::deliberate(
3420 &self.state.instruction,
3421 context,
3422 &transcript,
3423 round,
3424 rounds,
3425 &language,
3426 )
3427 };
3428 let block = self.candidate_block(&viable, &base_short);
3429 let full = build(Some(&block));
3430 let text = if has_context(&spec, &seat, sessions) {
3431 build(None)
3432 } else {
3433 full.clone()
3434 };
3435 let job = SeatJob {
3436 spec,
3437 seat: seat.clone(),
3438 prompt: text,
3439 cwd: root.join(format!("judge-{}", j + 1)),
3440 timeout,
3441 allow_write: false,
3442 sessions,
3443 artifacts: artifacts.clone(),
3444 stem: format!("delib-{round}-judge-{}", j + 1),
3445 handover: Some(full),
3446 };
3447 let cache = self.state.config.cache_dir();
3448 let ctx = WaveCtx {
3449 carry_seats: false,
3450 pauses: Some((&self.pause, &self.interrupt)),
3451 run: &run_id,
3452 node: "deliberate",
3453 prompts: &prompts,
3454 cache: cache.as_deref(),
3455 round: None,
3456 };
3457 let mut losses = Vec::new();
3461 let mut results = ask_wave_with::<String>(
3462 vec![job],
3463 Arc::clone(&self.sem),
3464 0,
3465 &self.roles.judge_roster,
3466 &ctx,
3467 &mut losses,
3468 &mut self.state,
3469 &|text: &str| {
3470 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3471 },
3472 )
3473 .await;
3474 self.state.quota.extend(losses);
3475 let (updated, res, _) = results.pop().expect("one job in, one result out");
3476 seat = updated;
3477 let agent_id = seat.agent.clone();
3478 self.state.seats.insert(seat.key.clone(), seat);
3479 let body = match res {
3480 Ok((body, _)) => body,
3481 Err(e) => {
3484 self.state
3485 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3486 continue;
3487 }
3488 };
3489 let tentative = verdict::extract_json::<Position>(&body)
3490 .ok()
3491 .and_then(|p| p.tentative)
3492 .and_then(|s| s.trim().chars().next())
3493 .map(|c| c.to_ascii_uppercase());
3494 self.state.event(
3495 "deliberate",
3496 format!(
3497 "round {round}: judge {} now favours {}",
3498 j + 1,
3499 tentative.map_or("—".to_owned(), |c| c.to_string())
3500 ),
3501 );
3502 turns.push(DeliberationTurn {
3503 judge: j + 1,
3504 agent: agent_id,
3505 body: blind::sanitize_prose(&body, &self.state.config.blind),
3506 tentative,
3507 });
3508 }
3509 self.state
3510 .deliberation
3511 .push(DeliberationRound { round, turns });
3512 self.state.save()?;
3513 }
3514
3515 self.state.status = RunStatus::Voting;
3516 self.state.save()?;
3517 Ok(())
3518 }
3519
3520 async fn vote(&mut self) -> Result<()> {
3523 let run_id = self.state.id.clone();
3528 let prompts = self.state.config.prompts.clone();
3529 if !self.state.votes.is_empty() {
3530 return Ok(());
3531 }
3532 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3533 if viable.len() == 1 {
3534 return Ok(());
3535 }
3536 self.state.status = RunStatus::Voting;
3537
3538 let language = self.state.config.graph.language.clone();
3539 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3540 let sessions = self.state.config.graph.sessions;
3541 let artifacts = agent::artifacts_dir(&self.state.dir());
3542 let root = self.state.worktree_root();
3543 let base_short = short(&self.state.base_commit);
3544 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3545
3546 let mut jobs = Vec::new();
3547 let mut seats_at = Vec::new();
3548 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3549 if self
3550 .state
3551 .judgements
3552 .get(j)
3553 .is_some_and(|r| r.failed.is_some())
3554 {
3555 continue;
3556 }
3557 let seat_key = format!("judge-{}", j + 1);
3558 let spec = self.occupant(&seat_key, spec);
3559 let seat = self.seat(&seat_key, &spec.id);
3560 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3561 let text = if has_context(&spec, &seat, sessions) {
3562 prompt::final_vote(&viable, &language)
3563 } else {
3564 full.clone()
3565 };
3566 jobs.push(SeatJob {
3567 spec,
3568 seat,
3569 prompt: text,
3570 cwd: root.join(format!("judge-{}", j + 1)),
3571 timeout,
3572 allow_write: false,
3573 sessions,
3574 artifacts: artifacts.clone(),
3575 stem: format!("vote-judge-{}", j + 1),
3576 handover: Some(full),
3577 });
3578 seats_at.push(j);
3579 }
3580
3581 self.state.event(
3582 "vote",
3583 format!(
3584 "collecting {} final votes one by one, privately",
3585 jobs.len()
3586 ),
3587 );
3588 let allowed = viable.clone();
3589 let mut quota_losses = Vec::new();
3590 let cache = self.state.config.cache_dir();
3591 let ctx = WaveCtx {
3592 carry_seats: false,
3593 pauses: Some((&self.pause, &self.interrupt)),
3594 run: &run_id,
3595 node: "vote",
3596 prompts: &prompts,
3597 cache: cache.as_deref(),
3598 round: None,
3599 };
3600 let results = ask_json_wave::<FinalVote>(
3601 jobs,
3602 Arc::clone(&self.sem),
3603 self.state.config.graph.retries,
3604 &self.roles.judge_roster,
3605 &ctx,
3606 &mut quota_losses,
3607 &mut self.state,
3608 &move |v: &FinalVote| match v.label() {
3609 Some(c) if allowed.contains(&c) => Ok(()),
3610 other => bail!("vote {other:?} is not one of {allowed:?}"),
3611 },
3612 )
3613 .await;
3614 if self.state.resource_block.is_some() {
3615 return self.stop_for_resource();
3616 }
3617 self.state.quota.extend(quota_losses);
3618
3619 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3620 let agent_id = seat.agent.clone();
3621 self.state.seats.insert(seat.key.clone(), seat);
3622 let initial = self
3623 .state
3624 .judgements
3625 .get(j)
3626 .and_then(|r| r.ranking.first().copied());
3627 let mut record = VoteRecord {
3628 judge: j + 1,
3629 agent: agent_id,
3630 vote: None,
3631 reason: String::new(),
3632 changed: false,
3633 };
3634 match res {
3635 Ok((v, _)) => {
3636 record.vote = v.label();
3637 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3638 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3639 self.state.event(
3640 "vote",
3641 format!(
3642 "judge {} voted {}{}",
3643 j + 1,
3644 record.vote.unwrap_or('?'),
3645 if record.changed { " (changed)" } else { "" }
3646 ),
3647 );
3648 }
3649 Err(e) => {
3650 self.state
3651 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3652 }
3653 }
3654 self.state.votes.push(record);
3655 self.state.save()?;
3656 }
3657 Ok(())
3658 }
3659
3660 fn tally(&mut self) -> Result<()> {
3663 if self.state.tally.is_some() {
3664 return Ok(());
3665 }
3666 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3667 let tops: Vec<char> = self
3668 .state
3669 .judgements
3670 .iter()
3671 .filter_map(|j| j.ranking.first().copied())
3672 .collect();
3673 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3674
3675 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3678 let mut cast: Vec<char> = Vec::new();
3679 for (i, j) in self.state.judgements.iter().enumerate() {
3680 let vote = self
3681 .state
3682 .votes
3683 .iter()
3684 .find(|v| v.judge == i + 1)
3685 .and_then(|v| v.vote)
3686 .or_else(|| j.ranking.first().copied());
3687 if let Some(v) = vote {
3688 *first_choice.entry(v).or_insert(0) += 1;
3689 cast.push(v);
3690 }
3691 }
3692
3693 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3694 for j in &self.state.judgements {
3695 let n = j.ranking.len();
3696 for (pos, label) in j.ranking.iter().enumerate() {
3697 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3698 }
3699 }
3700
3701 let best = first_choice.values().copied().max().unwrap_or(0);
3702 let mut leaders: Vec<char> = first_choice
3703 .iter()
3704 .filter(|(_, v)| **v == best)
3705 .map(|(k, _)| *k)
3706 .collect();
3707 let mut tie_break = None;
3708 if leaders.len() > 1 {
3709 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3710 let borda_leaders: Vec<char> = leaders
3711 .iter()
3712 .copied()
3713 .filter(|l| borda[l] == top_borda)
3714 .collect();
3715 tie_break = Some(if borda_leaders.len() == 1 {
3716 format!(
3717 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3718 leaders.len()
3719 )
3720 } else {
3721 format!(
3722 "{} way tie on both first-choice votes and Borda points, broken by label order",
3723 leaders.len()
3724 )
3725 });
3726 leaders = borda_leaders;
3727 leaders.sort_unstable();
3728 }
3729 let winner = *leaders
3730 .first()
3731 .or(viable.first())
3732 .context("no candidate to declare a winner from")?;
3733
3734 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3735 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3736 let deliberated = !self.state.deliberation.is_empty();
3737
3738 let quota_seats: std::collections::BTreeSet<&str> =
3742 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3743 let mut present = 0usize;
3744 for (i, j) in self.state.judgements.iter().enumerate() {
3745 if quota_seats.contains(j.seat.as_str()) {
3746 continue;
3747 }
3748 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3749 let voted = self
3750 .state
3751 .votes
3752 .iter()
3753 .any(|v| v.judge == i + 1 && v.vote.is_some());
3754 if ranked || voted {
3755 present += 1;
3756 }
3757 }
3758 let needs_quorum = viable.len() > 1;
3764 let judges_total = if needs_quorum {
3765 self.roles.judges.len()
3766 } else {
3767 0
3768 };
3769 let quorum = if needs_quorum {
3770 judges_total / 2 + 1
3771 } else {
3772 0
3773 };
3774 let met_quorum = !needs_quorum || present >= quorum;
3775 let uncontested = (!needs_quorum).then(|| {
3776 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3777 });
3778
3779 self.state.event(
3780 "tally",
3781 match &uncontested {
3782 Some(reason) => format!("winner {winner} — {reason}"),
3783 None => format!(
3784 "winner {winner} — votes {} | initial {} | {} changed | \
3785 {present}/{judges_total} judges{}",
3786 first_choice
3787 .iter()
3788 .map(|(k, v)| format!("{k}:{v}"))
3789 .collect::<Vec<_>>()
3790 .join(" "),
3791 if unanimous_initial {
3792 "unanimous"
3793 } else {
3794 "split"
3795 },
3796 changed_votes,
3797 if met_quorum {
3798 String::new()
3799 } else {
3800 format!(" — below quorum ({quorum} required)")
3801 },
3802 ),
3803 },
3804 );
3805 if !met_quorum {
3806 self.state.event(
3807 "stall",
3808 format!(
3809 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3810 the run stops here, resumable"
3811 ),
3812 );
3813 }
3814 self.state.tally = Some(Tally {
3815 first_choice,
3816 borda,
3817 winner,
3818 rankings: tops.len(),
3819 unanimous_initial,
3820 deliberated,
3821 changed_votes,
3822 unanimous_final,
3823 tie_break,
3824 judges: judges_total,
3825 present,
3826 quorum,
3827 met_quorum,
3828 uncontested,
3829 });
3830 self.state.status = if met_quorum {
3831 RunStatus::Reviewing
3832 } else {
3833 RunStatus::Stalled
3834 };
3835 self.state.save()?;
3836 Ok(())
3837 }
3838
3839 #[allow(clippy::too_many_lines)]
3860 async fn recover_stall(&mut self) -> Result<bool> {
3861 let run_id = self.state.id.clone();
3866 let prompts = self.state.config.prompts.clone();
3867 let quota_seats: BTreeSet<&str> =
3872 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3873 let absent: Vec<String> = self
3874 .state
3875 .judgements
3876 .iter()
3877 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3878 .map(|j| j.seat.clone())
3879 .collect();
3880 if absent.is_empty() {
3881 return Ok(false);
3882 }
3883 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3884 if viable.len() <= 1 {
3885 return Ok(false);
3886 }
3887 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3888 let language = self.state.config.graph.language.clone();
3889 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3890 let sessions = self.state.config.graph.sessions;
3891 let artifacts = agent::artifacts_dir(&self.state.dir());
3892 let root = self.state.worktree_root();
3893 let base_short = short(&self.state.base_commit);
3894 let candidates: Vec<Candidate> = viable.clone();
3895
3896 let mut positions: Vec<usize> = absent
3898 .iter()
3899 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3900 .collect();
3901 if positions.is_empty() {
3902 return Ok(false);
3903 }
3904 positions.sort_unstable();
3905 positions.dedup();
3906
3907 let mut judge_jobs = Vec::new();
3909 for &j in &positions {
3910 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3911 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3912 let seat_key = format!("judge-{}", j + 1);
3913 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3914 let seat = self.seat(&seat_key, &spec.id);
3915 judge_jobs.push(SeatJob {
3916 spec,
3917 seat,
3918 prompt: prompt::judge(
3919 &self.state.instruction,
3920 &views,
3921 self.roles.judges.len(),
3922 &base_short,
3923 &language,
3924 ),
3925 cwd: root.join(seat_key),
3926 timeout,
3927 allow_write: false,
3928 sessions,
3929 artifacts: artifacts.clone(),
3930 stem: format!("judge-{}-recover", j + 1),
3931 handover: None,
3932 });
3933 }
3934
3935 let labels_for_check = labels.clone();
3936 let mut judge_losses = Vec::new();
3937 let retries = self.state.config.graph.retries;
3938 let cache = self.state.config.cache_dir();
3939 let ctx = WaveCtx {
3940 carry_seats: false,
3941 pauses: Some((&self.pause, &self.interrupt)),
3942 run: &run_id,
3943 node: "judge",
3944 prompts: &prompts,
3945 cache: cache.as_deref(),
3946 round: None,
3947 };
3948 let results = ask_json_wave::<Ranking>(
3949 judge_jobs,
3950 Arc::clone(&self.sem),
3951 retries,
3952 &self.roles.judge_roster,
3953 &ctx,
3954 &mut judge_losses,
3955 &mut self.state,
3956 &move |r: &Ranking| r.validate(&labels_for_check),
3957 )
3958 .await;
3959 if self.state.resource_block.is_some() {
3960 self.stop_for_resource()?;
3961 return Ok(false);
3962 }
3963
3964 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3966 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3967 let agent_id = seat.agent.clone();
3968 self.state.seats.insert(seat.key.clone(), seat);
3969 let record = &mut self.state.judgements[j];
3970 match res {
3971 Ok((ranking, out)) => {
3972 record.agent = agent_id;
3973 record.ranking = ranking.normalized();
3974 record.reasons = ranking.reasons;
3975 record.strengths = ranking.strengths;
3976 record.confidence = ranking.confidence;
3977 record.failed = None;
3978 record.duration_ms = out.duration_ms;
3979 recovered.insert(j);
3980 self.state.event(
3981 "recover",
3982 format!("judge {} ranked again after the limit", j + 1),
3983 );
3984 }
3985 Err(e) => {
3986 self.state
3987 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3988 }
3989 }
3990 }
3991
3992 let mut vote_jobs = Vec::new();
3994 let mut vote_pos: Vec<usize> = Vec::new();
3995 for &j in &recovered {
3996 let seat_key = format!("judge-{}", j + 1);
3997 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3998 let seat = self.seat(&seat_key, &spec.id);
3999 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4000 let text = if has_context(&spec, &seat, sessions) {
4001 prompt::final_vote(&labels, &language)
4002 } else {
4003 full.clone()
4004 };
4005 vote_jobs.push(SeatJob {
4006 spec,
4007 seat,
4008 prompt: text,
4009 cwd: root.join(seat_key),
4010 timeout,
4011 allow_write: false,
4012 sessions,
4013 artifacts: artifacts.clone(),
4014 stem: format!("vote-judge-{}-recover", j + 1),
4015 handover: Some(full),
4016 });
4017 vote_pos.push(j);
4018 }
4019 let allowed = labels.clone();
4020 let mut vote_losses = Vec::new();
4021 let vote_retries = self.state.config.graph.retries;
4022 let vote_cache = self.state.config.cache_dir();
4023 let ctx = WaveCtx {
4024 carry_seats: false,
4025 pauses: Some((&self.pause, &self.interrupt)),
4026 run: &run_id,
4027 node: "vote",
4028 prompts: &prompts,
4029 cache: vote_cache.as_deref(),
4030 round: None,
4031 };
4032 let votes = ask_json_wave::<FinalVote>(
4033 vote_jobs,
4034 Arc::clone(&self.sem),
4035 vote_retries,
4036 &self.roles.judge_roster,
4037 &ctx,
4038 &mut vote_losses,
4039 &mut self.state,
4040 &move |v: &FinalVote| match v.label() {
4041 Some(c) if allowed.contains(&c) => Ok(()),
4042 other => bail!("vote {other:?} is not one of {allowed:?}"),
4043 },
4044 )
4045 .await;
4046 if self.state.resource_block.is_some() {
4047 self.stop_for_resource()?;
4048 return Ok(false);
4049 }
4050 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4051 let agent_id = seat.agent.clone();
4052 self.state.seats.insert(seat.key.clone(), seat);
4053 match res {
4054 Ok((v, _)) => {
4055 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4056 rec.vote = v.label();
4057 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4058 } else {
4059 self.state.votes.push(VoteRecord {
4060 judge: j + 1,
4061 agent: agent_id,
4062 vote: v.label(),
4063 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4064 changed: false,
4065 });
4066 }
4067 self.state.event(
4068 "recover",
4069 format!("judge {} voted again after the limit", j + 1),
4070 );
4071 }
4072 Err(e) => {
4073 self.state
4074 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4075 }
4076 }
4077 }
4078
4079 let recovered_keys: BTreeSet<String> = recovered
4083 .iter()
4084 .map(|&j| format!("judge-{}", j + 1))
4085 .collect();
4086 self.state
4087 .quota
4088 .retain(|q| !recovered_keys.contains(&q.seat));
4089 for loss in judge_losses.into_iter().chain(vote_losses) {
4093 if recovered_keys.contains(&loss.seat) {
4094 continue;
4095 }
4096 self.state.quota.retain(|q| q.seat != loss.seat);
4097 self.state.quota.push(loss);
4098 }
4099
4100 self.state.tally = None;
4102 self.tally()?;
4103 Ok(self
4104 .state
4105 .tally
4106 .as_ref()
4107 .map(|t| t.met_quorum)
4108 .unwrap_or(false))
4109 }
4110
4111 async fn graft(&mut self) -> Result<()> {
4128 let Err(e) = self.graft_inner().await else {
4129 return Ok(());
4130 };
4131 if e.is::<crate::repo_lock::ResourceBlocked>() {
4132 return Err(e);
4133 }
4134 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4135 self.state.status = RunStatus::Blocked;
4136 self.state.event("graft", format!("blocked: {e:#}"));
4137 self.state.save()?;
4138 return Err(e);
4139 }
4140 if let Some(rec) = self.state.graft.clone()
4145 && !rec.outcome.settled()
4146 {
4147 if let Some(winner) = self.state.winner().cloned() {
4148 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4149 if re.is::<crate::repo_lock::ResourceBlocked>() {
4150 return Err(re);
4151 }
4152 self.state.status = RunStatus::Blocked;
4153 self.state.event("graft", format!("blocked: {re:#}"));
4154 self.state.save()?;
4155 return Err(re);
4156 }
4157 }
4158 if let Some(g) = self.state.graft.as_mut() {
4159 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4160 }
4161 }
4162 self.state.event("graft", format!("skipped: {e:#}"));
4163 if let Err(e) = self.state.save() {
4164 tracing::warn!("save after graft: {e:#}");
4165 }
4166 Ok(())
4167 }
4168
4169 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4172 let _coordination = self.coordinate(wt).await?;
4173 crate::recovery::restore_repair(wt, pre)
4174 .await
4175 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4176 }
4177
4178 fn graft_skipped(&mut self, why: &str) {
4180 let message = format!("skipped: {why}");
4181 if !self
4182 .state
4183 .events
4184 .iter()
4185 .any(|e| e.node == "graft" && e.message == message)
4186 {
4187 self.state.event("graft", message);
4188 }
4189 }
4190
4191 async fn graft_inner(&mut self) -> Result<()> {
4192 if let Some(rec) = self.state.graft.clone()
4195 && !rec.outcome.settled()
4196 {
4197 if self.state.reviews.is_empty()
4201 && let Some(winner) = self.state.winner().cloned()
4202 {
4203 self.graft_restore(&winner.worktree, &rec.pre).await?;
4204 }
4205 if rec.outcome == GraftOutcome::WaitingForRestore {
4206 self.state.graft = None;
4207 self.state.save()?;
4208 } else {
4209 if let Some(g) = self.state.graft.as_mut() {
4210 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4211 }
4212 self.state.event(
4213 "graft",
4214 "rolled back: the previous round was interrupted; the winner is untouched",
4215 );
4216 self.state.save()?;
4217 return Ok(());
4218 }
4219 }
4220 let plan = match graft::plan(&self.state) {
4221 Ok(plan) => plan,
4222 Err(skip) => {
4223 if skip != graft::Skip::AlreadyDone {
4224 self.graft_skipped(skip.reason());
4225 self.state.save()?;
4226 }
4227 return Ok(());
4228 }
4229 };
4230 let Some(winner) = self.state.winner().cloned() else {
4231 return Ok(());
4232 };
4233 let wt = winner.worktree.clone();
4234 if !git::is_clean(&wt).await? {
4235 self.graft_skipped("the winner's worktree has uncommitted changes");
4236 self.state.save()?;
4237 return Ok(());
4238 }
4239 let pre = git::rev_parse(&wt, "HEAD").await?;
4240 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4241 else {
4242 self.graft_skipped("the winner's diff could not be measured");
4243 self.state.save()?;
4244 return Ok(());
4245 };
4246 let limit = graft::limit(winner_lines);
4247 self.state.graft = Some(GraftRecord {
4250 pre: pre.clone(),
4251 post: None,
4252 lines: None,
4253 limit: Some(limit),
4254 agent: None,
4255 outcome: GraftOutcome::Running,
4256 requested: graft::requested(&plan),
4257 adoption: None,
4258 adoption_error: None,
4259 });
4260 self.state.event(
4261 "graft",
4262 format!(
4263 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4264 plan.strengths
4265 .iter()
4266 .map(|(l, _, _)| l.to_string())
4267 .collect::<Vec<_>>()
4268 .join(", "),
4269 plan.winner
4270 ),
4271 );
4272 self.state.save()?;
4273
4274 let fix_prompt = prompt::graft(
4275 &self.state.instruction,
4276 plan.winner,
4277 &plan.strengths,
4278 limit,
4279 &self.state.config.graph.language,
4280 );
4281 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4282 let sessions = self.state.config.graph.sessions;
4283 let artifacts = agent::artifacts_dir(&self.state.dir());
4284 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4285 let (_, seat, out) = self
4286 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4287 SeatJob {
4288 prompt: fix_prompt.clone(),
4289 spec,
4290 seat,
4291 cwd: wt.clone(),
4292 timeout,
4293 allow_write: true,
4294 sessions,
4295 artifacts: artifacts.clone(),
4296 stem: "graft".to_owned(),
4297 handover: None,
4298 }
4299 })
4300 .await;
4301 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4302 self.state.graft.as_mut().expect("graft record").outcome =
4305 GraftOutcome::WaitingForRestore;
4306 return self.stop_for_resource();
4307 }
4308 let mut why: Option<String> = None;
4309 match &out {
4310 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4311 why = Some("the fixer produced nothing usable".to_owned());
4312 }
4313 AgentOutcome::Ok(_) => {}
4314 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4315 AgentOutcome::Quota(o) => {
4316 self.state.quota.push(QuotaLoss {
4317 seat: seat.key.clone(),
4318 node: "graft".to_owned(),
4319 at: Timestamp::now(),
4320 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4321 });
4322 why = Some("rate limited (quota)".to_owned());
4323 }
4324 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4325 why = Some(format!(
4326 "the fixer failed: {}",
4327 e.lines()
4328 .next()
4329 .unwrap_or("")
4330 .chars()
4331 .take(160)
4332 .collect::<String>()
4333 ));
4334 }
4335 }
4336 let agent_id = seat.agent.clone();
4337 self.state.seats.insert(seat.key.clone(), seat);
4338
4339 let mut post = None;
4340 let mut lines = None;
4341 let _coordination = self.coordinate(&wt).await?;
4342 if why.is_none() {
4343 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4344 if let Ok(r) = &rescued {
4345 self.state.note_withheld("graft", &r.withheld);
4346 }
4347 let head = git::rev_parse(&wt, "HEAD").await?;
4348 let dirty = match &rescued {
4352 Ok(r) => git::status_porcelain(&wt)
4353 .await
4354 .map(|st| {
4355 st.lines().any(|l| {
4356 !r.withheld
4357 .iter()
4358 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4359 })
4360 })
4361 .unwrap_or(true),
4362 Err(_) => true,
4363 };
4364 if dirty {
4365 why = Some("the fixer left changes that could not be committed".to_owned());
4366 } else if head == pre {
4367 why = Some("the fixer changed nothing".to_owned());
4368 } else {
4369 lines = git::changed_lines(&wt, &pre, &head).await;
4370 if graft::within_budget(lines, limit) {
4371 post = Some(head);
4372 } else {
4373 why = Some(format!(
4374 "{} changed lines against a limit of {limit}",
4375 lines.map_or_else(
4376 || "an unmeasurable number of".to_owned(),
4377 |l| l.to_string()
4378 )
4379 ));
4380 }
4381 }
4382 }
4383 let (adoption, adoption_error) = match &out {
4384 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4385 _ => (None, None),
4386 };
4387 let requested = graft::requested(&plan);
4388 let outcome = match why {
4389 None => {
4390 self.state.event(
4391 "graft",
4392 format!(
4393 "applied: {} changed lines (limit {limit})",
4394 lines.unwrap_or(0)
4395 ),
4396 );
4397 GraftOutcome::Applied
4398 }
4399 Some(why) => {
4400 crate::recovery::restore_repair(&wt, &pre)
4403 .await
4404 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4405 self.state.event(
4406 "graft",
4407 format!("rolled back: {why}; the winner is untouched"),
4408 );
4409 GraftOutcome::RolledBack(why)
4410 }
4411 };
4412 self.state.graft = Some(GraftRecord {
4413 pre,
4414 post,
4415 lines,
4416 limit: Some(limit),
4417 agent: Some(agent_id),
4418 outcome,
4419 requested,
4420 adoption,
4421 adoption_error,
4422 });
4423 self.state.save()?;
4424 Ok(())
4425 }
4426
4427 async fn fold_losers(&mut self) -> Result<()> {
4430 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4431 return Ok(());
4432 };
4433 if !self.state.candidate_preservation.is_empty() {
4434 bail!("candidate preservation incomplete; refusing cleanup");
4435 }
4436 let repo = self.state.repo.clone();
4437 let _coordination = self.coordinate(&repo).await?;
4438 let mut folded = Vec::new();
4439 for i in 0..self.state.candidates.len() {
4440 let c = &self.state.candidates[i];
4441 if c.label == winner || c.folded {
4442 continue;
4443 }
4444 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4445 remove_preserved_worktree(&repo, &wt).await?;
4446 if git::branch_exists(&repo, &branch).await?
4447 && !git::branch_delete(&repo, &branch).await?
4448 {
4449 bail!("candidate branch removal refused");
4450 }
4451 self.state.candidates[i].folded = true;
4452 self.state.save()?;
4453 folded.push(label.to_string());
4454 }
4455 let root = self.state.worktree_root();
4457 for j in 1..=self.roles.judges.len() {
4458 let wt = root.join(format!("judge-{j}"));
4459 if wt.exists() {
4460 remove_preserved_worktree(&repo, &wt).await?;
4461 }
4462 }
4463 if self.state.config.graph.advise {
4466 for k in 1..=self.state.config.graph.advisors {
4467 let wt = root.join(format!("advisor-{k}"));
4468 if wt.exists() {
4469 remove_preserved_worktree(&repo, &wt).await?;
4470 }
4471 }
4472 }
4473 if !folded.is_empty() {
4474 self.state
4475 .event("fold", format!("folded candidates {}", folded.join(", ")));
4476 self.state.save()?;
4477 }
4478 Ok(())
4479 }
4480
4481 async fn sync_to_base(&mut self) -> Result<()> {
4521 if self.state.status == RunStatus::AlreadyInBase {
4522 return Ok(());
4523 }
4524 let conflicted = self
4525 .state
4526 .base_sync
4527 .as_ref()
4528 .is_some_and(|s| s.conflict.is_some());
4529 let Some(winner) = self.state.winner().cloned() else {
4530 return Ok(());
4531 };
4532
4533 let repo = self.state.repo.clone();
4534 let remote = self.state.config.merge.remote.clone();
4535 let base_branch = self.state.base_branch.clone();
4536 let tracking = format!("{remote}/{base_branch}");
4537
4538 git::fetch(&repo, &remote, &base_branch).await.ok();
4539 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4544 return Ok(());
4545 };
4546
4547 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4548 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4549 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4550
4551 if (behind > 0 || conflicted || head == tip)
4560 && self
4561 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4562 .await?
4563 {
4564 return Ok(());
4565 }
4566 if conflicted {
4567 return Ok(());
4568 }
4569
4570 if behind == 0 {
4571 if let Some(from) = self
4580 .state
4581 .rebase_fixes
4582 .iter()
4583 .rev()
4584 .find_map(|r| r.from.clone())
4585 && from != head
4586 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4587 .await
4588 .is_ok_and(|o| o.ok())
4589 {
4590 git::sync_to_head(&winner.worktree).await?;
4591 }
4592 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4598 if let Some(why) = &conflict {
4599 self.state.status = RunStatus::Blocked;
4600 self.state.event("land", why.clone());
4601 }
4602 self.state.base_sync = Some(BaseSync {
4603 tip,
4604 behind: 0,
4605 attempts,
4606 conflict,
4607 already_in: None,
4608 });
4609 self.state.save()?;
4610 return Ok(());
4611 }
4612
4613 if attempts >= BASE_SYNC_ROUNDS {
4614 let why = format!(
4615 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4616 rebase(s); rebasing again would only race it",
4617 winner.branch
4618 );
4619 self.state.status = RunStatus::Blocked;
4620 self.state.base_sync = Some(BaseSync {
4621 tip,
4622 behind,
4623 attempts,
4624 conflict: Some(why.clone()),
4625 already_in: None,
4626 });
4627 self.state.event("land", why);
4628 self.state.save()?;
4629 return Ok(());
4630 }
4631
4632 self.state.event(
4633 "land",
4634 format!(
4635 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4636 winner.branch
4637 ),
4638 );
4639 self.state.save()?;
4640
4641 let branch_tracking = format!("{remote}/{}", winner.branch);
4647 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4648 let remote_tip = if self.state.task_pr_branch().is_none()
4649 && matches!(&fetched_branch, Ok(o) if o.ok())
4650 {
4651 git::rev_parse(&repo, &branch_tracking).await.ok()
4652 } else {
4653 None
4654 };
4655 if let Some(theirs) = &remote_tip
4659 && !git::is_ancestor(&repo, theirs, &head).await
4660 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4661 .await
4662 .is_ok_and(|missing| missing.is_empty())
4663 {
4664 let why = format!(
4665 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4666 them",
4667 short(theirs),
4668 winner.branch
4669 );
4670 self.state.status = RunStatus::Blocked;
4671 self.state.base_sync = Some(BaseSync {
4672 tip,
4673 behind,
4674 attempts,
4675 conflict: Some(why.clone()),
4676 already_in: None,
4677 });
4678 self.state.event("land", why);
4679 self.state.save()?;
4680 return Ok(());
4681 }
4682
4683 let scratch = self.state.dir().join("base-sync");
4684 let rebased = match crate::rebase::rebase_with_fixer(
4685 &mut self.state,
4686 &scratch,
4687 &winner.branch,
4688 &tracking,
4689 )
4690 .await
4691 {
4692 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4693 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4694 Err(e) => Err(e),
4695 };
4696 let attempts = attempts + 1;
4697 match rebased {
4698 Ok(None) => {
4699 git::sync_to_head(&winner.worktree).await?;
4703 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4704 let mut conflict = None;
4705 if let Some(pinned) = &remote_tip {
4706 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4707 match pushed {
4708 Ok(o) if o.ok() => self.state.event(
4709 "land",
4710 format!("pushed rebased {} to {remote}", winner.branch),
4711 ),
4712 Ok(o) => {
4713 conflict = Some(format!(
4714 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4715 winner.branch,
4716 short(pinned),
4717 o.stderr.chars().take(600).collect::<String>()
4718 ));
4719 }
4720 Err(e) => {
4721 conflict = Some(format!(
4722 "rebased {} locally but could not push it: {e:#}",
4723 winner.branch
4724 ));
4725 }
4726 }
4727 }
4728 if let Some(why) = &conflict {
4729 self.state.status = RunStatus::Blocked;
4730 self.state.event("land", why.clone());
4731 }
4732 self.state.base_sync = Some(BaseSync {
4733 tip: tip.clone(),
4734 behind: 0,
4735 attempts,
4736 conflict,
4737 already_in: None,
4738 });
4739 self.state
4740 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4741 }
4742 Ok(Some(conflict)) => {
4743 let why = format!(
4744 "{} conflicts with {tracking} and did not rebase: {}",
4745 winner.branch,
4746 conflict.chars().take(600).collect::<String>()
4747 );
4748 self.state.status = RunStatus::Blocked;
4749 self.state.base_sync = Some(BaseSync {
4750 tip,
4751 behind,
4752 attempts,
4753 conflict: Some(why.clone()),
4754 already_in: None,
4755 });
4756 self.state.event("land", why);
4757 }
4758 Err(e) => {
4759 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4760 self.state.status = RunStatus::Blocked;
4761 self.state.base_sync = Some(BaseSync {
4762 tip,
4763 behind,
4764 attempts,
4765 conflict: Some(why.clone()),
4766 already_in: None,
4767 });
4768 self.state.event("land", why);
4769 }
4770 }
4771 self.state.save()?;
4772 Ok(())
4773 }
4774
4775 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4781 if self.state.task_pr_branch().is_some() {
4784 return None;
4785 }
4786 let repo = self.state.repo.clone();
4787 let remote = self.state.config.merge.remote.clone();
4788 let fetched = git::fetch(&repo, &remote, branch).await;
4789 if !matches!(&fetched, Ok(o) if o.ok()) {
4790 return None;
4791 }
4792 let branch_tracking = format!("{remote}/{branch}");
4793 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4794 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4795 return None;
4796 }
4797 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4798 .await
4799 .is_ok_and(|missing| missing.is_empty())
4800 {
4801 return Some(format!(
4802 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4803 them",
4804 short(&theirs)
4805 ));
4806 }
4807 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4808 Ok(o) if o.ok() => {
4809 self.state
4810 .event("land", format!("pushed rebased {branch} to {remote}"));
4811 None
4812 }
4813 Ok(o) => Some(format!(
4814 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4815 someone may have pushed): {}",
4816 short(&theirs),
4817 o.stderr.chars().take(600).collect::<String>()
4818 )),
4819 Err(e) => Some(format!(
4820 "{branch} is rebased locally but could not be pushed: {e:#}"
4821 )),
4822 }
4823 }
4824
4825 async fn settle_already_in(
4842 &mut self,
4843 branch: &str,
4844 tip: &str,
4845 head: &str,
4846 attempts: usize,
4847 behind: usize,
4848 ) -> Result<bool> {
4849 let repo = self.state.repo.clone();
4850 let remote = self.state.config.merge.remote.clone();
4851 let start = self.state.base_commit.clone();
4852 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4853 Ok(Some(e)) => e,
4854 Ok(None) => return Ok(false),
4855 Err(e) => {
4856 tracing::warn!("already-in-base check for {branch}: {e:#}");
4857 return Ok(false);
4858 }
4859 };
4860 let mut verified = vec![head.to_owned()];
4861 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4862 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4863 if matches!(&fetched, Ok(o) if o.ok())
4864 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4865 && theirs != head
4866 {
4867 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4868 Ok(Some(_)) => verified.push(theirs),
4869 _ => return Ok(false),
4870 }
4871 }
4872 let base_branch = self.state.base_branch.clone();
4873 let message = format!(
4874 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4875 land",
4876 evidence.names(),
4877 evidence.proof.as_str()
4878 );
4879 let closed =
4880 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4881 .await;
4882 match closed {
4883 Ok(Ok(url)) => self
4884 .state
4885 .event("land", format!("closed {url}: superseded on {base_branch}")),
4886 Ok(Err(why)) => self
4887 .state
4888 .event("land", format!("did not close a pull request: {why}")),
4889 Err(e) => {
4890 let why = format!(
4891 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4892 not be closed ({e:#}); resume to retry"
4893 );
4894 self.state.status = RunStatus::Blocked;
4895 self.state.base_sync = Some(BaseSync {
4896 tip: tip.to_owned(),
4897 behind,
4898 attempts,
4899 conflict: Some(why.clone()),
4900 already_in: None,
4901 });
4902 self.state.event("land", why);
4903 self.state.save()?;
4904 return Ok(true);
4905 }
4906 }
4907 self.state.status = RunStatus::AlreadyInBase;
4908 self.state.base_sync = Some(BaseSync {
4909 tip: tip.to_owned(),
4910 behind,
4911 attempts,
4912 conflict: None,
4913 already_in: Some(evidence),
4914 });
4915 self.state.event("land", message);
4916 self.state.save()?;
4917 self.settle_questions();
4918 Ok(true)
4919 }
4920
4921 fn landing_base(&self) -> String {
4931 self.state
4932 .base_sync
4933 .as_ref()
4934 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4935 }
4936
4937 pub async fn fix_selected(
4970 &mut self,
4971 ids: &[String],
4972 reason: &str,
4973 allow_stale: bool,
4974 ) -> Result<()> {
4975 let reason = reason.trim();
4976 if reason.is_empty() {
4977 bail!("a fix request needs a reason — that is the operator's own record of why");
4978 }
4979 if ids.is_empty() {
4980 bail!("no finding id given");
4981 }
4982 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4983 bail!(
4984 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4985 has already concluded — can be given a targeted fix. A run still \
4986 in progress should simply be resumed; a `merged` run's branch has \
4987 already landed, so its answer is a fresh `magi review <branch>`, \
4988 not reopening this run's own record",
4989 self.state.id,
4990 self.state.status.as_str()
4991 );
4992 }
4993 let Some(winner) = self.state.winner().cloned() else {
4994 bail!("run {} has no winning candidate to fix", self.state.id);
4995 };
4996 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4997 bail!(
4998 "branch `{}` no longer exists; this run cannot be extended",
4999 winner.branch
5000 );
5001 }
5002 let home = crate::run::home();
5003 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5004 bail!(
5005 "run {} is currently being worked on by another magi process",
5006 self.state.id
5007 );
5008 }
5009 let _claim = FixClaim::acquire(&self.state.dir())?;
5015
5016 let mut seen = BTreeSet::new();
5020 let mut findings = Vec::new();
5021 let mut missing = Vec::new();
5022 for id in ids {
5023 if !seen.insert(id.clone()) {
5024 continue;
5025 }
5026 match self.state.finding(id) {
5027 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5028 id: f.id.clone(),
5029 severity: f.severity,
5030 reviewer_vote: rec.vote,
5031 round: round.round,
5032 round_head: round.head.clone(),
5033 reviewer: rec.reviewer,
5034 agent: rec.agent.clone(),
5035 file: f.file.clone(),
5036 line: f.line,
5037 title: f.title.clone(),
5038 detail: f.detail.clone(),
5039 outcome: OperatorFixOutcome::Pending,
5040 }),
5041 None => missing.push(id.clone()),
5042 }
5043 }
5044 if !missing.is_empty() {
5045 bail!(
5046 "unknown finding id(s): {}; nothing was changed",
5047 missing.join(", ")
5048 );
5049 }
5050
5051 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5052 let stale_details: Vec<(String, String)> = findings
5053 .iter()
5054 .filter(|f| f.round_head != head_at_request)
5055 .map(|f| (f.id.clone(), f.round_head.clone()))
5056 .collect();
5057 let stale = !stale_details.is_empty();
5058 if stale && !allow_stale {
5059 bail!(
5060 "the branch has moved since some finding(s) were raised — {} — now \
5061 at {}; pass --allow-stale to fix anyway, or re-run review first",
5062 stale_details
5063 .iter()
5064 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5065 .collect::<Vec<_>>()
5066 .join(", "),
5067 short(&head_at_request)
5068 );
5069 }
5070
5071 let request = OperatorFixRequest {
5072 requested_at: Timestamp::now(),
5073 reason: reason.to_owned(),
5074 findings,
5075 head_at_request: head_at_request.clone(),
5076 allow_stale,
5077 stale,
5078 fix: None,
5079 result_head: None,
5080 follow_up_review_run: None,
5081 };
5082 self.state.event(
5083 "fix",
5084 format!(
5085 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5086 request.findings.len(),
5087 request
5088 .findings
5089 .iter()
5090 .map(|f| f.id.as_str())
5091 .collect::<Vec<_>>()
5092 .join(", "),
5093 ),
5094 );
5095 self.state.operator_fixes.push(request);
5102 self.state.save()?;
5103 let request_index = self.state.operator_fixes.len() - 1;
5104
5105 let repo = self.state.repo.clone();
5114 let coordination = self.coordinate(&repo).await?;
5115 if winner.worktree.exists() {
5116 let dirty = git::git(
5119 &winner.worktree,
5120 &["status", "--porcelain", "--untracked-files=all"],
5121 )
5122 .await?;
5123 let only_withheld = dirty.lines().all(|l| {
5124 l.strip_prefix("?? ")
5125 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5126 });
5127 if !only_withheld {
5128 bail!(
5129 "`{}` has uncommitted changes; refusing to touch it — commit or \
5130 discard them first",
5131 winner.worktree.display()
5132 );
5133 }
5134 git::worktree_remove(&self.state.repo, &winner.worktree)
5135 .await
5136 .ok();
5137 }
5138 let fix_worktree = self.state.worktree_root().join("operator-fix");
5139 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5140 git::git(
5141 &self.state.repo,
5142 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5143 )
5144 .await
5145 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5146 if !git::is_clean(&fix_worktree).await? {
5147 git::worktree_remove(&self.state.repo, &fix_worktree)
5148 .await
5149 .ok();
5150 bail!(
5151 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5152 winner.branch
5153 );
5154 }
5155 let repo_for_setup = self.state.repo.clone();
5156 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5157 Ok(()) => {
5158 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5159 }
5160 Err(e) => Err(e),
5161 };
5162 if let Err(e) = setup {
5163 git::worktree_remove(&self.state.repo, &fix_worktree)
5164 .await
5165 .ok();
5166 return Err(e);
5167 }
5168
5169 let run_id = self.state.id.clone();
5170 let prompts = self.state.config.prompts.clone();
5171 let language = self.state.config.graph.language.clone();
5172 let sessions = self.state.config.graph.sessions;
5173 let artifacts = agent::artifacts_dir(&self.state.dir());
5174 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5175 .findings
5176 .iter()
5177 .map(|f| Finding {
5178 id: f.id.clone(),
5179 severity: f.severity,
5180 file: f.file.clone(),
5181 line: f.line,
5182 title: f.title.clone(),
5183 detail: f.detail.clone(),
5184 })
5185 .collect();
5186 let fix_prompt = prompt::operator_fix_in(
5187 &self.state.instruction,
5188 &finding_list,
5189 reason,
5190 &stale_details,
5191 &head_at_request,
5192 &language,
5193 &self.state.config.graph.github_language,
5194 );
5195 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5196 drop(coordination);
5197 let (job, seat, out) = self
5198 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5199 prompt: fix_prompt.clone(),
5200 spec,
5201 seat,
5202 cwd: fix_worktree.clone(),
5203 timeout,
5204 allow_write: true,
5205 sessions,
5206 artifacts: artifacts.clone(),
5207 stem: "operator-fix".to_owned(),
5208 handover: None,
5209 })
5210 .await;
5211 let agent_id = seat.agent.clone();
5212
5213 let mut fix = FixRecord {
5214 agent: agent_id,
5215 addressed: Vec::new(),
5216 rejected: Vec::new(),
5217 notes: String::new(),
5218 committed: false,
5219 failed: None,
5220 duration_ms: 0,
5221 continuation: None,
5222 };
5223 let mut final_seat = seat.clone();
5224 match out {
5225 AgentOutcome::Ok(o) => {
5226 fix.duration_ms = o.duration_ms;
5227 let parsed = verdict::extract_json::<FixReport>(&o.text);
5228 let incomplete_reason = match &parsed {
5229 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5230 "the reply parsed, but it reported a command whose own CLI \
5231 never confirmed an exit status"
5232 .to_owned(),
5233 ),
5234 Ok(_) => None,
5235 Err(e) => Some(e.to_string()),
5236 };
5237 match incomplete_reason {
5238 None => {
5239 let report = parsed.expect("checked Ok above");
5240 fix.addressed = report.addressed;
5241 fix.rejected = report.rejected;
5242 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5243 }
5244 Some(reason) => {
5245 let (resumed_seat, resolved, failure, cont) = self
5246 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5247 .await;
5248 fix.duration_ms += cont.cumulative_wait_ms;
5249 fix.continuation = Some(cont);
5250 final_seat = resumed_seat;
5251 match resolved {
5252 Some(report) => {
5253 fix.addressed = report.addressed;
5254 fix.rejected = report.rejected;
5255 fix.notes =
5256 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5257 }
5258 None => fix.failed = failure,
5259 }
5260 }
5261 }
5262 }
5263 AgentOutcome::Dropped(o) => {
5264 fix.duration_ms = o.duration_ms;
5265 let why = o
5266 .dropped
5267 .as_ref()
5268 .map(|d| d.why.as_str())
5269 .unwrap_or("the CLI ended the stream without delivering its answer");
5270 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5271 }
5272 AgentOutcome::Quota(o) => {
5273 self.state.quota.push(QuotaLoss {
5274 seat: final_seat.key.clone(),
5275 node: "fix".to_owned(),
5276 at: Timestamp::now(),
5277 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5278 });
5279 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5280 }
5281 AgentOutcome::Failed(e) => fix.failed = Some(e),
5282 AgentOutcome::ResourceBlocked(e) => {
5283 fix.failed = Some(format!("the fixer was not started: {e}"));
5284 }
5285 }
5286 if fix.continuation.is_none() {
5287 fix.continuation = Some(ContinuationRecord::not_needed());
5288 }
5289 self.state.seats.insert(final_seat.key.clone(), final_seat);
5290
5291 let rescue_message = format!(
5292 "magi: operator-selected fix ({}) (uncommitted work)",
5293 self.state.operator_fixes[request_index]
5294 .findings
5295 .iter()
5296 .map(|f| f.id.as_str())
5297 .collect::<Vec<_>>()
5298 .join(", ")
5299 );
5300 let repo = self.state.repo.clone();
5301 let coordination = self.coordinate(&repo).await?;
5302 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5303 self.state.note_withheld("fix", &r.withheld);
5304 }
5305 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5306 fix.committed = after != head_at_request;
5307 git::worktree_remove(&self.state.repo, &fix_worktree)
5308 .await
5309 .ok();
5310 drop(coordination);
5311
5312 self.state.event(
5313 "fix",
5314 match &fix.failed {
5315 Some(reason) => format!(
5316 "operator fix: adoption report was lost ({reason}); {}",
5317 if fix.committed {
5318 "committed"
5319 } else {
5320 "NO new commit"
5321 }
5322 ),
5323 None => format!(
5324 "operator fix: {} addressed, {} rejected, {}",
5325 fix.addressed.len(),
5326 fix.rejected.len(),
5327 if fix.committed {
5328 "committed"
5329 } else {
5330 "NO new commit"
5331 }
5332 ),
5333 },
5334 );
5335
5336 for f in &mut self.state.operator_fixes[request_index].findings {
5343 f.outcome = if fix.failed.is_some() {
5344 OperatorFixOutcome::Unreported
5345 } else if fix.addressed.contains(&f.id) {
5346 OperatorFixOutcome::Addressed
5347 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5348 OperatorFixOutcome::Rejected { why: r.why.clone() }
5349 } else {
5350 OperatorFixOutcome::Unreported
5351 };
5352 }
5353
5354 let committed = fix.committed;
5355 if committed {
5356 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5357 }
5358 self.state.operator_fixes[request_index].fix = Some(fix);
5359 self.state.save()?;
5362
5363 if committed {
5364 self.state.event(
5365 "fix",
5366 format!(
5367 "operator fix committed {}; opening a follow-up review-only run",
5368 short(&after)
5369 ),
5370 );
5371 let origin =
5374 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5375 match Self::review(
5376 &self.state.repo,
5377 &winner.branch,
5378 self.state.config.clone(),
5379 origin,
5380 )
5381 .await
5382 {
5383 Ok(mut follow_up) => {
5384 follow_up.state.event(
5385 "start",
5386 format!(
5387 "requested by an operator fix on run {} for finding(s) {}",
5388 self.state.id,
5389 self.state.operator_fixes[request_index]
5390 .findings
5391 .iter()
5392 .map(|f| f.id.as_str())
5393 .collect::<Vec<_>>()
5394 .join(", "),
5395 ),
5396 );
5397 follow_up.state.save()?;
5398 let follow_up_id = follow_up.state.id.clone();
5399 let adopted = match crate::direct::adopt(
5402 &follow_up.state,
5403 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5404 ) {
5405 Ok(a) => a,
5406 Err(e) => {
5407 self.state.event(
5410 "fix",
5411 format!(
5412 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5413 ),
5414 );
5415 self.state.save()?;
5416 return Ok(());
5417 }
5418 };
5419 let executed = follow_up.execute().await;
5420 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5421 if let Some(a) = adopted {
5422 a.finish(&follow_up.state, executed);
5423 }
5424 if let Some(e) = failure {
5425 self.state.event(
5426 "fix",
5427 format!(
5428 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5429 ),
5430 );
5431 }
5432 self.state.operator_fixes[request_index].follow_up_review_run =
5433 Some(follow_up_id);
5434 }
5435 Err(e) => {
5436 self.state.event(
5437 "fix",
5438 format!("committed the fix but could not open a follow-up review: {e:#}"),
5439 );
5440 }
5441 }
5442 self.state.save()?;
5443 }
5444
5445 Ok(())
5446 }
5447
5448 async fn ask_fixer(
5468 &mut self,
5469 winner: &Candidate,
5470 node: &'static str,
5471 round: Option<usize>,
5472 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5473 ) -> (SeatJob, SeatState, AgentOutcome) {
5474 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5475 self.ask_fixer_among(attempts, node, round, None, build)
5476 .await
5477 }
5478
5479 async fn ask_fixer_among(
5484 &mut self,
5485 attempts: Vec<(AgentSpec, String)>,
5486 node: &'static str,
5487 round: Option<usize>,
5488 restore: Option<(&Path, &str)>,
5489 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5490 ) -> (SeatJob, SeatState, AgentOutcome) {
5491 let run_id = self.state.id.clone();
5492 let prompts = self.state.config.prompts.clone();
5493 let cache = self.state.config.cache_dir();
5494 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5495 let mut last = None;
5496 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5497 let seat = self.seat(&key, &spec.id);
5498 let mut job = build(spec.clone(), seat);
5499 if i > 0 {
5500 job.stem = format!("{}-{}", job.stem, spec.id);
5501 }
5502 let ctx = WaveCtx {
5503 carry_seats: false,
5504 pauses: Some((&self.pause, &self.interrupt)),
5505 run: &run_id,
5506 node,
5507 prompts: &prompts,
5508 cache: cache.as_deref(),
5509 round,
5510 };
5511 let (seat, out) =
5512 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5513 let advances = match &out {
5516 AgentOutcome::Ok(o) => agent::output_advances(o),
5517 AgentOutcome::ResourceBlocked(_) => false,
5518 _ => true,
5519 };
5520 if let Some(next) = ids.get(i + 1)
5521 && advances
5522 {
5523 self.state.seats.insert(seat.key.clone(), seat.clone());
5524 let class =
5525 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5526 let reason = match &out {
5527 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5528 other => fail_reason(other),
5529 };
5530 if let Some((wt, rev)) = restore
5534 && let Err(e) = self.graft_restore(wt, rev).await
5535 {
5536 let why =
5537 format!("could not restore the worktree before the next fixer: {e:#}");
5538 tracing::warn!("{why}");
5539 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5540 AgentOutcome::ResourceBlocked(why)
5541 } else {
5542 AgentOutcome::Failed(why)
5543 };
5544 last = Some((job, seat, out));
5545 break;
5546 }
5547 record_handover(
5548 &mut self.state,
5549 node,
5550 &seat.key,
5551 &spec.id,
5552 next,
5553 &class,
5554 &reason,
5555 );
5556 continue;
5557 }
5558 last = Some((job, seat, out));
5559 break;
5560 }
5561 last.expect("the fixer chain always has an entry")
5562 }
5563
5564 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5575 use crate::recovery::{Mode, Repair};
5576 if !self.state.reviews.is_empty() {
5577 return Ok(());
5578 }
5579 let Some(mut rec) = self.state.recovery.clone() else {
5580 return Ok(());
5581 };
5582 if rec.mode != Mode::VerifyRepair {
5583 return Ok(());
5584 }
5585 let wt = winner.worktree.clone();
5586 let mut head = git::rev_parse(&wt, "HEAD").await?;
5587 match rec.repair.clone() {
5588 Repair::Done { .. } => return Ok(()),
5589 Repair::WaitingForRestore { before } => {
5590 self.graft_restore(&wt, &before).await?;
5591 head = before;
5592 rec.repair = Repair::NotStarted;
5593 self.state.recovery = Some(rec.clone());
5594 self.state.save()?;
5595 }
5596 Repair::Started { before } => {
5597 if let Err(e) = self.graft_restore(&wt, &before).await {
5601 if e.is::<crate::repo_lock::ResourceBlocked>() {
5602 return Err(e);
5603 }
5604 return self
5605 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5606 .await;
5607 }
5608 rec.repair = Repair::Done {
5609 before: before.clone(),
5610 after: before,
5611 committed: false,
5612 failed: Some("interrupted; rolled back".to_owned()),
5613 };
5614 self.state
5615 .event("recovery", "the repair was interrupted and rolled back");
5616 self.state.recovery = Some(rec);
5617 self.state.save()?;
5618 return Ok(());
5619 }
5620 Repair::NotStarted => {}
5621 }
5622 if crate::recovery::recheck(&mut rec, &head) {
5623 self.state.event("recovery", rec.summary());
5624 self.state.recovery = Some(rec);
5625 self.state.save()?;
5626 return Ok(());
5627 }
5628 rec.repair = Repair::Started {
5629 before: head.clone(),
5630 };
5631 self.state.recovery = Some(rec.clone());
5632 self.state.save()?;
5633
5634 let language = self.state.config.graph.language.clone();
5635 let sessions = self.state.config.graph.sessions;
5636 let artifacts = agent::artifacts_dir(&self.state.dir());
5637 let fix_prompt = prompt::recovery_fix(
5638 &self.state.instruction,
5639 &rec,
5640 self.state.config.graph.review_rounds,
5641 &language,
5642 );
5643 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5644 self.state.event(
5645 "recovery",
5646 format!(
5647 "repair: the fixer starts from the failed verification on {}",
5648 short(&head)
5649 ),
5650 );
5651 let (_job, seat, out) = self
5652 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5653 prompt: fix_prompt.clone(),
5654 spec,
5655 seat,
5656 cwd: wt.clone(),
5657 timeout,
5658 allow_write: true,
5659 sessions,
5660 artifacts: artifacts.clone(),
5661 stem: "fix-recovery".to_owned(),
5662 handover: None,
5663 })
5664 .await;
5665 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5666 rec.repair = Repair::WaitingForRestore { before: head };
5667 self.state.recovery = Some(rec);
5668 return self.stop_for_resource();
5669 }
5670 let failed = match &out {
5671 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5672 AgentOutcome::Quota(o) => {
5673 self.state.quota.push(QuotaLoss {
5674 seat: seat.key.clone(),
5675 node: "fix".to_owned(),
5676 at: Timestamp::now(),
5677 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5678 });
5679 Some("rate limited (quota); the fixer could not run".to_owned())
5680 }
5681 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5682 AgentOutcome::Failed(e) => Some(e.clone()),
5683 };
5684 self.state.seats.insert(seat.key.clone(), seat);
5685 let coordination = self.coordinate(&wt).await?;
5686 let failed = match failed {
5687 Some(why) => {
5689 if let Err(e) = git::restore_to(&wt, &head).await {
5690 return self
5691 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5692 .await;
5693 }
5694 Some(why)
5695 }
5696 None => {
5697 if let Ok(r) = git::rescue_commit(
5698 &wt,
5699 "magi: recovery repair of the failed verification (uncommitted work)",
5700 )
5701 .await
5702 {
5703 self.state.note_withheld("fix", &r.withheld);
5704 }
5705 None
5706 }
5707 };
5708 let after = git::rev_parse(&wt, "HEAD").await?;
5709 drop(coordination);
5710 self.state.event(
5711 "recovery",
5712 match &failed {
5713 Some(why) => format!("repair: nothing kept ({why})"),
5714 None if after != head => {
5715 format!("repair: committed {}", short(&after))
5716 }
5717 None => "repair: the fixer made no commit".to_owned(),
5718 },
5719 );
5720 rec.repair = Repair::Done {
5721 before: head.clone(),
5722 committed: after != head,
5723 after,
5724 failed,
5725 };
5726 self.state.recovery = Some(rec);
5727 self.state.save()?;
5728 Ok(())
5729 }
5730
5731 async fn stop_recovery(
5734 &mut self,
5735 mut rec: crate::recovery::RecoveryRecord,
5736 head: &str,
5737 why: String,
5738 ) -> Result<()> {
5739 rec.repair = crate::recovery::Repair::Started {
5741 before: head.to_owned(),
5742 };
5743 self.state.recovery = Some(rec);
5744 self.state
5745 .event("recovery", format!("repair could not be undone: {why}"));
5746 self.state.status = RunStatus::Blocked;
5747 self.state.save()?;
5748 bail!("recovery repair could not be rolled back: {why}")
5749 }
5750
5751 async fn review_loop(&mut self) -> Result<()> {
5752 if self
5757 .state
5758 .base_sync
5759 .as_ref()
5760 .is_some_and(|s| s.conflict.is_some())
5761 {
5762 return Ok(());
5763 }
5764 let run_id = self.state.id.clone();
5769 let prompts = self.state.config.prompts.clone();
5770 let Some(winner) = self.state.winner().cloned() else {
5771 return Ok(());
5772 };
5773 let max_rounds = self.state.config.graph.review_rounds;
5774 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5784 if status == RunStatus::Gating {
5787 self.record_contested_handoff();
5788 }
5789 self.state.status = status;
5790 self.state.save()?;
5791 return Ok(());
5792 }
5793 self.state.status = RunStatus::Reviewing;
5794 if self
5804 .state
5805 .reviews
5806 .last()
5807 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5808 {
5809 let shell = self.state.config.shell();
5810 return self
5811 .stop_reviewing(
5812 "the last round's own verification never resolved",
5813 &shell,
5814 &winner.worktree,
5815 )
5816 .await;
5817 }
5818
5819 self.recovery_repair(&winner).await?;
5820 if self.resource_halt()? {
5821 return Ok(());
5822 }
5823
5824 let repo = self.state.repo.clone();
5825 let root = self.state.worktree_root();
5826 let language = self.state.config.graph.language.clone();
5827 let sessions = self.state.config.graph.sessions;
5828 let artifacts = self.merge_repair_artifacts();
5829 let base = self.landing_base();
5830 let base_short = short(&base);
5831 let reviewers = self.roles.reviewers.clone();
5832 let shell = self.state.config.shell();
5833
5834 for round in (self.state.reviews.len() + 1)..=max_rounds {
5835 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5836 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5837 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5838 let prev_verification = self
5847 .state
5848 .reviews
5849 .last()
5850 .and_then(|r| r.verification_summary(&head));
5851 let prior_section = match (&self.state.recovery, round) {
5854 (Some(rec), 1) if self.state.reviews.is_empty() => {
5855 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5856 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5857 Some(crate::recovery::head_of(
5858 &git::diff(&winner.worktree, reviewed, "HEAD")
5859 .await
5860 .unwrap_or_default(),
5861 crate::recovery::SINCE_DIFF_MAX,
5862 ))
5863 }
5864 _ => None,
5865 };
5866 Some(prompt::recovery_review(rec, since.as_deref()))
5867 }
5868 _ => None,
5869 };
5870
5871 let mut jobs = Vec::new();
5875 let coordination = self.coordinate(&repo).await?;
5876 for (r, spec) in reviewers.iter().cloned().enumerate() {
5877 let wt = root.join(format!("review-{}", r + 1));
5878 if wt.exists() {
5881 git::reset_detached(&wt, &head).await?;
5882 } else {
5883 git::worktree_add_detached(&repo, &wt, &head).await?;
5884 }
5885 self.setup_seat_worktree(&repo, &wt, false).await?;
5886 let seat_key = format!("review-{}", r + 1);
5887 let spec = pick_start_spec(
5890 &self.roles.reviewer_roster,
5891 spec,
5892 self.state.seat_history.get(&seat_key),
5893 );
5894 let seat = self.seat(&seat_key, &spec.id);
5895 jobs.push(SeatJob {
5896 prompt: prompt::review(&prompt::ReviewCtx {
5897 instruction: &self.state.instruction,
5898 branch: &winner.branch,
5899 base_short: &base_short,
5900 stat: &stat,
5901 patch: &patch,
5902 verification: prev_verification.as_ref(),
5903 prior: prior_section.as_deref(),
5904 reviewers: reviewers.len(),
5905 round,
5906 rounds: max_rounds,
5907 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5910 lens: Lens::for_seat(r),
5911 language: &language,
5912 github_language: &self.state.config.graph.github_language,
5913 }),
5914 spec,
5915 seat,
5916 cwd: wt,
5917 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5918 allow_write: false,
5919 sessions,
5920 artifacts: artifacts.clone(),
5921 stem: format!("review-{round}-{}", r + 1),
5922 handover: None,
5923 });
5924 }
5925
5926 drop(coordination);
5927 self.state.event(
5928 "review",
5929 format!(
5930 "round {round}: {} reviewers on {}",
5931 jobs.len(),
5932 short(&head)
5933 ),
5934 );
5935 let mut quota_losses = Vec::new();
5936 let review_retries = self.state.config.graph.retries;
5937 let review_cache = self.state.config.cache_dir();
5938 let ctx = WaveCtx {
5939 carry_seats: true,
5940 pauses: Some((&self.pause, &self.interrupt)),
5941 run: &run_id,
5942 node: "review",
5943 prompts: &prompts,
5944 cache: review_cache.as_deref(),
5945 round: Some(round),
5946 };
5947 let results = ask_json_wave::<Review>(
5948 jobs,
5949 Arc::clone(&self.sem),
5950 review_retries,
5951 &self.roles.reviewer_roster,
5952 &ctx,
5953 &mut quota_losses,
5954 &mut self.state,
5955 &|_: &Review| Ok(()),
5956 )
5957 .await;
5958 if self.state.resource_block.is_some() {
5959 return self.stop_for_resource();
5960 }
5961 let round_quota_missing = quota_losses.len();
5965 self.state.quota.extend(quota_losses);
5966
5967 let mut records = Vec::new();
5968 let mut all_findings = Vec::new();
5969 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5970 let agent_id = seat.agent.clone();
5971 self.state.seats.insert(seat.key.clone(), seat);
5972 let mut record = ReviewRecord {
5973 reviewer: r + 1,
5974 agent: agent_id,
5975 summary: String::new(),
5976 findings: Vec::new(),
5977 vote: None,
5978 failed: None,
5979 duration_ms: 0,
5980 attempts,
5986 };
5987 match res {
5988 Ok((review, out)) => {
5989 record.summary =
5997 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5998 record.vote = Some(review.vote);
5999 record.duration_ms = out.duration_ms;
6000 for (n, mut f) in review.findings.into_iter().enumerate() {
6001 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6004 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6005 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6006 f.file = f
6012 .file
6013 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6014 all_findings.push(f.clone());
6015 record.findings.push(f);
6016 }
6017 self.state.event(
6018 "review",
6019 format!(
6020 "round {round}: reviewer {} voted {} with {} finding(s)",
6021 r + 1,
6022 review.vote.label(),
6023 record.findings.len()
6024 ),
6025 );
6026 }
6027 Err(e) => {
6028 record.failed = Some(e.to_string());
6029 self.state.event(
6030 "review",
6031 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6032 );
6033 }
6034 }
6035 records.push(record);
6036 }
6037
6038 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6045 let vote_split =
6046 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6047 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6048 if vote_split {
6049 self.state.event(
6050 "review",
6051 format!(
6052 "round {round}: votes split ({}) — one round of reconsideration",
6053 initial_votes
6054 .iter()
6055 .map(|v| v.label())
6056 .collect::<Vec<_>>()
6057 .join(", ")
6058 ),
6059 );
6060 let panel: Vec<ReviewSeatReport<'_>> = records
6063 .iter()
6064 .filter_map(|r| {
6065 r.vote.map(|vote| ReviewSeatReport {
6066 reviewer: r.reviewer,
6067 vote,
6068 summary: &r.summary,
6069 findings: &r.findings,
6070 })
6071 })
6072 .collect();
6073
6074 let mut jobs = Vec::new();
6075 let mut seats_at = Vec::new();
6076 for (r, spec) in reviewers.iter().cloned().enumerate() {
6077 if records[r].vote.is_none() {
6081 continue;
6082 }
6083 let wt = root.join(format!("review-{}", r + 1));
6084 let seat_key = format!("review-{}", r + 1);
6085 let spec = self.occupant(&seat_key, spec);
6086 let seat = self.seat(&seat_key, &spec.id);
6087 let build = |with_patch: bool| {
6095 prompt::review_reconsider(&ReviewReconsiderCtx {
6096 instruction: &self.state.instruction,
6097 reviewer: r + 1,
6098 lens: Lens::for_seat(r),
6099 panel: &panel,
6100 patch: with_patch.then_some(ReviewPatch {
6101 branch: &winner.branch,
6102 base_short: &base_short,
6103 stat: &stat,
6104 patch: &patch,
6105 }),
6106 round,
6107 rounds: max_rounds,
6108 language: &language,
6109 })
6110 };
6111 let full = build(true);
6112 let prompt = if has_context(&spec, &seat, sessions) {
6113 build(false)
6114 } else {
6115 full.clone()
6116 };
6117 jobs.push(SeatJob {
6118 prompt,
6119 spec,
6120 seat,
6121 cwd: wt,
6122 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6123 allow_write: false,
6124 sessions,
6125 artifacts: artifacts.clone(),
6126 stem: format!("review-{round}-reconsider-{}", r + 1),
6127 handover: Some(full),
6128 });
6129 seats_at.push(r);
6130 }
6131
6132 let mut recon_quota_losses = Vec::new();
6133 let recon_cache = self.state.config.cache_dir();
6134 let recon_ctx = WaveCtx {
6135 carry_seats: true,
6136 pauses: Some((&self.pause, &self.interrupt)),
6137 run: &run_id,
6138 node: "review",
6139 prompts: &prompts,
6140 cache: recon_cache.as_deref(),
6141 round: Some(round),
6142 };
6143 let recon_results = ask_json_wave::<ReviewRevote>(
6144 jobs,
6145 Arc::clone(&self.sem),
6146 review_retries,
6147 &self.roles.reviewer_roster,
6148 &recon_ctx,
6149 &mut recon_quota_losses,
6150 &mut self.state,
6151 &|_: &ReviewRevote| Ok(()),
6152 )
6153 .await;
6154 if self.state.resource_block.is_some() {
6155 return self.stop_for_resource();
6156 }
6157 self.state.quota.extend(recon_quota_losses);
6158
6159 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6160 let agent_id = seat.agent.clone();
6161 self.state.seats.insert(seat.key.clone(), seat);
6162 let mut rec = ReviewRevoteRecord {
6163 reviewer: r + 1,
6164 agent: agent_id,
6165 vote: None,
6166 reason: String::new(),
6167 failed: None,
6168 };
6169 match res {
6170 Ok((rv, _)) => {
6171 rec.vote = Some(rv.vote);
6172 rec.reason =
6173 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6174 self.state.event(
6175 "review",
6176 format!(
6177 "round {round}: reviewer {} revoted {}",
6178 r + 1,
6179 rv.vote.label()
6180 ),
6181 );
6182 }
6183 Err(e) => {
6184 rec.failed = Some(e.to_string());
6185 self.state.event(
6186 "review",
6187 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6188 );
6189 }
6190 }
6191 reconsideration.push(rec);
6192 }
6193 } else if initial_votes.len() > 1 {
6194 self.state.event(
6195 "review",
6196 format!(
6197 "round {round}: votes agreed ({}) — no reconsideration",
6198 initial_votes[0].label()
6199 ),
6200 );
6201 }
6202
6203 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6204 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6205 let defer_e2e =
6216 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6217 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6218 let reason =
6219 format!("{blocking} blocking finding(s) already required a fix this round");
6220 self.state.event(
6221 "verify",
6222 format!(
6223 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6224 {}); it will run once a round has none left",
6225 short(&head)
6226 ),
6227 );
6228 (Vec::new(), false, true, Some(reason))
6229 } else {
6230 let e2e_commands = self.state.config.verify.e2e.clone();
6231 let cache_dir = self.state.config.cache_dir();
6232 let context = format!("round {round}");
6233 let wait = self.repository_wait();
6234 let (e2e, verify_retried) = with_cache_lease(
6235 &mut self.state,
6236 cache_dir.as_deref(),
6237 "e2e",
6238 "e2e",
6239 &winner.worktree,
6240 &head,
6241 verify_timeout,
6242 &context,
6243 &wait,
6244 |state, budget| {
6245 let shell = shell.clone();
6246 let e2e_commands = e2e_commands.clone();
6247 let worktree = winner.worktree.clone();
6248 let context = context.clone();
6249 async move {
6250 run_e2e_with_retry(
6251 state,
6252 &shell,
6253 &e2e_commands,
6254 &worktree,
6255 budget,
6256 &context,
6257 )
6258 .await
6259 }
6260 },
6261 )
6262 .await?;
6263 (e2e, verify_retried, false, None)
6264 };
6265
6266 let expected = records.len();
6267 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6268 let incomplete = answered < expected;
6269 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6270 let policy = self.state.config.graph.incomplete_review;
6271 let clean = round_is_clean(
6272 blocking,
6273 e2e_ok,
6274 answered,
6275 expected,
6276 round_quota_missing,
6277 policy,
6278 );
6279
6280 let mut round_record = ReviewRound {
6281 round,
6282 head: head.clone(),
6283 verified_head: None,
6284 verified_at: None,
6285 reviews: records,
6286 e2e,
6287 verify_retried,
6288 e2e_deferred,
6289 e2e_defer_reason,
6290 fix: None,
6291 blocking,
6292 answered,
6293 expected,
6294 clean,
6295 progressed: false,
6296 vote_split,
6297 reconsideration,
6298 verdict: None,
6299 };
6300 round_record.verdict = ReviewVote::worst(
6304 round_record
6305 .final_votes()
6306 .into_iter()
6307 .map(|(_, _, vote)| vote),
6308 );
6309 if !matches!(
6320 round_record.e2e_status(),
6321 E2eStatus::Deferred | E2eStatus::NotConfigured
6322 ) {
6323 round_record.verified_head = Some(head.clone());
6324 round_record.verified_at = Some(Timestamp::now());
6325 }
6326 let this_round_verification = round_record.verification_summary(&head);
6327
6328 if incomplete {
6329 let missing: Vec<String> = round_record
6330 .reviews
6331 .iter()
6332 .filter(|r| r.failed.is_some())
6333 .map(|r| format!("review-{}", r.reviewer))
6334 .collect();
6335 self.state.event(
6336 "review",
6337 format!(
6338 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6339 missing.join(", ")
6340 ),
6341 );
6342 }
6343
6344 if clean {
6345 self.state.event(
6346 "review",
6347 if incomplete && policy == IncompleteReviewPolicy::Warn {
6348 format!(
6349 "round {round}: clean (warn policy, incomplete panel) — no \
6350 blocking findings from the seats that answered, verification green"
6351 )
6352 } else if incomplete {
6353 format!(
6354 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6355 quorum) — no blocking findings from the seats that answered, \
6356 verification green",
6357 expected - answered
6358 )
6359 } else {
6360 format!("round {round}: clean — no blocking findings, verification green")
6361 },
6362 );
6363 self.state.reviews.push(round_record);
6364 self.state.status = RunStatus::Gating;
6365 self.state.save()?;
6366 return Ok(());
6367 }
6368
6369 if incomplete && blocking == 0 && e2e_ok {
6377 self.state.reviews.push(round_record);
6378 self.state.save()?;
6379 if round == max_rounds {
6380 self.state.status = RunStatus::Blocked;
6381 self.state.event(
6382 "review",
6383 format!(
6384 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6385 refusing to call it clean",
6386 expected - answered
6387 ),
6388 );
6389 return Ok(());
6390 }
6391 continue;
6392 }
6393
6394 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6406 self.state.reviews.push(round_record);
6407 return self
6408 .stop_reviewing(
6409 "the round's own verification could not run",
6410 &shell,
6411 &winner.worktree,
6412 )
6413 .await;
6414 }
6415
6416 if round == max_rounds {
6417 self.state.reviews.push(round_record);
6418 return self
6419 .stop_reviewing(
6420 &format!(
6421 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6422 ),
6423 &shell,
6424 &winner.worktree,
6425 )
6426 .await;
6427 }
6428
6429 let blocking_findings: Vec<_> = all_findings
6432 .iter()
6433 .filter(|f| f.severity.blocks())
6434 .cloned()
6435 .collect();
6436 let fix_prompt = prompt::fix_in(
6437 &self.state.instruction,
6438 &blocking_findings,
6439 this_round_verification.as_ref(),
6440 round,
6441 max_rounds,
6442 &language,
6443 &self.state.config.graph.github_language,
6444 );
6445 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6446 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6447 let (job, seat, out) = self
6448 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6449 prompt: fix_prompt.clone(),
6450 spec,
6451 seat,
6452 cwd: winner.worktree.clone(),
6453 timeout,
6454 allow_write: true,
6455 sessions,
6456 artifacts: artifacts.clone(),
6457 stem: format!("fix-{round}"),
6458 handover: None,
6459 })
6460 .await;
6461 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6464 return self.stop_for_resource();
6465 }
6466 let agent_id = seat.agent.clone();
6467
6468 let mut fix = FixRecord {
6469 agent: agent_id,
6470 addressed: Vec::new(),
6471 rejected: Vec::new(),
6472 notes: String::new(),
6473 committed: false,
6474 failed: None,
6475 duration_ms: 0,
6476 continuation: None,
6477 };
6478 let mut continuation = ContinuationRecord::not_needed();
6479 let mut final_seat = seat.clone();
6480 match out {
6481 AgentOutcome::Ok(o) => {
6482 fix.duration_ms = o.duration_ms;
6483 let parsed = verdict::extract_json::<FixReport>(&o.text);
6484 let incomplete_reason = match &parsed {
6491 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6492 "the reply parsed, but it reported a command whose own CLI never \
6493 confirmed an exit status"
6494 .to_owned(),
6495 ),
6496 Ok(_) => None,
6497 Err(e) => Some(e.to_string()),
6498 };
6499 match incomplete_reason {
6500 None => {
6501 let report = parsed.expect("checked Ok above");
6502 fix.addressed = report.addressed;
6503 fix.rejected = report.rejected;
6504 fix.notes =
6505 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6506 }
6507 Some(reason) => {
6508 let (resumed_seat, resolved, failure, cont) = self
6509 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6510 .await;
6511 fix.duration_ms += cont.cumulative_wait_ms;
6512 continuation = cont;
6513 final_seat = resumed_seat;
6514 match resolved {
6515 Some(report) => {
6516 fix.addressed = report.addressed;
6517 fix.rejected = report.rejected;
6518 fix.notes = blind::sanitize_prose(
6519 &report.notes,
6520 &self.state.config.blind,
6521 );
6522 }
6523 None => fix.failed = failure,
6524 }
6525 }
6526 }
6527 }
6528 AgentOutcome::Dropped(o) => {
6530 fix.duration_ms = o.duration_ms;
6531 let why = o
6532 .dropped
6533 .as_ref()
6534 .map(|d| d.why.as_str())
6535 .unwrap_or("the CLI ended the stream without delivering its answer");
6536 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6537 }
6538 AgentOutcome::Quota(o) => {
6539 self.state.quota.push(QuotaLoss {
6540 seat: final_seat.key.clone(),
6541 node: "fix".to_owned(),
6542 at: Timestamp::now(),
6543 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6544 });
6545 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6546 }
6547 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6548 }
6549 fix.continuation = Some(continuation);
6550 self.state.seats.insert(final_seat.key.clone(), final_seat);
6551 let coordination = self.coordinate(&winner.worktree).await?;
6552 if let Ok(r) = git::rescue_commit(
6553 &winner.worktree,
6554 &format!("magi: review round {round} fixes (uncommitted work)"),
6555 )
6556 .await
6557 {
6558 self.state.note_withheld("fix", &r.withheld);
6559 }
6560 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6561 fix.committed = after != before;
6562 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6570 drop(coordination);
6571 let progressed = diff_after != patch;
6572 let commit_note = if fix.committed {
6573 "committed"
6574 } else {
6575 "NO new commit"
6576 };
6577 let tree_note = if progressed {
6578 "changed vs base"
6579 } else {
6580 "unchanged vs base"
6581 };
6582 self.state.event(
6583 "fix",
6584 match &fix.failed {
6585 Some(reason) => {
6591 format!(
6592 "round {round}: fixer's adoption report was lost ({reason}); \
6593 {commit_note}, tree {tree_note}"
6594 )
6595 }
6596 None => format!(
6597 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6598 {tree_note}{}",
6599 fix.addressed.len(),
6600 fix.rejected.len(),
6601 if continuation.outcome == ContinuationOutcome::Resumed {
6602 format!(
6603 " (adoption report recovered after {} continuation(s))",
6604 continuation.attempts
6605 )
6606 } else {
6607 String::new()
6608 },
6609 ),
6610 },
6611 );
6612 round_record.fix = Some(fix);
6613 round_record.progressed = progressed;
6614 self.state.reviews.push(round_record);
6615 self.state.save()?;
6616
6617 if matches!(
6630 continuation.outcome,
6631 ContinuationOutcome::Exhausted
6632 | ContinuationOutcome::QuotaLost
6633 | ContinuationOutcome::NoSession
6634 ) {
6635 return self
6636 .stop_reviewing(
6637 "the fixer's adoption report never came back, even after resuming its \
6638 own seat; refusing to start another round against the same worktree \
6639 while that is unresolved",
6640 &shell,
6641 &winner.worktree,
6642 )
6643 .await;
6644 }
6645
6646 let streak = self
6647 .state
6648 .reviews
6649 .iter()
6650 .rev()
6651 .take_while(|r| !r.progressed)
6652 .count();
6653 if streak >= STAGNANT_LIMIT {
6654 return self
6655 .stop_reviewing(
6656 &format!(
6657 "the tree has not moved against base for {streak} round(s) in a row"
6658 ),
6659 &shell,
6660 &winner.worktree,
6661 )
6662 .await;
6663 }
6664 }
6665 Ok(())
6666 }
6667
6668 fn record_contested_handoff(&mut self) {
6702 if self.state.contested_handoff.is_some() {
6703 return;
6704 }
6705 let Some(contested) = self
6706 .state
6707 .reviews
6708 .last()
6709 .and_then(ReviewRound::contested_handoff)
6710 else {
6711 return;
6712 };
6713 self.state.event(
6714 "review",
6715 format!(
6716 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6717 wait for the owner's approval",
6718 contested.findings.len(),
6719 contested.rejecters.len()
6720 ),
6721 );
6722 self.state.contested_handoff = Some(contested);
6723 }
6724
6725 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6726 let round_idx = self.state.reviews.len() - 1;
6727 let needs_catchup_run = matches!(
6735 self.state.reviews[round_idx].e2e_status(),
6736 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6737 );
6738 if needs_catchup_run {
6739 let round = self.state.reviews[round_idx].round;
6740 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6741 let commands = self.state.config.verify.e2e.clone();
6742 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6743 let cache_dir = self.state.config.cache_dir();
6744 let context = format!(
6745 "round {round}: verification unresolved, catching up before the final decision"
6746 );
6747 let wait = self.repository_wait();
6748 let (outcomes, verify_retried) = with_cache_lease(
6749 &mut self.state,
6750 cache_dir.as_deref(),
6751 "e2e",
6752 "e2e",
6753 worktree,
6754 &attempted_head,
6755 timeout,
6756 &context,
6757 &wait,
6758 |state, budget| {
6759 let shell = shell.to_vec();
6760 let commands = commands.clone();
6761 let context = context.clone();
6762 async move {
6763 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6764 .await
6765 }
6766 },
6767 )
6768 .await?;
6769 let last = &mut self.state.reviews[round_idx];
6770 last.e2e = outcomes;
6771 last.verify_retried = verify_retried;
6772 last.verified_head = Some(attempted_head);
6779 last.verified_at = Some(Timestamp::now());
6780 if verify_inconclusive(&last.e2e) {
6781 self.state.save()?;
6788 return Ok(());
6789 }
6790 last.e2e_deferred = false;
6791 }
6792 let last = &self.state.reviews[round_idx];
6793 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6794
6795 match last.e2e_status() {
6796 E2eStatus::Failed => {
6797 let red: Vec<String> = last
6798 .e2e
6799 .iter()
6800 .filter(|o| !o.ok())
6801 .map(|o| {
6802 format!(
6803 "`{}` -> {:?}\n{}",
6804 o.command,
6805 o.code,
6806 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6807 )
6808 })
6809 .collect();
6810 self.state
6811 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6812 self.state.status = RunStatus::Blocked;
6813 }
6814 E2eStatus::ResourceBlocked => {
6819 self.state.event(
6820 "review",
6821 format!(
6822 "{why}; e2e could not run (shared build cache unavailable); not \
6823 deciding yet"
6824 ),
6825 );
6826 }
6827 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6828 self.state.event(
6829 "review",
6830 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6831 );
6832 self.record_contested_handoff();
6833 self.state.status = RunStatus::Gating;
6834 }
6835 }
6836 self.state.save()?;
6837 Ok(())
6838 }
6839
6840 async fn gate(&mut self) -> Result<()> {
6843 if self.state.status == RunStatus::Failed
6855 || self
6856 .state
6857 .base_sync
6858 .as_ref()
6859 .is_some_and(|s| s.conflict.is_some())
6860 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6861 != Some(RunStatus::Gating)
6862 {
6863 return Ok(());
6864 }
6865 if self.state.gate_ran {
6866 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6877 self.state.status = RunStatus::Blocked;
6878 self.state.save()?;
6879 }
6880 return Ok(());
6881 }
6882 let Some(winner) = self.state.winner().cloned() else {
6883 return Ok(());
6884 };
6885 self.state.status = RunStatus::Gating;
6886 let mut outcomes = self.run_gate(&winner).await?;
6887 loop {
6888 if verify_inconclusive(&outcomes) {
6899 self.state.save()?;
6900 return Ok(());
6901 }
6902 if outcomes.iter().all(CommandOutcome::ok) {
6903 break;
6904 }
6905 match self.gate_fix_round(&winner, &outcomes).await? {
6906 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6907 GateFix::Stop => break,
6908 GateFix::Defer => {
6909 self.state.save()?;
6910 return Ok(());
6911 }
6912 }
6913 }
6914 let passed = outcomes.iter().all(CommandOutcome::ok);
6915 self.state.gate = outcomes;
6916 self.state.gate_ran = true;
6917 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6918 if !passed {
6919 self.state.status = RunStatus::Blocked;
6920 let spent = self.state.gate_fixes.len();
6921 self.state.event(
6922 "gate",
6923 if spent == 0 {
6924 "gate failed; not merging".to_owned()
6925 } else {
6926 format!("gate failed after {spent} gate-fix round(s); not merging")
6927 },
6928 );
6929 }
6930 self.state.save()?;
6931 Ok(())
6932 }
6933
6934 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6944 let commands = state.config.verify.pre_gate.clone();
6945 if commands.is_empty() {
6946 return;
6947 }
6948 let shell = state.config.shell();
6949 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6950 let (outcomes, _) = run_commands(
6951 state,
6952 "pre_gate",
6953 "pre_gate",
6954 0,
6955 &shell,
6956 &commands,
6957 &winner.worktree,
6958 timeout,
6959 )
6960 .await;
6961 for o in &outcomes {
6962 if !o.ok() {
6963 tracing::warn!(
6964 "pre_gate `{}` failed ({:?}); the gate decides",
6965 o.command,
6966 o.code
6967 );
6968 }
6969 state.event(
6970 "pre_gate",
6971 format!(
6972 "`{}` -> {}",
6973 o.command,
6974 if o.ok() {
6975 "pass".to_owned()
6976 } else {
6977 format!(
6978 "FAIL ({:?})\n{}",
6979 o.code,
6980 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6981 )
6982 }
6983 ),
6984 );
6985 }
6986 state.pre_gate = outcomes;
6987 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6988 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6989 Ok(head) => {
6990 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6991 state.pre_gate_commit = Some(head);
6992 }
6993 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6994 },
6995 Ok(false) => {}
6996 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6997 }
6998 if let Err(e) = state.save() {
6999 tracing::warn!("could not persist the pre_gate record: {e:#}");
7000 }
7001 }
7002
7003 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7006 let shell = self.state.config.shell();
7007 let gate_commands = self.state.config.verify.gate.clone();
7008 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7009 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7010 Vec::new()
7011 } else {
7012 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7013 let cache_dir = self.state.config.cache_dir();
7014 let wait = self.repository_wait();
7015 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7016 let (outcomes, _) = with_cache_lease(
7017 &mut self.state,
7018 cache_dir.as_deref(),
7019 "gate",
7020 "gate",
7021 &winner.worktree,
7022 &head,
7023 timeout,
7024 "final gate",
7025 &wait,
7026 |state, budget| {
7027 let shell = shell.clone();
7028 let gate_commands = gate_commands.clone();
7029 let winner = winner.clone();
7030 async move {
7031 Self::run_pre_gate(state, &winner).await;
7032 let (outcomes, timed_out_pids) = run_commands(
7033 state,
7034 "gate",
7035 "gate",
7036 0,
7037 &shell,
7038 &gate_commands,
7039 &winner.worktree,
7040 budget,
7041 )
7042 .await;
7043 (outcomes, false, timed_out_pids)
7044 }
7045 },
7046 )
7047 .await?;
7048 outcomes
7049 };
7050 if outcomes.is_empty() {
7051 self.state.event(
7056 "gate",
7057 "no gate commands configured; nothing to check, passing",
7058 );
7059 }
7060 for o in &outcomes {
7061 self.state.event(
7062 "gate",
7063 format!(
7064 "`{}` -> {}",
7065 o.command,
7066 if o.ok() {
7067 "pass".to_owned()
7068 } else {
7069 format!(
7070 "FAIL ({:?})\n{}",
7071 o.code,
7072 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7073 )
7074 }
7075 ),
7076 );
7077 }
7078 Ok(outcomes)
7079 }
7080
7081 async fn gate_fix_round(
7094 &mut self,
7095 winner: &Candidate,
7096 outcomes: &[CommandOutcome],
7097 ) -> Result<GateFix> {
7098 let cap = self.state.config.graph.gate_fix_rounds;
7099 let spent = self.state.gate_fixes.len();
7100 if spent >= cap {
7101 if cap > 0 {
7102 self.state.event(
7103 "gate",
7104 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7105 );
7106 }
7107 return Ok(GateFix::Stop);
7108 }
7109 if !gate_fixable(outcomes) {
7110 self.state.event(
7111 "gate",
7112 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7113 command or similar); not spending a fix round on it",
7114 );
7115 return Ok(GateFix::Stop);
7116 }
7117 let min_free = self.state.config.disk.min_free_bytes;
7118 if min_free > 0 {
7119 match crate::disk::free_bytes(&winner.worktree) {
7120 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7121 Ok(free) => {
7122 self.state.event(
7123 "gate",
7124 format!(
7125 "only {free} bytes free ({min_free} required by `[disk] \
7126 min_free_bytes`); not spending a fix round on a failure the disk \
7127 may explain"
7128 ),
7129 );
7130 return Ok(GateFix::Stop);
7131 }
7132 Err(e) => {
7133 self.state.event(
7134 "gate",
7135 format!("free disk space could not be measured ({e:#}); no fix round"),
7136 );
7137 return Ok(GateFix::Stop);
7138 }
7139 }
7140 }
7141
7142 let attempt = spent + 1;
7143 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7144 let base = self.landing_base();
7145 let fix_prompt = prompt::gate_fix_in(
7146 &self.state.instruction,
7147 &failed,
7148 attempt,
7149 cap,
7150 &self.state.config.graph.language,
7151 &self.state.config.graph.github_language,
7152 );
7153 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7154 let sessions = self.state.config.graph.sessions;
7155 let artifacts = self.merge_repair_artifacts();
7156 self.state.event(
7157 "gate",
7158 format!("gate failed; gate-fix round {attempt} of {cap}"),
7159 );
7160 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7161 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7162 let (_, seat, out) = self
7163 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7164 prompt: fix_prompt.clone(),
7165 spec,
7166 seat,
7167 cwd: winner.worktree.clone(),
7168 timeout,
7169 allow_write: true,
7170 sessions,
7171 artifacts: artifacts.clone(),
7172 stem: format!("gate-fix-{attempt}"),
7173 handover: None,
7174 })
7175 .await;
7176 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7179 return Ok(GateFix::Defer);
7180 }
7181 let mut record = GateFixRecord {
7182 agent: seat.agent.clone(),
7183 failed,
7184 notes: String::new(),
7185 committed: false,
7186 error: None,
7187 };
7188 match out {
7189 AgentOutcome::Ok(o) => {
7190 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7193 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7194 }
7195 }
7196 AgentOutcome::Dropped(_) => {
7197 record.error = Some("the CLI dropped the stream".to_owned());
7198 }
7199 AgentOutcome::Quota(o) => {
7200 self.state.quota.push(QuotaLoss {
7201 seat: seat.key.clone(),
7202 node: "gate-fix".to_owned(),
7203 at: Timestamp::now(),
7204 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7205 });
7206 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7207 }
7208 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7209 }
7210 self.state.seats.insert(seat.key.clone(), seat);
7211 let coordination = self.coordinate(&winner.worktree).await?;
7212 if let Ok(r) = git::rescue_commit(
7213 &winner.worktree,
7214 &format!("magi: gate fix {attempt} (uncommitted work)"),
7215 )
7216 .await
7217 {
7218 self.state.note_withheld("gate-fix", &r.withheld);
7219 }
7220 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7221 record.committed = after != before;
7222 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7223 let note = record.error.clone();
7224 self.state.gate_fixes.push(record);
7225 self.state.save()?;
7226 if !changed {
7227 self.state.event(
7228 "gate",
7229 match note {
7230 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7231 None => format!("gate-fix round {attempt}: the tree did not change"),
7232 },
7233 );
7234 return Ok(GateFix::Stop);
7235 }
7236 self.state.event(
7237 "gate",
7238 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7239 );
7240
7241 drop(coordination);
7242 let commands = self.state.config.verify.e2e.clone();
7243 if !commands.is_empty() {
7244 let shell = self.state.config.shell();
7245 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7246 let cache_dir = self.state.config.cache_dir();
7247 let context = format!("gate-fix round {attempt}");
7248 let wait = self.repository_wait();
7249 let (e2e, _) = with_cache_lease(
7250 &mut self.state,
7251 cache_dir.as_deref(),
7252 "e2e",
7253 "e2e",
7254 &winner.worktree,
7255 &after,
7256 timeout,
7257 &context,
7258 &wait,
7259 |state, budget| {
7260 let shell = shell.clone();
7261 let commands = commands.clone();
7262 let context = context.clone();
7263 let worktree = winner.worktree.clone();
7264 async move {
7265 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7266 .await
7267 }
7268 },
7269 )
7270 .await?;
7271 if verify_inconclusive(&e2e) {
7272 return Ok(GateFix::Defer);
7273 }
7274 if e2e.iter().any(|o| !o.ok()) {
7275 self.state.event(
7276 "gate",
7277 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7278 );
7279 return Ok(GateFix::Stop);
7280 }
7281 }
7282 Ok(GateFix::Retry)
7283 }
7284
7285 async fn guarded_pr_message(
7292 &mut self,
7293 winner: &Candidate,
7294 facts: Option<&BranchFacts>,
7295 posting: bool,
7296 ) -> Result<Option<PrMessage>> {
7297 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7298 let head = match final_head.as_deref() {
7299 Some(h) => crate::pr_outcome::FinalHead::At(h),
7300 None => crate::pr_outcome::FinalHead::Unknown,
7301 };
7302 let summarized = if posting {
7304 Some(self.summarize_change(winner, final_head.as_deref()).await)
7305 } else {
7306 None
7307 };
7308 if self.state.resource_block.is_some() {
7313 self.state.event(
7314 "pr-summary",
7315 "a shared resource was busy; nothing was published, resume to summarize again",
7316 );
7317 return Ok(None);
7318 }
7319 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7320 anyhow::bail!(
7321 "the final commit moved during public description generation; retry before publishing"
7322 );
7323 }
7324 let note = match (&summarized, &self.state.change_summary) {
7325 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7326 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7327 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7328 (None, _) => ChangeNote::NotAsked,
7329 };
7330 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7331 if !posting {
7332 let identity = crate::scrub::Identity::current();
7333 return Ok(Some(PrMessage {
7334 title: crate::scrub::scrub(&pr.title, &identity),
7335 body: crate::scrub::scrub(&pr.body, &identity),
7336 }));
7337 }
7338 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7341 if let Some(known) = self
7342 .state
7343 .github_text
7344 .clone()
7345 .filter(|g| g.fingerprint == raw_fp)
7346 {
7347 return self.settle_withheld(winner, &pr, known).await;
7348 }
7349 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7350 let mut violations =
7351 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7352 let identity = crate::scrub::Identity::current();
7353 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7354 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7355 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7356 {
7357 violations.push(crate::github_text::Violation::SensitiveData);
7358 }
7359 let has_language = violations
7363 .iter()
7364 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7365 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7366 self.state.event(
7367 "github-text",
7368 if has_language {
7369 format!("description rejected: {violations:?}; requesting one rewrite")
7370 } else {
7371 format!("description rejected: {violations:?}; asking the owner")
7372 },
7373 );
7374 let mut rewritten = false;
7375 let mut rejected_rewrite = None;
7376 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7377 {
7378 let output = text;
7379 if let Some(title) = summary_title(&output) {
7380 let raw_body = summary_without_title(&output);
7383 let clean_title = crate::scrub::scrub(&title, &identity);
7384 let mut body = crate::scrub::scrub(&raw_body, &identity);
7385 let redacted = clean_title != title || body != raw_body;
7386 let title = clean_title;
7387 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7388 if !complete {
7389 self.state.event(
7390 "github-text",
7391 "description rewrite omitted required findings",
7392 );
7393 }
7394 if !body.trim().is_empty() {
7395 body.push_str("\n\n");
7396 body.push_str(&public_pr_evidence(&self.state, head));
7397 }
7398 let marker = format!("magi:run/{}", self.state.id);
7399 if !body.trim().is_empty() && !body.contains(&marker) {
7400 body.push_str(&format!("\n\n{marker}"));
7401 }
7402 let rewrite = PrMessage {
7403 title: title.clone(),
7404 body: body.clone(),
7405 };
7406 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7407 let mut failed = crate::github_text::check_for(
7408 &self.state.config.graph,
7409 &title,
7410 &body,
7411 redecision,
7412 );
7413 if (!crate::github_text::shareable(&title)
7414 || !crate::github_text::shareable(&body))
7415 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7416 {
7417 failed.push(crate::github_text::Violation::SensitiveData);
7418 }
7419 if complete
7420 && !title.trim().is_empty()
7421 && !body.trim().is_empty()
7422 && !failed.is_empty()
7423 {
7424 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7425 title: title.clone(),
7426 body: body.clone(),
7427 failed: failed
7428 .iter()
7429 .map(|v| crate::github_text::category(*v).to_owned())
7430 .collect(),
7431 });
7432 }
7433 if complete
7434 && !title.trim().is_empty()
7435 && !raw_body.trim().is_empty()
7436 && failed.is_empty()
7437 {
7438 decision = redecision;
7439 pr = PrMessage { title, body };
7440 rewritten = true;
7441 self.state
7442 .event("github-text", "description rewrite passed");
7443 if redacted {
7444 self.state.event(
7445 "github-text",
7446 "sensitive data removed from description rewrite",
7447 );
7448 }
7449 }
7450 }
7451 }
7452 if !rewritten {
7453 self.state.event(
7454 "github-text",
7455 "description rewrite unavailable or rejected; asking the owner",
7456 );
7457 if let Some(w) =
7458 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7459 {
7460 self.retire_github_text_question("the withheld text changed");
7463 let record = crate::run::GithubTextAsk {
7464 rewrite: rejected_rewrite,
7465 fingerprint: raw_fp,
7466 title: w.title,
7467 body: w.body,
7468 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7469 question: None,
7470 asks: 0,
7471 resolved: false,
7472 chosen_title: None,
7473 choice: None,
7474 };
7475 return self.settle_withheld(winner, &pr, record).await;
7476 }
7477 }
7478 }
7479 let (title, body) =
7480 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7481 Ok(Some(PrMessage { title, body }))
7482 }
7483
7484 async fn summarize_change(
7495 &mut self,
7496 winner: &Candidate,
7497 head: Option<&str>,
7498 ) -> std::result::Result<(), String> {
7499 let Some(head) = head else {
7500 return Err("the final commit could not be read".to_owned());
7501 };
7502 let remote = self.state.config.merge.remote.clone();
7503 let tracking = format!("{remote}/{}", self.state.base_branch);
7504 if !git::rev_exists(&winner.worktree, &tracking).await {
7505 return Err(format!("the remote base `{tracking}` could not be read"));
7506 }
7507 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7508 Ok(b) if !b.is_empty() => b,
7509 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7510 };
7511 let evidence = crate::pr_outcome::summary_evidence(
7512 &self.state,
7513 crate::pr_outcome::FinalHead::At(head),
7514 );
7515 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7516 if self
7517 .state
7518 .change_summary
7519 .as_ref()
7520 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7521 {
7522 return Ok(());
7523 }
7524 if self.state.change_summary.take().is_some() {
7525 self.state.event(
7526 "pr-summary",
7527 "the final diff or evidence changed; the earlier change summary was discarded",
7528 );
7529 }
7530 if let Err(e) = write_artifact(
7532 &self.state,
7533 &format!("public-description-{fingerprint}.md"),
7534 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7535 ) {
7536 tracing::warn!("could not save public description evidence: {e:#}");
7537 }
7538 let chain = match agent::pick_chain(
7539 &self.state.config.agents,
7540 self.state.config.roles.synthesizer.as_ref(),
7541 &agent::installed,
7542 "synthesizer",
7543 ) {
7544 Ok(chain) => chain,
7545 Err(e) => {
7546 tracing::warn!("no agent for the change summary: {e:#}");
7547 self.state.event("pr-summary", "no summary agent available");
7548 return Err("no agent was available to summarize it".to_owned());
7549 }
7550 };
7551 let subjects = git::subjects(&winner.worktree, &base, head)
7552 .await
7553 .unwrap_or_default();
7554 let stat = git::diff_stat(&winner.worktree, &base, head)
7555 .await
7556 .unwrap_or_default();
7557 let stat: String = stat.chars().take(6_000).collect();
7558 let mut prompt = crate::prompt::change_summary_in(
7559 &base,
7560 head,
7561 &subjects,
7562 stat.trim(),
7563 &self.state.config.graph.github_language,
7564 );
7565 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7566 let sessions = self.state.config.graph.sessions;
7567 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7568 let prompts = self.state.config.prompts.clone();
7569 let cache = self.state.config.cache_dir();
7570 let run = self.state.id.clone();
7571 for (i, spec) in chain.into_iter().enumerate() {
7572 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7573 let job = SeatJob {
7574 spec: spec.clone(),
7575 seat,
7576 cwd: winner.worktree.clone(),
7577 prompt: prompt.clone(),
7578 timeout,
7579 allow_write: false,
7580 sessions,
7581 artifacts: agent::artifacts_dir(&self.state.dir()),
7582 stem: if i == 0 {
7583 "change-summary".to_owned()
7584 } else {
7585 format!("change-summary-{}", spec.id)
7586 },
7587 handover: None,
7588 };
7589 let ctx = WaveCtx {
7590 run: &run,
7591 node: "pr-summary",
7592 prompts: &prompts,
7593 cache: cache.as_deref(),
7594 round: None,
7595 carry_seats: false,
7596 pauses: Some((&self.pause, &self.interrupt)),
7597 };
7598 let (seat, outcome) =
7599 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7600 self.state.seats.insert(seat.key.clone(), seat);
7601 let text = match outcome {
7602 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7603 AgentOutcome::ResourceBlocked(why) => {
7604 if self.state.resource_block.is_none() {
7607 self.state.resource_block = Some(crate::run::ResourceBlock {
7608 node: "pr-summary".to_owned(),
7609 kind: "shared resource".to_owned(),
7610 detail: why,
7611 at: Timestamp::now(),
7612 });
7613 }
7614 return Err("a shared resource was busy".to_owned());
7615 }
7616 _ => continue,
7617 };
7618 let blind = &self.state.config.blind;
7619 let Some(title) = summary_title(&text) else {
7620 continue;
7621 };
7622 let body: String = summary_without_title(&text)
7623 .lines()
7624 .filter(|l| !l.contains("magi:run/"))
7625 .collect::<Vec<_>>()
7626 .join("\n");
7627 let body = blind::sanitize_prose(body.trim(), blind);
7628 let title = blind::sanitize_prose(&title, blind);
7629 if body.trim().is_empty() || title.trim().is_empty() {
7630 continue;
7631 }
7632 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7633 continue;
7634 }
7635 if git::rev_parse(&winner.worktree, "HEAD")
7637 .await
7638 .ok()
7639 .as_deref()
7640 != Some(head)
7641 {
7642 return Err("the branch moved while the summary was written".to_owned());
7643 }
7644 if git::merge_base(&winner.worktree, &tracking, head)
7645 .await
7646 .ok()
7647 .as_deref()
7648 != Some(base.as_str())
7649 || crate::pr_outcome::summary_evidence(
7650 &self.state,
7651 crate::pr_outcome::FinalHead::At(head),
7652 ) != evidence
7653 {
7654 return Err("the summary evidence changed while it was written".to_owned());
7655 }
7656 self.state.change_summary = Some(crate::run::ChangeSummary {
7657 head: head.to_owned(),
7658 base,
7659 evidence: fingerprint,
7660 title,
7661 body,
7662 });
7663 self.state.event(
7664 "pr-summary",
7665 format!("change summary written by {} for the final diff", spec.id),
7666 );
7667 if let Err(e) = self.state.save() {
7668 tracing::warn!("could not save the change summary: {e:#}");
7669 }
7670 return Ok(());
7671 }
7672 self.state
7673 .event("pr-summary", "no agent produced a change summary");
7674 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7675 }
7676
7677 async fn rewrite_pr_text(
7685 &mut self,
7686 winner: &Candidate,
7687 pr: &PrMessage,
7688 violations: &[crate::github_text::Violation],
7689 ) -> Option<String> {
7690 let prompt = format!(
7691 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7692 pr.title, pr.body
7693 );
7694 let sessions = self.state.config.graph.sessions;
7695 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7696 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7698 let key = format!("impl-{}", winner.label);
7699 if let Some(spec) = self
7700 .state
7701 .config
7702 .agents
7703 .iter()
7704 .find(|a| a.id == winner.agent)
7705 .cloned()
7706 && let Some(existing) = self.state.seats.get(&key)
7707 && existing.agent == spec.id
7708 && agent::has_session(spec.kind, existing, sessions)
7709 {
7710 attempts.push((spec, existing.clone(), true));
7711 } else {
7712 let chain = agent::pick_chain(
7713 &self.state.config.agents,
7714 self.state.config.roles.language_judge.as_ref(),
7715 &agent::installed,
7716 "language_judge",
7717 );
7718 match chain {
7719 Ok(chain) => {
7720 for spec in chain {
7721 let seat = crate::fixer::seat_for(
7722 &mut self.state,
7723 "github-text-rewrite",
7724 &spec.id,
7725 );
7726 attempts.push((spec, seat, false));
7727 }
7728 }
7729 Err(e) => {
7730 tracing::warn!("no agent for the description rewrite: {e:#}");
7731 self.state
7732 .event("github-text", "no rewrite agent available");
7733 return None;
7734 }
7735 }
7736 }
7737 let prompts = self.state.config.prompts.clone();
7738 let cache = self.state.config.cache_dir();
7739 let run = self.state.id.clone();
7740 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7741 let job = SeatJob {
7742 spec: spec.clone(),
7743 seat,
7744 cwd: winner.worktree.clone(),
7745 prompt: prompt.clone(),
7746 timeout: retry_budget(timeout_full, resumed),
7747 allow_write: false,
7748 sessions,
7749 artifacts: agent::artifacts_dir(&self.state.dir()),
7750 stem: if i == 0 {
7751 "github-text-rewrite".to_owned()
7752 } else {
7753 format!("github-text-rewrite-{}", spec.id)
7754 },
7755 handover: None,
7756 };
7757 let ctx = WaveCtx {
7758 run: &run,
7759 node: "github-text",
7760 prompts: &prompts,
7761 cache: cache.as_deref(),
7762 round: None,
7763 carry_seats: false,
7764 pauses: Some((&self.pause, &self.interrupt)),
7765 };
7766 let (seat, outcome) =
7767 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7768 self.state.seats.insert(seat.key.clone(), seat);
7769 match outcome {
7770 AgentOutcome::Ok(o)
7771 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7772 {
7773 return Some(o.text);
7774 }
7775 AgentOutcome::ResourceBlocked(_) => return None,
7776 _ => {}
7777 }
7778 }
7779 None
7780 }
7781
7782 fn retire_github_text_question(&mut self, why: &str) {
7784 let Some(id) = self
7785 .state
7786 .github_text
7787 .as_ref()
7788 .and_then(|g| g.question.clone())
7789 else {
7790 return;
7791 };
7792 let _ = ask::Questions::open().update(&id, |q| {
7793 q.abandon(why);
7794 Ok(())
7795 });
7796 }
7797
7798 async fn settle_withheld(
7802 &mut self,
7803 winner: &Candidate,
7804 pr: &PrMessage,
7805 mut g: crate::run::GithubTextAsk,
7806 ) -> Result<Option<PrMessage>> {
7807 use crate::github_text as gt;
7808 let store = ask::Questions::open();
7809 let timeout = self.state.config.graph.answer_timeout;
7810 if !g.resolved {
7811 let mut standing = g
7812 .question
7813 .as_ref()
7814 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7815 if let Some(q) = standing.as_ref()
7816 && q.status.open()
7817 && gt::expired(q, timeout)
7818 {
7819 let why = format!("no answer within {}s of asking", timeout.max(1));
7820 standing = store
7821 .update(&q.id, |q| {
7822 q.abandon(&why);
7823 Ok(())
7824 })
7825 .ok()
7826 .map(|(q, ())| q);
7827 }
7828 let mut ask_again: Option<Option<Vec<String>>> = None;
7829 match standing {
7830 None if g.asks <= 1 => {
7833 g.asks = 0;
7834 ask_again = Some(None);
7835 }
7836 None => {
7837 g.resolved = true;
7838 g.chosen_title = None;
7839 g.choice = Some(GithubTextChoice::Fallback);
7840 }
7841 Some(q) if q.status.open() => {
7842 self.park_on_github_text(&g)?;
7843 return Ok(None);
7844 }
7845 Some(q) => match gt::read_reply(&q) {
7846 gt::Reply::Fallback => {
7847 g.resolved = true;
7848 g.chosen_title = None;
7849 g.choice = Some(GithubTextChoice::Fallback);
7850 }
7851 gt::Reply::Original => {
7852 g.resolved = true;
7853 g.chosen_title = None;
7854 g.choice = Some(GithubTextChoice::Original);
7855 }
7856 gt::Reply::Rewrite => {
7857 g.resolved = true;
7858 g.chosen_title = None;
7859 g.choice = Some(if g.rewrite.is_some() {
7860 GithubTextChoice::Rewrite
7861 } else {
7862 GithubTextChoice::Fallback
7863 });
7864 }
7865 gt::Reply::Title(text) => {
7866 let first = text
7867 .lines()
7868 .map(str::trim)
7869 .find(|l| !l.is_empty())
7870 .unwrap_or("");
7871 let decision = if first.is_empty() {
7872 None
7873 } else {
7874 let candidate = PrMessage {
7875 title: first.to_owned(),
7876 body: String::new(),
7877 };
7878 self.judge_pr_language(&winner.worktree, &candidate).await
7879 };
7880 match gt::vet_title(
7881 &text,
7882 decision,
7883 self.state.config.graph.github_english(),
7884 ) {
7885 Ok(title) => {
7886 g.resolved = true;
7887 g.chosen_title = Some(title);
7888 g.choice = Some(GithubTextChoice::MyTitle);
7889 }
7890 Err(cats) if g.asks < 2 => {
7891 ask_again = Some(Some(
7892 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7893 ))
7894 }
7895 Err(_) => {
7896 g.resolved = true;
7897 g.chosen_title = None;
7898 g.choice = Some(GithubTextChoice::Fallback);
7899 }
7900 }
7901 }
7902 },
7903 }
7904 if let Some(rejected) = ask_again {
7905 let w = gt::Withheld {
7908 title: g.title,
7909 body: g.body,
7910 categories: g.categories.clone(),
7911 };
7912 let artifact = match crate::run::write_artifact(
7915 &self.state,
7916 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7917 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7918 ) {
7919 Ok(p) => Some(p),
7920 Err(e) => {
7921 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7922 None
7923 }
7924 };
7925 let (detail, comparison) = gt::question_content(
7926 &w,
7927 &pr.title,
7928 &pr.body,
7929 &self.state.id,
7930 rejected.as_deref(),
7931 artifact.as_deref(),
7932 g.rewrite.as_ref(),
7933 );
7934 let mut q = ask::Question::new(
7935 self.state.id.clone(),
7936 gt::ASK_NODE.to_owned(),
7937 gt::ASK_SEAT.to_owned(),
7938 gt::question_summary(&self.state.config.graph.language, &w),
7939 detail,
7940 gt::question_choices(&w, g.rewrite.as_ref()),
7941 );
7942 q.posting_comparison = Some(comparison);
7943 q.answer_timeout = timeout;
7944 g.question = Some(q.id.clone());
7947 g.asks += 1;
7948 self.state.github_text = Some(g.clone());
7949 self.state.event(
7950 "github-text",
7951 format!("asking the owner about the withheld text ({})", q.short()),
7952 );
7953 self.state.save()?;
7954 store
7955 .put(&mut q)
7956 .context("file the posting-gate question")?;
7957 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7958 tracing::warn!(
7959 "could not notify about posting-gate question {}: {e:#}",
7960 q.short()
7961 );
7962 }
7963 self.park_on_github_text(&g)?;
7964 return Ok(None);
7965 }
7966 }
7967 let resolved = g.chosen_title.clone();
7968 let original = g.choice == Some(GithubTextChoice::Original);
7969 self.state.event(
7971 "github-text",
7972 if original {
7973 format!(
7974 "posting the original text for: {}",
7975 match (g.title, g.body) {
7976 (true, true) => "title, description",
7977 (true, false) => "title",
7978 _ => "description",
7979 }
7980 )
7981 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7982 "posting the saved rewrite title and description".to_owned()
7983 } else if resolved.is_some() {
7984 "posting the owner's replacement title".to_owned()
7985 } else {
7986 "posting the neutral text for the withheld fields".to_owned()
7987 },
7988 );
7989 self.state.github_text = Some(g.clone());
7990 self.state.save()?;
7991 if g.choice == Some(GithubTextChoice::Rewrite)
7992 && let Some(r) = g.rewrite.as_ref()
7993 {
7994 return Ok(Some(PrMessage {
7995 title: r.title.clone(),
7996 body: r.body.clone(),
7997 }));
7998 }
7999 let w = gt::Withheld {
8000 title: g.title,
8001 body: g.body,
8002 categories: g.categories.clone(),
8003 };
8004 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8005 if let Some(t) = resolved {
8006 title = t;
8007 }
8008 let english = gt::LanguageDecision {
8010 title_english: true,
8011 body_english: true,
8012 body_complete: true,
8013 };
8014 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8015 if original {
8016 if g.title {
8019 title = pr.title.clone();
8020 }
8021 if g.body {
8022 body = pr.body.clone();
8023 }
8024 }
8025 Ok(Some(PrMessage { title, body }))
8026 }
8027
8028 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8029 self.state.github_text = Some(g.clone());
8030 self.state.end_pass_parked(ParkCause::Question);
8031 self.state.parked = true;
8032 self.state.event(
8033 "github-text",
8034 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8035 );
8036 self.state.save()
8037 }
8038
8039 async fn judge_pr_language(
8042 &mut self,
8043 cwd: &Path,
8044 pr: &PrMessage,
8045 ) -> Option<crate::github_text::LanguageDecision> {
8046 self.state.config.roles.language_judge.as_ref()?;
8047 if !self.state.config.graph.language_gate() {
8048 return None;
8049 }
8050 let decision =
8051 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8052 self.state.event(
8053 "github-text",
8054 if decision.is_some() {
8055 "language decided by the language judge"
8056 } else {
8057 "language judge unavailable; using built-in heuristics"
8058 },
8059 );
8060 decision
8061 }
8062
8063 async fn merge(&mut self) -> Result<()> {
8064 if self
8079 .state
8080 .base_sync
8081 .as_ref()
8082 .is_some_and(|s| s.conflict.is_some())
8083 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8084 != Some(RunStatus::Gating)
8085 || !self.state.gate_status().ok()
8094 {
8095 return Ok(());
8096 }
8097 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8106 return Ok(());
8107 }
8108 let Some(winner) = self.state.winner().cloned() else {
8109 return Ok(());
8110 };
8111 let repo = self.state.repo.clone();
8112 let base = self.state.base_branch.clone();
8113 let mode = self.state.config.merge.mode;
8114 let style = self.state.config.merge.style;
8115 let facts = if is_review_run(&self.state) {
8116 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8117 let start = review_base(
8118 &repo,
8119 &self.state.config.merge.remote,
8120 &base,
8121 &self.state.base_commit,
8122 &winner.branch,
8123 )
8124 .await;
8125 branch_facts(&repo, &start, &winner.branch).await
8126 } else {
8127 None
8128 };
8129 let leaked = if is_review_run(&self.state) {
8132 leaked_subjects(&self.state, &winner.branch).await
8133 } else {
8134 Some(Vec::new())
8135 };
8136 let Some(pr) = self
8140 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8141 .await?
8142 else {
8143 return Ok(());
8144 };
8145 let message = pr.commit_message();
8146
8147 let outcome = match mode {
8148 MergeMode::None => MergeOutcome {
8149 mode,
8150 ok: true,
8151 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8152 empty: false,
8153 },
8154 MergeMode::Pr | MergeMode::Local
8155 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8156 {
8157 MergeOutcome {
8158 mode,
8159 ok: false,
8160 detail: empty_candidate_detail(&self.state, &base),
8161 empty: true,
8162 }
8163 }
8164 MergeMode::Local => {
8165 let on = git::current_branch(&repo).await?;
8166 if on.as_deref() != Some(base.as_str()) {
8167 MergeOutcome {
8168 mode,
8169 ok: false,
8170 detail: format!(
8171 "{} has {} checked out, not the base branch {base}",
8172 repo.display(),
8173 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8174 ),
8175 empty: false,
8176 }
8177 } else if !git::is_clean(&repo).await? {
8178 MergeOutcome {
8179 mode,
8180 ok: false,
8181 detail: format!("{} is dirty; refusing to merge", repo.display()),
8182 empty: false,
8183 }
8184 } else {
8185 let out = match style {
8186 MergeStyle::Merge => {
8187 git::merge_no_ff(&repo, &winner.branch, &message).await?
8188 }
8189 MergeStyle::Squash => {
8190 git::merge_squash(&repo, &winner.branch, &message).await?
8191 }
8192 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8193 };
8194 MergeOutcome {
8195 mode,
8196 ok: out.ok(),
8197 detail: if out.ok() { out.stdout } else { out.stderr },
8198 empty: false,
8199 }
8200 }
8201 }
8202 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8203 self.state.pr_publication_pending = true;
8204 self.state.save()?;
8205 let out =
8206 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8207 self.state.pr_publication_pending = out.is_err();
8208 MergeOutcome {
8209 mode,
8210 ok: out.is_ok(),
8211 detail: match out {
8212 Ok(url) => url,
8213 Err(e) => format!("{e:#}"),
8214 },
8215 empty: false,
8216 }
8217 }
8218 MergeMode::Pr => {
8219 let remote = self.state.config.merge.remote.clone();
8220 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8221 if !pushed.ok() {
8222 MergeOutcome {
8223 mode,
8224 ok: false,
8225 detail: pushed.stderr,
8226 empty: false,
8227 }
8228 } else {
8229 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8235 let out = match pr_merge_plan(found) {
8236 PrPlan::Create => {
8237 gh_pr_create(
8238 &winner.worktree,
8239 &base,
8240 &winner.branch,
8241 &pr.title,
8242 &pr.body,
8243 )
8244 .await
8245 }
8246 PrPlan::Adopt { url, title } => {
8247 self.state
8248 .event("merge", format!("Pr: adopted open pull request {url}"));
8249 if self.state.github_text.as_ref().is_some_and(|g| {
8250 g.resolved
8251 && g.choice == Some(GithubTextChoice::Rewrite)
8252 && g.rewrite.is_some()
8253 }) {
8254 land::set_pr_text_verbatim(
8257 &winner.worktree,
8258 &url,
8259 &pr.title,
8260 &pr.body,
8261 )
8262 .await
8263 .map(|()| url)
8264 } else {
8265 if title != pr.title
8266 && (!is_review_run(&self.state)
8267 || leaked
8268 .as_deref()
8269 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8270 && let Err(e) =
8271 if self.state.github_text.as_ref().is_some_and(|g| {
8272 g.resolved
8273 && g.title
8274 && g.choice == Some(GithubTextChoice::Original)
8275 }) {
8276 land::set_pr_title_verbatim(
8278 &winner.worktree,
8279 &url,
8280 &pr.title,
8281 )
8282 .await
8283 } else {
8284 land::set_pr_title(
8285 &mut self.state,
8286 &winner.worktree,
8287 &url,
8288 &pr.title,
8289 )
8290 .await
8291 }
8292 {
8293 tracing::warn!("could not refresh title of {url}: {e:#}");
8294 self.state
8295 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8296 }
8297 Ok(url)
8298 }
8299 }
8300 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8301 };
8302 match out {
8303 Ok(url) => MergeOutcome {
8304 mode,
8305 ok: true,
8306 detail: url,
8307 empty: false,
8308 },
8309 Err(e) => MergeOutcome {
8310 mode,
8311 ok: false,
8312 detail: e.to_string(),
8313 empty: false,
8314 },
8315 }
8316 }
8317 }
8318 };
8319
8320 self.state.status = match (mode, outcome.ok) {
8321 (MergeMode::None, _) => RunStatus::Ready,
8322 (_, true) => RunStatus::Merged,
8323 (_, false) => RunStatus::Blocked,
8324 };
8325 self.state.event(
8326 "merge",
8327 format!(
8328 "{:?}: {}",
8329 mode,
8330 outcome.detail.lines().next().unwrap_or("")
8331 ),
8332 );
8333 self.state.merge = Some(outcome);
8334 self.state.save()?;
8335
8336 if self.state.config.graph.land
8342 && mode == MergeMode::Pr
8343 && self.state.status == RunStatus::Merged
8344 {
8345 self.run_land().await?;
8346 }
8347 self.settle_questions();
8352 Ok(())
8353 }
8354
8355 async fn run_land(&mut self) -> Result<()> {
8366 crate::pr_head::ensure_owner(&self.state)?;
8367 let url = self
8368 .state
8369 .merge
8370 .as_ref()
8371 .map(|m| m.detail.clone())
8372 .unwrap_or_default();
8373 let url = url.lines().next().unwrap_or("").trim().to_owned();
8374 if !url.starts_with("http") {
8375 return Ok(());
8376 }
8377 loop {
8380 match land::land(&mut self.state, &url).await {
8381 Ok(_)
8382 if self
8383 .state
8384 .merge_repairs
8385 .last()
8386 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8387 {
8388 if self.repair_merge_review().await? {
8389 continue;
8390 }
8391 }
8392 Ok(pr) if self.state.parked => {
8393 let _ = pr;
8397 }
8398 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8399 self.state.status = RunStatus::Superseded;
8400 self.state.save()?;
8401 }
8402 Ok(pr) => {
8403 self.state.status = match pr.state {
8404 land::PrLifecycle::Merged => RunStatus::Merged,
8405 _ => RunStatus::Blocked,
8406 };
8407 if bump::should_release_bump(self.state.status)
8414 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8415 {
8416 self.state
8422 .event("bump", format!("release bump skipped: {e:#}"));
8423 }
8424 if self.state.status == RunStatus::Merged {
8427 crate::followup::after_merge(&mut self.state, &pr.url).await;
8428 }
8429 self.state.save()?;
8430 }
8431 Err(e) => {
8432 if e.is::<crate::repo_lock::ResourceBlocked>() {
8433 return Err(e);
8434 }
8435 self.state.status = RunStatus::Blocked;
8436 self.state.event("land", format!("gave up: {e}"));
8437 self.state.save()?;
8438 }
8439 }
8440 break;
8441 }
8442 Ok(())
8443 }
8444
8445 async fn repair_merge_review(&mut self) -> Result<bool> {
8448 use crate::run::MergeRepairPhase as Phase;
8449 crate::pr_head::ensure_owner(&self.state)?;
8450 let i = self.state.merge_repairs.len() - 1;
8451 let request = self.state.merge_repairs[i].clone();
8452 if request.phase == Phase::Fixing {
8453 let restored = match self.state.winner().cloned() {
8454 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8455 None => Err(anyhow::anyhow!("repair has no winning branch")),
8456 };
8457 let why = match restored {
8458 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8459 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8460 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8461 };
8462 self.hold_merge_repair(i, why)?;
8463 return Ok(false);
8464 }
8465 let failure = if i >= self.state.config.graph.land_rounds
8466 || self.state.config.graph.review_rounds == 0
8467 {
8468 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8469 } else {
8470 None
8471 };
8472 if let Some(why) = failure {
8473 self.hold_merge_repair(i, why)?;
8474 return Ok(false);
8475 }
8476 if request.phase == Phase::Requested {
8477 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8478 Ok(pr) => pr,
8479 Err(e) => {
8480 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8481 return Ok(false);
8482 }
8483 };
8484 self.state.merge_repairs[i].phase = Phase::Fixing;
8485 self.state.land_approval = None;
8486 self.state.land_armed_head = None;
8487 self.state.save()?;
8488 let evidence = serde_json::to_string_pretty(&request)?;
8491 let reason = format!(
8492 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8493 );
8494 let budget = self.state.config.graph.land_rounds;
8495 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8496 match result {
8497 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8498 Ok(land::Fixed::Failed(why)) => {
8499 self.hold_merge_repair(
8500 i,
8501 format!(
8502 "repair failed: {why}; inspect the branch and run an explicit review"
8503 ),
8504 )?;
8505 return Ok(false);
8506 }
8507 Err(e) => {
8508 self.hold_merge_repair(
8509 i,
8510 format!(
8511 "repair failed: {e:#}; inspect the branch and run an explicit review"
8512 ),
8513 )?;
8514 return Ok(false);
8515 }
8516 }
8517 self.begin_merge_repair_review(i)?;
8521 }
8522 self.review_loop().await?;
8523 self.gate().await?;
8524 if self.resource_halt()? || self.state.parked {
8525 return Ok(false);
8526 }
8527 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8528 != Some(RunStatus::Gating)
8529 || !self.state.gate_status().ok()
8530 {
8531 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8532 return Ok(false);
8533 }
8534 let winner = self
8537 .state
8538 .winner()
8539 .cloned()
8540 .context("repair has no winning branch")?;
8541 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8542 if self.state.gate_head.as_deref() != Some(after.as_str())
8543 || !git::is_clean(&winner.worktree).await?
8544 {
8545 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8546 return Ok(false);
8547 }
8548 let expected = self
8549 .state
8550 .pr_pushed_head
8551 .as_deref()
8552 .unwrap_or(&request.head)
8553 .to_owned();
8554 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8555 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8558 let refspec = format!("{after}:refs/heads/{public}");
8559 let push = git::git_raw(
8560 &winner.worktree,
8561 &["push", &lease, &self.state.config.merge.remote, &refspec],
8562 )
8563 .await;
8564 match push {
8565 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8566 Ok(out) => {
8567 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8568 return Ok(false);
8569 }
8570 Err(e) => {
8571 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8572 return Ok(false);
8573 }
8574 }
8575 self.state.merge_repairs[i].phase = Phase::Complete;
8576 self.state.status = RunStatus::Landing;
8577 self.state.event(
8578 "repair",
8579 "new review and gate completed; fresh owner merge decision required",
8580 );
8581 self.state.save()?;
8582 Ok(true)
8583 }
8584
8585 fn merge_repair_artifacts(&self) -> PathBuf {
8586 let artifacts = agent::artifacts_dir(&self.state.dir());
8587 match self
8588 .state
8589 .merge_repairs
8590 .last()
8591 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8592 {
8593 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8594 None => artifacts,
8595 }
8596 }
8597
8598 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8599 self.state.reviews.clear();
8600 self.state.gate.clear();
8601 self.state.gate_ran = false;
8602 self.state.gate_head = None;
8603 self.state.gate_fixes.clear();
8604 self.state.pre_gate.clear();
8605 self.state.pre_gate_commit = None;
8606 self.state.contested_handoff = None;
8607 self.state.land_reruns = None;
8608 self.state.land_approval = None;
8609 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8610 self.state.status = RunStatus::Reviewing;
8611 self.state.save()
8612 }
8613
8614 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8615 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8616 self.state.merge_repairs[index].failure = Some(why.clone());
8617 self.state.status = RunStatus::Blocked;
8618 self.state.event("repair", why);
8619 self.state.save()
8620 }
8621
8622 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8626 if let Some(existing) = self.state.seats.get(key)
8627 && existing.agent == agent
8628 {
8629 return existing.clone();
8630 }
8631 let fresh = if self.state.seats.contains_key(key) {
8634 handover_seat(key, agent, self.state.next_seat_seed())
8635 } else {
8636 SeatState::new(key, agent, self.state.seed)
8637 };
8638 self.state.seats.insert(key.to_owned(), fresh.clone());
8639 fresh
8640 }
8641
8642 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8648 match self.state.seats.get(key) {
8649 Some(s) if s.agent != spec.id => {
8650 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8651 }
8652 _ => spec,
8653 }
8654 }
8655
8656 fn view(&self, c: &Candidate) -> CandidateView {
8658 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8659 .unwrap_or_default();
8660 let (patch, _) = blind::sanitize_patch(
8661 &format!("candidate {} patch", c.label),
8662 &raw,
8663 &self.state.config.blind,
8664 );
8665 CandidateView {
8666 label: c.label,
8667 branch: c.branch.clone(),
8668 summary: c.summary.clone(),
8669 stat: c.stat.clone(),
8670 patch,
8671 }
8672 }
8673
8674 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8676 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8677 prompt::judge(
8678 "(see above)",
8679 &views,
8680 self.roles.judges.len(),
8681 base_short,
8682 "en",
8683 )
8684 }
8685
8686 fn vote_prompt_full(
8692 &self,
8693 j: usize,
8694 labels: &[char],
8695 language: &str,
8696 candidates: &[Candidate],
8697 base_short: &str,
8698 ) -> String {
8699 let mut text = format!(
8700 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8701 prompt::final_vote(labels, language),
8702 self.state.instruction,
8703 self.candidate_block(candidates, base_short)
8704 );
8705 if let Some(own) = self
8706 .state
8707 .judgements
8708 .get(j)
8709 .filter(|r| !r.ranking.is_empty())
8710 {
8711 let reasons = own
8712 .reasons
8713 .iter()
8714 .map(|(k, v)| format!("- {k}: {v}"))
8715 .collect::<Vec<_>>()
8716 .join("\n");
8717 text.push_str(&format!(
8718 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8719 own.ranking.iter().collect::<String>(),
8720 if reasons.is_empty() {
8721 ""
8722 } else {
8723 ", because:\n"
8724 }
8725 ));
8726 }
8727 if !self.state.deliberation.is_empty() {
8728 text.push_str("\n# What was argued before this vote\n");
8729 for t in self.transcript(&[], j) {
8730 text.push_str(&format!(
8731 "\n## {}{}\n\n{}\n",
8732 t.who,
8733 if t.is_self { " (you)" } else { "" },
8734 t.body.trim()
8735 ));
8736 }
8737 }
8738 text
8739 }
8740
8741 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8748 let mut turns = Vec::new();
8749 for j in &self.state.judgements {
8750 if j.ranking.is_empty() {
8751 continue;
8752 }
8753 let reasons = j
8754 .reasons
8755 .iter()
8756 .map(|(k, v)| format!("- {k}: {v}"))
8757 .collect::<Vec<_>>()
8758 .join("\n");
8759 turns.push(Turn {
8760 who: format!("Judge {} (opening ranking)", j.judge),
8761 is_self: j.judge == self_idx + 1,
8762 body: format!(
8763 "Ranked {}{}{reasons}",
8764 j.ranking.iter().collect::<String>(),
8765 if reasons.is_empty() {
8766 ""
8767 } else {
8768 ", because:\n"
8769 }
8770 ),
8771 });
8772 }
8773 for t in self
8774 .state
8775 .deliberation
8776 .iter()
8777 .flat_map(|r| r.turns.iter())
8778 .chain(current)
8779 {
8780 turns.push(Turn {
8781 who: format!("Judge {}", t.judge),
8782 is_self: t.judge == self_idx + 1,
8783 body: t.body.clone(),
8784 });
8785 }
8786 turns
8787 }
8788}
8789
8790fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8792 agent::has_session(spec.kind, seat, sessions)
8793}
8794
8795fn next_untried_in_roster<'a>(
8816 roster: &'a [AgentSpec],
8817 start: usize,
8818 tried: &BTreeSet<String>,
8819) -> Option<&'a AgentSpec> {
8820 roster
8821 .get(start + 1..)?
8822 .iter()
8823 .find(|s| !tried.contains(&s.id))
8824}
8825
8826fn pick_successor<'a>(
8847 roster: &'a [AgentSpec],
8848 start: usize,
8849 tried: &BTreeSet<String>,
8850 carried: Option<&BTreeSet<String>>,
8851 others: &BTreeSet<String>,
8852) -> Option<&'a AgentSpec> {
8853 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8854 roster
8855 .get(start + 1..)
8856 .and_then(|tail| {
8857 tail.iter()
8858 .filter(free)
8859 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8860 })
8861 .or_else(|| {
8862 carried
8863 .is_some()
8864 .then(|| roster.iter().find(free))
8865 .flatten()
8866 })
8867 .or_else(|| match carried {
8868 Some(c) => next_for_seat(roster, start, tried, c),
8869 None => next_untried_in_roster(roster, start, tried),
8870 })
8871}
8872
8873fn next_for_seat<'a>(
8883 roster: &'a [AgentSpec],
8884 start: usize,
8885 round_tried: &BTreeSet<String>,
8886 carried_failed: &BTreeSet<String>,
8887) -> Option<&'a AgentSpec> {
8888 roster
8889 .get(start + 1..)?
8890 .iter()
8891 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8892 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8893}
8894
8895fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8901 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8902 return spec;
8903 };
8904 let ok = |id: &str| !h.failed.contains(id);
8905 if let Some(last) = h.last_ok.as_deref()
8906 && ok(last)
8907 && let Some(s) = roster.iter().find(|s| s.id == last)
8908 {
8909 return s.clone();
8910 }
8911 if ok(&spec.id) {
8912 return spec;
8913 }
8914 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8915 roster
8916 .get(from + 1..)
8917 .into_iter()
8918 .flatten()
8919 .chain(roster.iter())
8920 .find(|s| ok(&s.id))
8921 .cloned()
8922 .unwrap_or(spec)
8923}
8924
8925pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8929 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8930}
8931
8932const TIMED_OUT: &str = "timed out";
8935
8936impl FailClass {
8937 fn of(out: &AgentOutcome) -> Option<Self> {
8939 match out {
8940 AgentOutcome::Ok(_) => None,
8941 AgentOutcome::Quota(_) => Some(Self::Quota),
8942 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8943 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8944 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8945 AgentOutcome::ResourceBlocked(_) => None,
8948 }
8949 }
8950
8951 fn stem_word(&self) -> &'static str {
8953 match self {
8954 Self::Quota => "quota",
8955 _ => "handover",
8956 }
8957 }
8958}
8959
8960fn failure_signature(msg: &str) -> String {
8964 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8965 if let Some(code) = line
8968 .strip_prefix("process exited with no usable output (exit code ")
8969 .and_then(|s| s.strip_suffix(')'))
8970 .and_then(|s| s.parse::<i32>().ok())
8971 {
8972 line = format!("exited with some({code}) and no usable output");
8973 } else if matches!(
8974 line.as_str(),
8975 "process exited with no usable output (no exit code reported)"
8976 | "process exited with no usable output (exit code unavailable)"
8977 ) {
8978 line = "exited with none and no usable output".to_owned();
8979 }
8980 let mut out = Vec::new();
8981 for word in line.split_whitespace() {
8982 if word.contains('/') || word.contains('\\') {
8983 out.push("<path>".to_owned());
8984 continue;
8985 }
8986 let mut w = String::new();
8987 let mut in_digits = false;
8988 for c in word.chars() {
8989 if c.is_ascii_digit() {
8990 if !in_digits {
8991 w.push('#');
8992 }
8993 in_digits = true;
8994 } else {
8995 in_digits = false;
8996 w.push(c);
8997 }
8998 }
8999 out.push(w);
9000 }
9001 out.join(" ").chars().take(120).collect()
9002}
9003
9004fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9011 match cur {
9012 FailClass::Quota | FailClass::Timeout => true,
9013 FailClass::Other(_) => prev != Some(cur),
9014 }
9015}
9016
9017fn fail_reason(out: &AgentOutcome) -> String {
9019 match out {
9020 AgentOutcome::Ok(_) => String::new(),
9021 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9022 AgentOutcome::Dropped(o) => format!(
9023 "the CLI dropped the stream ({})",
9024 o.dropped
9025 .as_ref()
9026 .map(|d| d.why.as_str())
9027 .unwrap_or("it ended without delivering its answer")
9028 ),
9029 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9030 e.lines().next().unwrap_or("").chars().take(160).collect()
9031 }
9032 }
9033}
9034
9035pub(crate) fn record_handover(
9038 state: &mut RunState,
9039 node: &str,
9040 seat: &str,
9041 from: &str,
9042 to: &str,
9043 class: &FailClass,
9044 reason: &str,
9045) {
9046 let message = if *class == FailClass::Quota {
9047 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9048 } else {
9049 format!("{seat}: handed over {from} -> {to} ({reason})")
9050 };
9051 state.event(node, message);
9052 state.handovers.push(Handover {
9053 at: Timestamp::now(),
9054 node: node.to_owned(),
9055 seat: seat.to_owned(),
9056 from: from.to_owned(),
9057 to: to.to_owned(),
9058 reason: reason.to_owned(),
9059 });
9060}
9061
9062fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9076 commands.iter().any(|c| c.exit_code.is_none())
9077}
9078
9079fn verified_noop_claim(
9092 usable: bool,
9093 commands: &[agent::CommandEvidence],
9094 text: &str,
9095) -> Option<String> {
9096 (usable && !has_unconfirmed_command(commands))
9097 .then(|| verdict::verified_noop(text))
9098 .flatten()
9099}
9100
9101fn short(commit: &str) -> String {
9102 commit.chars().take(7).collect()
9103}
9104
9105fn make_executable(path: &Path) -> Result<()> {
9106 #[cfg(unix)]
9107 {
9108 use std::os::unix::fs::PermissionsExt as _;
9109 let mut perms = std::fs::metadata(path)?.permissions();
9110 perms.set_mode(0o755);
9111 std::fs::set_permissions(path, perms)?;
9112 }
9113 #[cfg(not(unix))]
9114 {
9115 let _ = path;
9116 }
9117 Ok(())
9118}
9119
9120struct WaveCtx<'a> {
9127 run: &'a str,
9130 node: &'a str,
9132 prompts: &'a Prompts,
9133 cache: Option<&'a Path>,
9135 round: Option<usize>,
9138 carry_seats: bool,
9142 pauses: Option<(&'a Pause, &'a Pause)>,
9145}
9146
9147fn repository_wait(
9149 state: &RunState,
9150 pauses: Option<(&Pause, &Pause)>,
9151) -> crate::repo_lock::WaitPolicy {
9152 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9153 let inherited = crate::repo_lock::WaitPolicy::current();
9154 crate::repo_lock::WaitPolicy::new(
9155 Duration::from_secs(state.config.graph.timeout_resource_wait),
9156 move || {
9157 inherited.cancelled()
9158 || pauses
9159 .as_ref()
9160 .is_some_and(|(a, b)| a.parked() || b.parked())
9161 },
9162 )
9163}
9164
9165fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9166 if error.is::<crate::repo_lock::ResourceBlocked>() {
9167 state.resource_block = Some(crate::run::ResourceBlock {
9168 node: "repository".into(),
9169 kind: "repository coordination".into(),
9170 detail: error.to_string(),
9171 at: Timestamp::now(),
9172 });
9173 state.event("resource", error.to_string());
9174 }
9175}
9176
9177async fn coordinate_operation(
9178 state: &mut RunState,
9179 cwd: &Path,
9180 wait: &crate::repo_lock::WaitPolicy,
9181) -> Result<Option<crate::repo_lock::SeatLease>> {
9182 let result =
9183 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9184 .await;
9185 if let Err(e) = &result {
9186 record_repository_block(state, e);
9187 }
9188 result
9189}
9190
9191async fn run_one(
9193 job: SeatJob,
9194 sem: Arc<Semaphore>,
9195 ctx: &WaveCtx<'_>,
9196 state: &mut RunState,
9197 attempt: usize,
9198) -> (SeatState, AgentOutcome) {
9199 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9200 .await
9201 .pop()
9202 .expect("one job in, one result out");
9203 (seat, out)
9204}
9205
9206async fn wave(
9212 jobs: Vec<SeatJob>,
9213 sem: Arc<Semaphore>,
9214 ctx: &WaveCtx<'_>,
9215 state: &mut RunState,
9216 attempt: usize,
9217) -> Vec<(usize, SeatState, AgentOutcome)> {
9218 let WaveCtx {
9219 run,
9220 node,
9221 prompts,
9222 cache,
9223 round,
9224 carry_seats: _,
9225 pauses,
9226 } = *ctx;
9227 for job in &jobs {
9228 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9229 }
9230 if let Err(e) = state.save() {
9231 tracing::warn!("could not persist in-progress seats: {e:#}");
9236 }
9237 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9253 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9257 let cache_guard = if let Some(cache_dir) = cache {
9258 if jobs_had_a_writer {
9259 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9260 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9261 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9262 {
9263 Ok(Some(g)) => Some(g),
9264 Ok(None) => {
9267 state.resource_block = Some(crate::run::ResourceBlock {
9268 node: node.to_owned(),
9269 kind: "shared build cache".to_owned(),
9270 detail: "the run is parking".to_owned(),
9271 at: Timestamp::now(),
9272 });
9273 return block_wave(
9274 jobs,
9275 state,
9276 node,
9277 attempt,
9278 "parked while waiting for the shared build cache",
9279 );
9280 }
9281 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9284 Err(e) => {
9285 let why = format!("{e:#}");
9289 state.resource_block = Some(crate::run::ResourceBlock {
9290 node: node.to_owned(),
9291 kind: "shared build cache".to_owned(),
9292 detail: why.clone(),
9293 at: Timestamp::now(),
9294 });
9295 state.event(
9296 "resource",
9297 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9298 );
9299 return block_wave(jobs, state, node, attempt, &why);
9300 }
9301 }
9302 } else {
9303 None
9304 }
9305 } else {
9306 None
9307 };
9308 state.seats_spawning(node, attempt);
9311 if let Err(e) = state.save() {
9312 tracing::warn!("could not persist the start of a wave: {e:#}");
9313 }
9314 let slot_limit = resource_limit
9318 + jobs
9319 .iter()
9320 .map(|j| j.timeout)
9321 .max()
9322 .unwrap_or_default()
9323 .saturating_mul(jobs.len() as u32);
9324 let mut set = tokio::task::JoinSet::new();
9325 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9328 let overlay = prompts.overlay(node);
9329 let repository_wait = repository_wait(state, pauses);
9330 for (i, mut job) in jobs.into_iter().enumerate() {
9331 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9332 if cache.is_some() {
9333 job.prompt.push('\n');
9334 job.prompt
9335 .push_str(&prompt::build_cache_note(node, job.allow_write));
9336 }
9337 let sem = Arc::clone(&sem);
9338 let run = run.to_owned();
9339 let node = node.to_owned();
9340 let attachments = if node == "implement" {
9343 state.attachments.clone()
9344 } else {
9345 Vec::new()
9346 };
9347 let cache = cache
9358 .filter(|_| job.allow_write && cache_guard.is_some())
9359 .map(Path::to_path_buf);
9360 let spawned_tx = spawned_tx.clone();
9361 let repository_wait = repository_wait.clone();
9362 let coordination = state.config.coordination.clone();
9363 let fetched_base = state.fetched_base.clone();
9364 set.spawn(async move {
9365 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9369 Ok(p) => p,
9370 Err(_) => {
9371 return (
9372 i,
9373 job.seat,
9374 AgentOutcome::ResourceBlocked(format!(
9375 "no parallel slot became free within {}s",
9376 slot_limit.as_secs()
9377 )),
9378 );
9379 }
9380 };
9381 spawned_tx.send(job.seat.key.clone()).ok();
9382 let mut seat = job.seat;
9383 let out = repository_wait
9384 .scope(agent::invoke(
9385 &job.spec,
9386 &mut seat,
9387 &Invocation {
9388 scope: crate::agent::InvocationScope::Repository,
9389 coordination: coordination.as_ref(),
9390 fetched_base: fetched_base.as_ref(),
9391 cwd: &job.cwd,
9392 prompt: &job.prompt,
9393 timeout: job.timeout,
9394 allow_write: job.allow_write,
9395 unsandboxed: false,
9396 sessions: job.sessions,
9397 artifacts: &job.artifacts,
9398 stem: &job.stem,
9399 run: &run,
9400 node: &node,
9401 cache_dir: cache.as_deref(),
9402 attachments: &attachments,
9403 writable: &[],
9404 },
9405 ))
9406 .await;
9407 let out = match out {
9408 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9409 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9410 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9418 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9419 Ok(o) => AgentOutcome::Failed(format!(
9420 "process exited with no usable output ({})",
9421 crate::notices::exit_status(o.exit_code)
9422 )),
9423 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9424 AgentOutcome::ResourceBlocked(e.to_string())
9425 }
9426 Err(e) => AgentOutcome::Failed(e.to_string()),
9427 };
9428 (i, seat, out)
9429 });
9430 }
9431 drop(spawned_tx);
9432 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9433 loop {
9434 let joined = tokio::select! {
9435 Some(key) = spawned_rx.recv() => {
9436 state.seat_spawned(&key);
9438 if let Err(e) = state.save() {
9439 tracing::warn!("could not persist a seat's start: {e:#}");
9440 }
9441 continue;
9442 }
9443 joined = set.join_next() => joined,
9444 };
9445 let Some(joined) = joined else {
9446 break;
9447 };
9448 let (i, seat, out) = match joined {
9449 Ok(v) => v,
9450 Err(e) => {
9454 tracing::error!("agent task panicked: {e}");
9455 continue;
9456 }
9457 };
9458 state.seat_finished(&seat.key);
9459 if let AgentOutcome::ResourceBlocked(why) = &out
9460 && state.resource_block.is_none()
9461 {
9462 state.resource_block = Some(crate::run::ResourceBlock {
9463 node: node.to_owned(),
9464 kind: if why.contains("repository coordination") {
9465 "repository coordination"
9466 } else {
9467 "parallel slot"
9468 }
9469 .to_owned(),
9470 detail: why.clone(),
9471 at: Timestamp::now(),
9472 });
9473 state.event("resource", format!("{node}: no agent was started - {why}"));
9474 }
9475 record_jobs(state, node, round, &seat.key, &out);
9476 if let Err(e) = state.save() {
9477 tracing::warn!("could not persist a seat's completion: {e:#}");
9478 }
9479 if collected.len() <= i {
9480 collected.resize_with(i + 1, || None);
9481 }
9482 collected[i] = Some((i, seat, out));
9483 }
9484 if state
9490 .active
9491 .values()
9492 .any(|a| a.node == node && a.attempt == attempt)
9493 {
9494 state
9495 .active
9496 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9497 if let Err(e) = state.save() {
9498 tracing::warn!("could not persist the end of a wave: {e:#}");
9499 }
9500 }
9501 if let Some(cache_dir) = cache
9508 && jobs_had_a_writer
9509 {
9510 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9511 }
9512 if let Some(guard) = cache_guard {
9513 guard.release();
9514 }
9515 collected.into_iter().flatten().collect()
9516}
9517
9518fn record_jobs(
9529 state: &mut RunState,
9530 node: &str,
9531 round: Option<usize>,
9532 seat: &str,
9533 out: &AgentOutcome,
9534) {
9535 let commands: &[agent::CommandEvidence] = match out {
9536 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9537 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9538 };
9539 let checked_at = Timestamp::now();
9540 for c in commands {
9541 state.jobs.push(JobRecord {
9542 node: node.to_owned(),
9543 round,
9544 seat: seat.to_owned(),
9545 id: c.id.clone(),
9546 description: c.description.clone(),
9547 checked_at,
9548 status: match c.exit_code {
9549 Some(0) => JobStatus::Completed,
9550 Some(_) => JobStatus::Failed,
9551 None => JobStatus::Unknown,
9552 },
9553 exit_code: c.exit_code,
9554 result_summary: c.result_summary.clone(),
9555 source: c.source.clone(),
9556 });
9557 }
9558}
9559
9560fn round_is_clean(
9581 blocking: usize,
9582 e2e_ok: bool,
9583 answered: usize,
9584 expected: usize,
9585 quota_missing: usize,
9586 policy: IncompleteReviewPolicy,
9587) -> bool {
9588 if blocking != 0 || !e2e_ok {
9589 return false;
9590 }
9591 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9592 return true;
9593 }
9594 answered > 0 && expected - answered <= quota_missing
9595}
9596
9597fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9621 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9622 return Some(RunStatus::Gating);
9623 }
9624 let last = reviews.last()?;
9625 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9626 if reviews.len() < max_rounds && !stagnant {
9627 return None;
9628 }
9629 if last.incomplete() && last.blocking == 0 {
9630 return Some(RunStatus::Blocked);
9631 }
9632 if last.e2e_status() == E2eStatus::ResourceBlocked {
9633 return None;
9634 }
9635 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9636 RunStatus::Gating
9637 } else {
9638 RunStatus::Blocked
9639 })
9640}
9641
9642pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9657 if nudged {
9658 (full / 4).max(Duration::from_secs(120)).min(full)
9659 } else {
9660 full
9661 }
9662}
9663
9664#[allow(clippy::too_many_arguments)]
9693async fn ask_json_wave<T>(
9694 jobs: Vec<SeatJob>,
9695 sem: Arc<Semaphore>,
9696 retries: usize,
9697 roster: &[AgentSpec],
9698 ctx: &WaveCtx<'_>,
9699 losses: &mut Vec<QuotaLoss>,
9700 state: &mut RunState,
9701 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9702) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9703where
9704 T: serde::de::DeserializeOwned + Send + 'static,
9705{
9706 ask_wave_with(
9707 jobs,
9708 sem,
9709 retries,
9710 roster,
9711 ctx,
9712 losses,
9713 state,
9714 &|text: &str| {
9715 let v = verdict::extract_json::<T>(text)?;
9716 validate(&v)?;
9717 Ok(v)
9718 },
9719 )
9720 .await
9721}
9722
9723#[allow(clippy::too_many_arguments)]
9732async fn ask_wave_with<T>(
9733 jobs: Vec<SeatJob>,
9734 sem: Arc<Semaphore>,
9735 retries: usize,
9736 roster: &[AgentSpec],
9737 ctx: &WaveCtx<'_>,
9738 losses: &mut Vec<QuotaLoss>,
9739 state: &mut RunState,
9740 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9741) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9742where
9743 T: Send + 'static,
9744{
9745 let n = jobs.len();
9746 let originals: Vec<SeatJob> = jobs;
9747 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9748 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9749 let mut nudges: Vec<usize> = vec![0; n];
9756 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9760 let mut tried: Vec<BTreeSet<String>> = specs
9761 .iter()
9762 .map(|s| BTreeSet::from([s.id.clone()]))
9763 .collect();
9764 let starts: Vec<usize> = specs
9765 .iter()
9766 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9767 .collect();
9768 let carry = ctx.carry_seats && !roster.is_empty();
9769 let carried: Vec<BTreeSet<String>> = originals
9772 .iter()
9773 .map(|j| {
9774 state
9775 .seat_history
9776 .get(&j.seat.key)
9777 .filter(|_| carry)
9778 .map(|h| h.failed.clone())
9779 .unwrap_or_default()
9780 })
9781 .collect();
9782 let mut prev: Vec<Option<FailClass>> = originals
9783 .iter()
9784 .map(|j| {
9785 state
9786 .seat_history
9787 .get(&j.seat.key)
9788 .filter(|_| carry)
9789 .and_then(|h| h.last_fail.clone())
9790 })
9791 .collect();
9792 let next_agent =
9793 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9794 let others: BTreeSet<String> = specs
9795 .iter()
9796 .enumerate()
9797 .filter(|(j, _)| *j != i)
9798 .map(|(_, s)| s.id.clone())
9799 .collect();
9800 pick_successor(
9801 roster,
9802 starts[i],
9803 tried,
9804 carry.then(|| &carried[i]),
9805 &others,
9806 )
9807 .cloned()
9808 };
9809 let mut fresh: Vec<Option<String>> = vec![None; n];
9810 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9811 let mut pending: Vec<usize> = (0..n).collect();
9812
9813 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9818 for round in 0..max_rounds {
9819 if pending.is_empty() {
9820 break;
9821 }
9822 let mut batch = Vec::with_capacity(pending.len());
9823 let mut renudged: Vec<&str> = Vec::new();
9824 for &i in &pending {
9825 let src = &originals[i];
9826 let full: &str = match &src.handover {
9831 Some(h) if specs[i].id != src.spec.id => h,
9832 _ => &src.prompt,
9833 };
9834 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9837 (
9838 full.to_owned(),
9839 src.timeout,
9840 format!("{}-{word}-{}", src.stem, specs[i].id),
9841 )
9842 } else if nudges[i] == 0 {
9843 (full.to_owned(), src.timeout, src.stem.clone())
9844 } else {
9845 renudged.push(src.seat.key.as_str());
9846 let why = done[i]
9847 .as_ref()
9848 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9849 .unwrap_or_else(|| "no parsable answer".to_owned());
9850 let nudge = prompt::nudge(&why);
9851 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9852 let prompt = if nudged {
9853 nudge
9854 } else {
9855 format!("{full}\n\n---\n\n{nudge}")
9856 };
9857 (
9858 prompt,
9859 retry_budget(src.timeout, nudged),
9860 format!("{}-retry{}", src.stem, nudges[i]),
9861 )
9862 };
9863 batch.push(SeatJob {
9864 spec: specs[i].clone(),
9865 seat: seats[i].clone(),
9866 cwd: src.cwd.clone(),
9867 prompt,
9868 timeout,
9869 allow_write: src.allow_write,
9870 sessions: src.sessions,
9871 artifacts: src.artifacts.clone(),
9872 stem,
9873 handover: None,
9874 });
9875 }
9876
9877 if !renudged.is_empty() {
9878 state.event(
9879 ctx.node,
9880 format!("retry {round}: re-asking {}", renudged.join(", ")),
9881 );
9882 }
9883 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9884 let mut still = Vec::new();
9885 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9886 seats[i] = seat;
9887 if let AgentOutcome::ResourceBlocked(why) = &out {
9890 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9891 continue;
9892 }
9893 let class = FailClass::of(&out);
9894 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9897 && nudges[i] < retries
9898 && !roster.is_empty();
9899 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9900 let next = should_hand_over(prev[i].as_ref(), &cur)
9901 .then(|| next_agent(i, &tried[i], &specs))
9902 .flatten();
9903 if carry {
9904 let h = state
9905 .seat_history
9906 .entry(originals[i].seat.key.clone())
9907 .or_default();
9908 h.failed.insert(specs[i].id.clone());
9909 h.last_fail = Some(cur.clone());
9910 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9911 h.last_ok = None;
9912 }
9913 if let Err(e) = state.save() {
9916 tracing::warn!("could not persist a seat's failure history: {e:#}");
9917 }
9918 }
9919 if let Some(next) = next {
9920 record_handover(
9921 state,
9922 ctx.node,
9923 &originals[i].seat.key,
9924 &specs[i].id,
9925 &next.id,
9926 &cur,
9927 &fail_reason(&out),
9928 );
9929 tried[i].insert(next.id.clone());
9930 prev[i] = Some(cur.clone());
9931 seats[i] =
9932 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9933 specs[i] = next;
9934 fresh[i] = Some(cur.stem_word().to_owned());
9935 nudges[i] = 0;
9936 done[i] = Some(Err(anyhow::anyhow!(
9937 "handed over after: {}",
9938 fail_reason(&out)
9939 )));
9940 still.push(i);
9941 continue;
9942 }
9943 }
9944 if carry
9945 && !nudge_first
9946 && let Some(cur) = class.clone()
9947 {
9948 let h = state
9950 .seat_history
9951 .entry(originals[i].seat.key.clone())
9952 .or_default();
9953 h.failed.insert(specs[i].id.clone());
9954 h.last_fail = Some(cur);
9955 }
9956 let parsed = match out {
9957 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9958 AgentOutcome::Quota(o) => {
9959 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9960 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9961 }
9962 AgentOutcome::Dropped(o) => {
9966 let why = o
9967 .dropped
9968 .as_ref()
9969 .map(|d| d.why.as_str())
9970 .unwrap_or("the CLI ended the stream without delivering its answer");
9971 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9972 }
9973 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9974 Err(anyhow::anyhow!(e))
9975 }
9976 };
9977 let quota = class == Some(FailClass::Quota);
9978 let failed = parsed.is_err();
9979 done[i] = Some(parsed);
9980 if carry && !failed {
9981 let h = state
9982 .seat_history
9983 .entry(originals[i].seat.key.clone())
9984 .or_default();
9985 h.failed.remove(&specs[i].id);
9986 h.last_ok = Some(specs[i].id.clone());
9987 h.last_fail = None;
9988 }
9989 let agent_failure = class.is_some()
9997 && !nudge_first
9998 && !roster.is_empty()
9999 && next_agent(i, &tried[i], &specs).is_some();
10000 if failed && !quota && !agent_failure && nudges[i] < retries {
10001 nudges[i] += 1;
10002 still.push(i);
10003 }
10004 }
10005 pending = still;
10006 }
10007
10008 for (i, q) in last_quota.into_iter().enumerate() {
10011 if let Some(reset) = q {
10012 losses.push(QuotaLoss {
10013 seat: originals[i].seat.key.clone(),
10014 node: ctx.node.to_owned(),
10015 at: Timestamp::now(),
10016 reset,
10017 });
10018 }
10019 }
10020
10021 seats
10022 .into_iter()
10023 .zip(done)
10024 .zip(nudges)
10025 .map(|((seat, res), attempts)| {
10026 (
10027 seat,
10028 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10029 attempts,
10030 )
10031 })
10032 .collect()
10033}
10034
10035fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10037 results
10038 .iter()
10039 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10040}
10041
10042fn block_wave(
10046 jobs: Vec<SeatJob>,
10047 state: &mut RunState,
10048 node: &str,
10049 attempt: usize,
10050 why: &str,
10051) -> Vec<(usize, SeatState, AgentOutcome)> {
10052 state.resource_wait = None;
10053 state
10054 .active
10055 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10056 if let Err(e) = state.save() {
10057 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10058 }
10059 jobs.into_iter()
10060 .enumerate()
10061 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10062 .collect()
10063}
10064
10065async fn acquire_cache_lease(
10078 state: &mut RunState,
10079 cache_dir: &Path,
10080 owner: &crate::cache::Owner,
10081 budget: Duration,
10082 context: &str,
10083 cancel: &(dyn Fn() -> bool + Sync),
10084) -> Result<Option<crate::cache::Guard>> {
10085 let home = crate::run::home();
10086 let started = Instant::now();
10087 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10088 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10089 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10090 Err(e) => {
10091 state.event(
10092 "verify",
10093 format!("{context}: could not check the shared build cache: {e:#}"),
10094 );
10095 if let Err(e2) = state.save() {
10096 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10097 }
10098 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10099 }
10100 };
10101 state.event(
10102 "verify",
10103 format!(
10104 "{context}: waiting for the shared build cache at {} ({})",
10105 cache_dir.display(),
10106 busy.describe()
10107 ),
10108 );
10109 state.resource_wait = Some(crate::run::ResourceWait {
10112 node: context.to_owned(),
10113 kind: "shared build cache".to_owned(),
10114 holder: busy.describe(),
10115 since: Timestamp::now(),
10116 limit_secs: budget.as_secs(),
10117 });
10118 if let Err(e) = state.save() {
10119 tracing::warn!("could not persist a cache wait: {e:#}");
10120 }
10121 let remaining = budget.saturating_sub(started.elapsed());
10122 let waited = crate::cache::wait_for_unless(
10123 &home,
10124 cache_dir,
10125 owner,
10126 remaining,
10127 Duration::from_secs(5),
10128 cancel,
10129 )
10130 .await;
10131 state.resource_wait = None;
10132 match waited {
10133 Ok(g) => Ok(g),
10134 Err(e) => {
10135 state.event("verify", format!("{context}: {e:#}"));
10136 if let Err(e2) = state.save() {
10137 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10138 }
10139 Err(e)
10140 }
10141 }
10142}
10143
10144#[derive(Debug)]
10149struct CacheCheckError(String);
10150
10151impl std::fmt::Display for CacheCheckError {
10152 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10153 f.write_str(&self.0)
10154 }
10155}
10156
10157impl std::error::Error for CacheCheckError {}
10158
10159#[allow(clippy::too_many_arguments)]
10180async fn with_cache_lease<'s, F, Fut>(
10181 state: &'s mut RunState,
10182 cache_dir: Option<&Path>,
10183 node: &str,
10184 seat: &str,
10185 worktree: &Path,
10186 head: &str,
10187 budget: Duration,
10188 context: &str,
10189 wait: &crate::repo_lock::WaitPolicy,
10190 body: F,
10191) -> Result<(Vec<CommandOutcome>, bool)>
10192where
10193 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10194 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10195{
10196 let started = Instant::now();
10199 let home = crate::run::home();
10200 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10201 let guard = if let Some(cache_dir) = cache_dir {
10202 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10203 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10204 wait.cancelled()
10205 })
10206 .await
10207 .and_then(|g| {
10208 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10209 }) {
10210 Ok(g) => Some(g),
10211 Err(e) => {
10212 return Ok((
10213 vec![CommandOutcome {
10214 command: "(waiting for the shared build cache)".into(),
10215 code: None,
10216 output_tail: e.to_string(),
10217 duration_ms: started.elapsed().as_millis() as u64,
10218 resource_blocked: true,
10219 }],
10220 false,
10221 ));
10222 }
10223 }
10224 } else {
10225 None
10226 };
10227 let _coordination = coordinate_operation(state, worktree, wait).await?;
10228 if let Some(cache_dir) = cache_dir {
10229 let identity = crate::cache::Identity::new(worktree, head);
10230 if let Err(e) = crate::cache::ensure_fresh(
10231 &home,
10232 cache_dir,
10233 &identity,
10234 &state.config.verify.cargo_manifests(),
10235 ) {
10236 state.event(
10237 "verify",
10238 format!(
10239 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10240 worktree.display(),
10241 short(head)
10242 ),
10243 );
10244 if let Some(guard) = guard {
10245 guard.release();
10246 }
10247 return Ok((
10248 vec![CommandOutcome {
10249 command: "(confirming the shared build cache is fresh)".into(),
10250 code: None,
10251 output_tail: e.to_string(),
10252 duration_ms: started.elapsed().as_millis() as u64,
10253 resource_blocked: true,
10254 }],
10255 false,
10256 ));
10257 }
10258 }
10259 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10260 if !timed_out_pids.is_empty() {
10261 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10262 }
10263 drop(_coordination);
10266 if let Some(guard) = guard {
10267 guard.release();
10268 }
10269 Ok((outcomes, retried))
10270}
10271
10272async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10284 wait_for_pids_with(
10285 pids,
10286 crate::proc::pid_alive,
10287 LEASE_RELEASE_POLL,
10288 LEASE_RELEASE_MAX_WAIT,
10289 )
10290 .await;
10291}
10292
10293async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10299 pids: &[u32],
10300 alive: F,
10301 poll: Duration,
10302 max_wait: Duration,
10303) {
10304 let deadline = Instant::now() + max_wait;
10305 loop {
10306 if pids.iter().all(|&pid| !alive(pid)) {
10307 return;
10308 }
10309 if Instant::now() >= deadline {
10310 return;
10311 }
10312 tokio::time::sleep(poll).await;
10313 }
10314}
10315
10316fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10323 outcomes.iter().any(|o| o.resource_blocked)
10324}
10325
10326enum GateFix {
10328 Retry,
10330 Stop,
10333 Defer,
10336}
10337
10338fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10346 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10347 red.peek().is_some()
10348 && red.all(|o| {
10349 !o.resource_blocked
10350 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10351 && !o.output_tail.trim().is_empty()
10352 })
10353}
10354
10355fn e2e_outcome_label(o: &CommandOutcome) -> String {
10359 if o.ok() {
10360 return "pass".to_owned();
10361 }
10362 let reason = if o.build_failed() {
10363 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10364 } else {
10365 format!("FAIL ({:?})", o.code)
10366 };
10367 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10368}
10369
10370async fn run_e2e_with_retry(
10378 state: &mut RunState,
10379 shell: &[String],
10380 commands: &[String],
10381 worktree: &Path,
10382 timeout: Duration,
10383 context: &str,
10384) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10385 let (mut e2e, mut timed_out_pids) = run_commands(
10386 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10387 )
10388 .await;
10389 for o in &e2e {
10390 state.event(
10391 "verify",
10392 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10393 );
10394 }
10395 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10399 if verify_retried {
10400 state.event(
10401 "verify",
10402 format!(
10403 "{context}: verify could not build/link, not a test result — retrying once \
10404 before concluding"
10405 ),
10406 );
10407 let retried = run_commands(
10408 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10409 )
10410 .await;
10411 e2e = retried.0;
10412 timed_out_pids.extend(retried.1);
10415 for o in &e2e {
10416 state.event(
10417 "verify",
10418 format!(
10419 "{context}: retry `{}` -> {}",
10420 o.command,
10421 e2e_outcome_label(o)
10422 ),
10423 );
10424 }
10425 }
10426 (e2e, verify_retried, timed_out_pids)
10427}
10428
10429#[allow(clippy::too_many_arguments)]
10445async fn run_commands(
10446 state: &mut RunState,
10447 node: &str,
10448 task: &str,
10449 attempt: usize,
10450 shell: &[String],
10451 commands: &[String],
10452 cwd: &Path,
10453 timeout: Duration,
10454) -> (Vec<CommandOutcome>, Vec<u32>) {
10455 if commands.is_empty() {
10456 return (Vec::new(), Vec::new());
10461 }
10462 let mut out = Vec::new();
10463 let mut timed_out_pids = Vec::new();
10464 let total = commands.len();
10465 for (idx, command) in commands.iter().enumerate() {
10466 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10467 if let Err(e) = state.save() {
10468 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10469 }
10470 let started = Instant::now();
10471 let mut cmd = tokio::process::Command::new(&shell[0]);
10472 cmd.quiet();
10473 cmd.args(&shell[1..])
10474 .arg(command)
10475 .current_dir(cwd)
10476 .stdin(std::process::Stdio::null())
10477 .stdout(std::process::Stdio::piped())
10478 .stderr(std::process::Stdio::piped())
10479 .kill_on_drop(true);
10480 let spawned = cmd.spawn();
10481 let (code, body) = match spawned {
10482 Ok(child) => {
10483 let pid = child.id();
10488 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10489 Ok(Ok(o)) => {
10490 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10491 body.push_str(&String::from_utf8_lossy(&o.stderr));
10492 (o.status.code(), body)
10493 }
10494 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10495 Err(_) => {
10496 if let Some(pid) = pid {
10497 timed_out_pids.push(pid);
10498 }
10499 (None, format!("timed out after {}s", timeout.as_secs()))
10500 }
10501 }
10502 }
10503 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10504 };
10505 out.push(CommandOutcome {
10506 command: command.clone(),
10507 code,
10508 output_tail: tail(&body, OUTPUT_TAIL),
10509 duration_ms: started.elapsed().as_millis() as u64,
10510 resource_blocked: false,
10511 });
10512 }
10513 state.task_finished(task);
10514 if let Err(e) = state.save() {
10515 tracing::warn!("could not persist the end of {task}: {e:#}");
10516 }
10517 (out, timed_out_pids)
10518}
10519
10520fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10532 let repo = repo.display();
10533 match style {
10534 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10535 MergeStyle::Squash => {
10536 let subject = message
10539 .lines()
10540 .next()
10541 .unwrap_or(branch)
10542 .replace(['\\', '"', '$', '`'], "");
10543 format!(
10544 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10545 )
10546 }
10547 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10548 }
10549}
10550
10551const PR_TITLE_MAX: usize = 240;
10562
10563fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10574 let line = queue::first_line(text)?;
10575 let chars: Vec<char> = line.chars().collect();
10576 let (kept, mark) = if chars.len() <= max {
10577 (line.to_owned(), "")
10578 } else if let Some(end) = sentence_end(&chars, max) {
10579 (chars[..end].iter().collect::<String>(), "")
10580 } else {
10581 let room = max.saturating_sub(3);
10582 let cut = if chars[room].is_whitespace() {
10585 room
10586 } else {
10587 chars[..room]
10588 .iter()
10589 .rposition(|c| c.is_whitespace())
10590 .unwrap_or(room)
10591 };
10592 let head: String = chars[..cut].iter().collect();
10593 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10594 (head.to_owned(), "...")
10595 };
10596 if kept.is_empty() {
10597 return None;
10598 }
10599 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10600 return None;
10601 }
10602 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10603 return None;
10604 }
10605 Some(format!("{kept}{mark}"))
10606}
10607
10608fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10612 const MIN: usize = 20;
10613 for i in MIN..max.min(chars.len()) {
10614 if !matches!(chars[i], '.' | '!' | '?') {
10615 continue;
10616 }
10617 let Some(&next) = chars.get(i + 1) else {
10618 continue;
10619 };
10620 if !next.is_whitespace() {
10621 continue;
10622 }
10623 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10624 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10625 continue;
10626 }
10627 let word: String = chars[..i]
10628 .iter()
10629 .rev()
10630 .take_while(|c| !c.is_whitespace())
10631 .collect::<Vec<_>>()
10632 .into_iter()
10633 .rev()
10634 .collect();
10635 let word = word.to_ascii_lowercase();
10636 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10637 continue;
10638 }
10639 let end = chars[..i]
10640 .iter()
10641 .rposition(|c| !c.is_whitespace())
10642 .map_or(i, |p| p + 1);
10643 return Some(end);
10644 }
10645 None
10646}
10647
10648struct PrMessage {
10653 title: String,
10654 body: String,
10655}
10656
10657impl PrMessage {
10658 fn commit_message(&self) -> String {
10662 format!("{}\n\n{}", self.title, self.body)
10663 }
10664}
10665
10666fn title_marker(line: &str) -> Option<&str> {
10668 let line = line.trim();
10669 let head = line.get(..6)?;
10670 head.eq_ignore_ascii_case("title:")
10671 .then(|| line[6..].trim())
10672}
10673
10674fn summary_title(summary: &str) -> Option<String> {
10679 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10680 let raw = title_marker(first)?;
10681 if raw.is_empty() {
10682 return None;
10683 }
10684 let title = queue::title_from(raw, PR_TITLE_MAX);
10685 let lower = title.to_ascii_lowercase();
10686 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10687 return None;
10688 }
10689 Some(title)
10690}
10691
10692const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10694
10695pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10697
10698fn is_review_run(state: &RunState) -> bool {
10702 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10703}
10704
10705fn review_title(state: &RunState) -> Option<String> {
10710 english_subject(
10711 state.reviewed_commits.as_ref()?.first()?,
10712 state.config.graph.github_english(),
10713 )
10714}
10715
10716fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10719 let raw = raw.trim();
10720 if raw.is_empty() {
10721 return None;
10722 }
10723 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10724 return None;
10725 }
10726 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10727 return None;
10728 }
10729 let title = queue::title_from(raw, PR_TITLE_MAX);
10730 let lower = title.to_ascii_lowercase();
10731 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10732 return None;
10733 }
10734 Some(title)
10735}
10736
10737#[derive(Debug, Clone, PartialEq, Eq)]
10740struct BranchFacts {
10741 commits: Vec<(String, String)>,
10743 stat: String,
10745}
10746
10747const STAT_FILE_LINES: usize = 25;
10749const COMMITS_MAX_CHARS: usize = 20_000;
10751
10752async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10755 let commits = git::commit_log(repo, base, branch).await.ok()?;
10756 if commits.is_empty() {
10757 return None;
10758 }
10759 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10760 let lines: Vec<&str> = stat.lines().collect();
10761 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10762 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10763 let more = format!(" ... {omitted} more file(s)");
10764 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10765 kept.push(&more);
10766 kept.push(lines[lines.len() - 1]);
10767 kept.join("\n")
10768 } else {
10769 lines.join("\n")
10770 };
10771 Some(BranchFacts { commits, stat })
10772}
10773
10774fn markdown_safe(text: &str) -> String {
10777 text.replace("</details>", "</details>")
10778 .replace("\x60\x60\x60", "~~~")
10779}
10780
10781fn neutral_title(state: &RunState, winner: char) -> String {
10782 format!(
10783 "chore: land candidate {} of run {}",
10784 winner.to_ascii_uppercase(),
10785 state.id
10786 )
10787}
10788
10789pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10795 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10796 ""
10797 } else {
10798 pr_title
10799 }
10800}
10801
10802pub fn landing_subject_source(state: &RunState) -> String {
10806 if is_review_run(state) {
10807 let winner = state.candidates.first().map_or('A', |c| c.label);
10808 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10809 }
10810 state.instruction.clone()
10811}
10812
10813fn summary_without_title(summary: &str) -> String {
10816 let mut lines = summary.trim().lines().peekable();
10817 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10818 lines.next();
10819 }
10820 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10821}
10822
10823#[cfg(test)]
10835fn pr_message(state: &RunState, winner: char) -> PrMessage {
10836 pr_message_with(state, winner, None)
10837}
10838
10839#[cfg(test)]
10842fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10843 let raw = pr_message_raw(
10844 state,
10845 winner,
10846 facts,
10847 crate::pr_outcome::FinalHead::Unreported,
10848 ChangeNote::NotAsked,
10849 );
10850 let id = crate::scrub::Identity::current();
10851 PrMessage {
10852 title: crate::scrub::scrub(&raw.title, &id),
10853 body: crate::scrub::scrub(&raw.body, &id),
10854 }
10855}
10856
10857#[derive(Debug, Clone, Copy)]
10859enum ChangeNote<'a> {
10860 NotAsked,
10862 Summary(&'a crate::run::ChangeSummary),
10864 Unavailable(&'a str),
10866}
10867
10868fn pr_message_raw(
10869 state: &RunState,
10870 winner: char,
10871 facts: Option<&BranchFacts>,
10872 head: crate::pr_outcome::FinalHead<'_>,
10873 change: ChangeNote<'_>,
10874) -> PrMessage {
10875 if !matches!(change, ChangeNote::NotAsked) {
10876 let (title, mut body) = match change {
10877 ChangeNote::Summary(c) => (
10878 c.title.clone(),
10879 format!("## Summary\n\n{}\n\n", c.body.trim()),
10880 ),
10881 ChangeNote::Unavailable(why) => (
10882 neutral_title(state, winner),
10883 format!(
10884 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10885 ),
10886 ),
10887 ChangeNote::NotAsked => unreachable!(),
10888 };
10889 body.push_str(&public_pr_evidence(state, head));
10890 body.push_str(&format!(
10891 "\n---\nmagi:run/{} magi:candidate-{}\n",
10892 state.id,
10893 winner.to_ascii_lowercase()
10894 ));
10895 return PrMessage { title, body };
10896 }
10897 pr_preview(state, winner, facts, head)
10898}
10899
10900fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10903 let mut body = crate::pr_outcome::review_rounds(state);
10904 body.push_str(&crate::pr_outcome::final_verification(state, head));
10905 let open = state.last_round_findings();
10906 if !open.is_empty() {
10907 body.push_str("## Open review findings\n\n");
10908 for f in open {
10909 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10910 }
10911 body.push('\n');
10912 }
10913 if let Some(c) = state.change_summary.as_ref()
10914 && head == crate::pr_outcome::FinalHead::At(&c.head)
10915 {
10916 body.push_str(&format!(
10917 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10918 c.base, c.head
10919 ));
10920 }
10921 body
10922}
10923
10924fn pr_preview(
10926 state: &RunState,
10927 winner: char,
10928 facts: Option<&BranchFacts>,
10929 head: crate::pr_outcome::FinalHead<'_>,
10930) -> PrMessage {
10931 let summary = state
10932 .candidates
10933 .iter()
10934 .find(|c| c.label == winner)
10935 .map(|c| c.summary.as_str())
10936 .unwrap_or_default();
10937 let review = is_review_run(state);
10940 let title = if review {
10941 facts
10942 .and_then(|f| {
10943 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10944 })
10945 .or_else(|| review_title(state))
10946 .or_else(|| {
10947 state
10948 .candidates
10949 .iter()
10950 .find(|c| c.label == winner)
10951 .filter(|c| !c.branch.starts_with("magi/"))
10952 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10953 })
10954 .unwrap_or_else(|| neutral_title(state, winner))
10955 } else {
10956 summary_title(summary).unwrap_or_else(|| {
10957 english_title(
10958 &state.instruction,
10959 PR_TITLE_MAX,
10960 state.config.graph.github_english(),
10961 )
10962 .unwrap_or_else(|| neutral_title(state, winner))
10963 })
10964 };
10965
10966 let mut body = String::new();
10967 let what = summary_without_title(summary);
10968 if !what.is_empty() {
10969 body.push_str("## Summary\n\n");
10970 body.push_str(&what);
10971 body.push_str("\n\n");
10972 }
10973
10974 body.push_str(&crate::pr_outcome::review_rounds(state));
10976 body.push_str(&crate::pr_outcome::final_verification(state, head));
10977 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10978
10979 let open = state.open_findings();
10980 if !open.is_empty() {
10981 body.push_str("## Open review findings\n\n");
10982 for f in &open {
10983 body.push_str(&format!(
10984 "- `{}` [{:?}] {}\n",
10985 f.id, f.severity, "Context retained in local review records."
10986 ));
10987 }
10988 body.push('\n');
10989 }
10990
10991 if let Some(fix) = fix
10992 && !fix.rejected.is_empty()
10993 {
10994 body.push_str("## Declined by the fixer\n\n");
10995 for r in &fix.rejected {
10996 body.push_str(&format!(
10997 "- `{}`: {}\n",
10998 r.id, "Reason retained in local review records."
10999 ));
11000 }
11001 body.push('\n');
11002 }
11003
11004 if review {
11005 body.push_str("## Commits under review\n\n");
11007 if let Some(facts) = facts {
11008 let mut left = COMMITS_MAX_CHARS;
11009 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11010 let mut entry = format!("- {}\n", markdown_safe(subject));
11011 for l in markdown_safe(text).lines() {
11012 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11013 }
11014 if left == 0 {
11015 body.push_str(&format!(
11016 "- ... {} more commit(s)\n",
11017 facts.commits.len() - i
11018 ));
11019 break;
11020 }
11021 if entry.len() > left {
11022 let mut end = left;
11025 while !entry.is_char_boundary(end) {
11026 end -= 1;
11027 }
11028 entry.truncate(end);
11029 entry.push_str("\n ... (truncated)\n");
11030 left = 0;
11031 } else {
11032 left -= entry.len();
11033 }
11034 body.push_str(&entry);
11035 }
11036 if !facts.stat.trim().is_empty() {
11037 body.push_str(&format!(
11038 "\n## Diff stat\n\n```\n{}\n```\n",
11039 markdown_safe(facts.stat.trim())
11040 ));
11041 }
11042 } else {
11043 match &state.reviewed_commits {
11044 Some(subjects) => {
11045 for s in subjects {
11046 body.push_str(&format!("- {}\n", s.trim()));
11047 }
11048 }
11049 None => {
11050 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11053 body.push_str(rest.trim());
11054 body.push('\n');
11055 }
11056 }
11057 }
11058 } else {
11059 let task = state.instruction.trim();
11060 let task = if task.is_empty() {
11061 "(empty task)"
11062 } else {
11063 task
11064 };
11065 body.push_str(&format!(
11066 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11067 task.replace("</details>", "</details>")
11068 ));
11069 }
11070
11071 body.push_str(&format!(
11072 "\n---\nmagi:run/{} magi:candidate-{}\n",
11073 state.id,
11074 winner.to_ascii_lowercase()
11075 ));
11076
11077 PrMessage { title, body }
11078}
11079
11080fn seeded_instruction(state: &RunState) -> String {
11084 match refs::describe(&state.seeds) {
11085 Some(facts) => format!(
11086 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11087 Candidates start from the unmerged branch named above, when there \
11088 is one, and carry any unmerged commit named by sha as a \
11089 cherry-pick. Check that this is what the task meant before \
11090 building on it.",
11091 state.instruction
11092 ),
11093 None => state.instruction.clone(),
11094 }
11095}
11096
11097async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11101 let base = &state.base_branch;
11102 let mut against = base.clone();
11103 if mode == MergeMode::Pr {
11104 let remote = &state.config.merge.remote;
11108 let tracking = format!("{remote}/{base}");
11109 let fetched = git::fetch(repo, remote, base).await;
11110 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11111 against = tracking;
11112 } else {
11113 return false;
11114 }
11115 }
11116 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11117}
11118
11119fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11122 let mut detail = format!(
11123 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11124 nothing to open a pull request for"
11125 );
11126 match refs::describe(&state.seeds) {
11127 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11128 None => detail.push_str(
11129 "\nThe task names no existing branch or commit; if it means to land work \
11130 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11131 ),
11132 }
11133 detail
11134}
11135
11136#[derive(Debug, PartialEq, Eq)]
11139enum PrPlan {
11140 Create,
11141 Adopt { url: String, title: String },
11142 Stop(String),
11143}
11144
11145fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11148 match found {
11149 Ok(land::OpenPr::None) => PrPlan::Create,
11150 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11151 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11152 "several open pull requests exist for this branch, not picking one: {}",
11153 urls.join(" ")
11154 )),
11155 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11156 }
11157}
11158
11159pub(crate) async fn gh_pr_create(
11161 cwd: &Path,
11162 base: &str,
11163 head: &str,
11164 title: &str,
11165 body: &str,
11166) -> Result<String> {
11167 let out = tokio::process::Command::new("gh")
11168 .args([
11169 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11170 ])
11171 .current_dir(cwd)
11172 .quiet()
11173 .stdin(std::process::Stdio::null())
11174 .output()
11175 .await
11176 .context("spawn gh")?;
11177 if out.status.success() {
11178 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11179 } else {
11180 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11181 }
11182}
11183
11184async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11186 match std::fs::symlink_metadata(path) {
11187 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11188 git::worktree_remove_clean(repo, path).await?;
11191 return Ok(());
11192 }
11193 Err(e) => return Err(e.into()),
11194 Ok(_) => {}
11195 }
11196 if !git::is_clean(path).await? {
11197 bail!(
11198 "worktree has uncommitted work; refusing removal: {}",
11199 path.display()
11200 );
11201 }
11202 if !git::worktree_remove_clean(repo, path).await? {
11203 bail!("worktree removal refused: {}", path.display());
11204 }
11205 Ok(())
11206}
11207
11208pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11217 if crate::pr_recovery::sources_retained(home, &state.id) {
11218 return Ok(Vec::new());
11219 }
11220 if !state.candidate_preservation.is_empty() {
11221 bail!("candidate preservation incomplete; refusing cleanup");
11222 }
11223 let repo = state.repo.clone();
11224 let wait = repository_wait(state, None);
11227 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11228 Ok(lease) => lease,
11229 Err(e) => {
11230 state.save_under(home)?;
11231 return Err(e);
11232 }
11233 };
11234 let root = state.worktree_root();
11235 let winner = state.tally.as_ref().map(|t| t.winner);
11236 let mut removed = Vec::new();
11237
11238 for i in 0..state.candidates.len() {
11239 let c = state.candidates[i].clone();
11240 let is_winner = Some(c.label) == winner;
11241 if is_winner && !drop_winner {
11242 continue;
11243 }
11244 let existed = c.worktree.exists();
11245 remove_preserved_worktree(&repo, &c.worktree).await?;
11248 if existed {
11249 removed.push(c.worktree.to_string_lossy().into_owned());
11250 }
11251 let handed_over = state.released_branches.contains(&c.branch);
11254 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11255 if !git::branch_delete(&repo, &c.branch).await? {
11256 bail!("candidate branch removal refused");
11257 }
11258 removed.push(c.branch.clone());
11259 }
11260 state.candidates[i].folded = true;
11261 }
11262
11263 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11264 let path = name.path();
11265 let keep = !drop_winner
11266 && winner.is_some_and(|w| {
11267 path.file_name()
11268 .is_some_and(|n| n == format!("cand-{w}").as_str())
11269 });
11270 if keep {
11271 continue;
11272 }
11273 remove_preserved_worktree(&repo, &path).await?;
11274 removed.push(path.to_string_lossy().into_owned());
11275 }
11276
11277 remove_if_empty(&root);
11286
11287 if state.enabled_worktree_config && drop_winner {
11288 git::release_worktree_config(&repo).await.ok();
11292 state.enabled_worktree_config = false;
11293 }
11294 state.save_under(home)?;
11295 Ok(removed)
11296}
11297
11298fn remove_if_empty(dir: &Path) {
11309 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11310 std::fs::remove_dir(dir).ok();
11311 }
11312}
11313
11314pub fn worst_open(state: &RunState) -> Option<Severity> {
11316 state
11317 .reviews
11318 .last()?
11319 .reviews
11320 .iter()
11321 .flat_map(|r| r.findings.iter())
11322 .map(|f| f.severity)
11323 .max()
11324}
11325
11326async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11330 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11331 git::acquire_worktree_config(repo).await?;
11332 state.enabled_worktree_config = true;
11333 }
11334 Ok(())
11335}
11336
11337#[cfg(test)]
11338mod tests {
11339 #[tokio::test]
11340 async fn launcher_base_fetch_requires_remote_success() {
11341 let tmp = tempfile::tempdir().unwrap();
11342 let repo = tmp.path().join("repo");
11343 let remote = tmp.path().join("remote");
11344 std::fs::create_dir(&repo).unwrap();
11345 std::fs::create_dir(&remote).unwrap();
11346 git::git(&remote, &["init", "--bare", "-b", "main"])
11347 .await
11348 .unwrap();
11349 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11350 git::git(
11351 &repo,
11352 &[
11353 "-c",
11354 "user.name=Fixture",
11355 "-c",
11356 "user.email=fixture@example.invalid",
11357 "commit",
11358 "--allow-empty",
11359 "-m",
11360 "fixture",
11361 ],
11362 )
11363 .await
11364 .unwrap();
11365 git::git(
11366 &repo,
11367 &["remote", "add", "origin", remote.to_str().unwrap()],
11368 )
11369 .await
11370 .unwrap();
11371 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11372 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11373 assert_eq!(
11374 resolve_base(&repo, "main", "origin", None)
11375 .await
11376 .unwrap()
11377 .commit,
11378 head
11379 );
11380 #[cfg(unix)]
11381 {
11382 let policy: crate::config::Coordination = toml::from_str(
11383 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11384 ).unwrap();
11385 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11386 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11387 let held = crate::repo_lock::SeatLease::operation_with(
11388 &repo,
11389 Some(&policy),
11390 &crate::repo_lock::WaitPolicy::default(),
11391 )
11392 .await
11393 .unwrap();
11394 assert!(held.is_some());
11395 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11396 tokio::pin!(fetch);
11397 assert!(
11398 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11399 .await
11400 .is_err()
11401 );
11402 assert!(
11403 !repo.join(".git/FETCH_HEAD").exists(),
11404 "no fetch before admission"
11405 );
11406 drop(held);
11407 let fetched = fetch.await.unwrap();
11408 assert_eq!(fetched.commit, head);
11409 assert_eq!(fetched.remote, "origin");
11410 assert_eq!(fetched.branch, "main");
11411 }
11412 git::git(
11414 &repo,
11415 &[
11416 "remote",
11417 "set-url",
11418 "origin",
11419 tmp.path().join("missing").to_str().unwrap(),
11420 ],
11421 )
11422 .await
11423 .unwrap();
11424 let error = resolve_base(&repo, "main", "origin", None)
11425 .await
11426 .unwrap_err()
11427 .to_string();
11428 assert!(error.contains("cannot read origin/main"));
11429 assert!(error.contains("refusing to branch off"));
11430 }
11431
11432 #[test]
11433 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11434 let leaked = vec!["chore(deps): update a crate".to_owned()];
11435 let own = "fix(daemon): apply a chosen action";
11436 assert!(should_retitle("", own, &leaked));
11437 assert!(should_retitle(
11438 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11439 own,
11440 &leaked
11441 ));
11442 assert!(should_retitle(
11443 "chore: land candidate A of run 1",
11444 own,
11445 &leaked
11446 ));
11447 assert!(should_retitle(
11448 "magi: candidate A (uncommitted work)",
11449 own,
11450 &leaked
11451 ));
11452 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11453 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11454 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11455 }
11456
11457 #[test]
11458 fn pr_merge_plan_creates_adopts_or_stops() {
11459 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11460 assert_eq!(
11461 pr_merge_plan(Ok(land::OpenPr::One {
11462 url: "u".into(),
11463 title: "t".into()
11464 })),
11465 PrPlan::Adopt {
11466 url: "u".into(),
11467 title: "t".into()
11468 }
11469 );
11470 let PrPlan::Stop(many) =
11471 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11472 else {
11473 panic!("many must stop");
11474 };
11475 assert!(many.contains('a') && many.contains('b'));
11476 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11477 panic!("a failed lookup must stop");
11478 };
11479 assert!(err.contains("bad token"));
11480 }
11481
11482 use super::*;
11483 use crate::run::GateStatus;
11484 use std::collections::BTreeMap;
11485 use std::time::Duration;
11486
11487 fn conductor() -> AgentSpec {
11488 AgentSpec {
11489 id: "conductor".to_owned(),
11490 kind: crate::config::AgentKind::Command,
11491 model: None,
11492 command: vec!["true".to_owned()],
11493 extra_args: Vec::new(),
11494 env: BTreeMap::new(),
11495 prompt_delivery: None,
11496 }
11497 }
11498
11499 fn spec(id: &str) -> AgentSpec {
11500 AgentSpec {
11501 id: id.to_owned(),
11502 kind: crate::config::AgentKind::Command,
11503 model: None,
11504 command: vec!["true".to_owned()],
11505 extra_args: Vec::new(),
11506 env: BTreeMap::new(),
11507 prompt_delivery: None,
11508 }
11509 }
11510
11511 fn ids(xs: &[&str]) -> BTreeSet<String> {
11512 xs.iter().map(|s| (*s).to_owned()).collect()
11513 }
11514
11515 #[test]
11516 fn pick_successor_skips_an_agent_another_seat_holds() {
11517 let roster = [spec("a"), spec("b"), spec("c")];
11519 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11520 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11521 }
11522
11523 #[test]
11524 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11525 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11527 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11528 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11529 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11530 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11531 }
11532
11533 #[test]
11534 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11535 let roster = [spec("a"), spec("b")];
11536 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11537 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11538 let carried = ids(&["b"]);
11539 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11540 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11541 }
11542
11543 #[test]
11544 fn pick_successor_returns_none_without_any_untried_successor() {
11545 let roster = [spec("a"), spec("b")];
11546 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11547 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11548 let carried = ids(&["a"]);
11549 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11550 }
11551
11552 #[test]
11553 fn pick_successor_rescue_avoids_another_seats_occupant() {
11554 let roster = [spec("a"), spec("b"), spec("c")];
11556 let carried = ids(&["b", "c"]);
11557 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11558 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11559 }
11560
11561 #[test]
11562 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11563 let roster = [spec("a"), spec("b"), spec("c")];
11564 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11565 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11566 }
11567
11568 #[test]
11569 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11570 let roster = [spec("a"), spec("b"), spec("c")];
11571 let failed = ids(&["a", "b", "c"]);
11572 let mut tried = ids(&["b"]);
11574 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11575 assert_eq!(first.id, "a");
11576 tried.insert(first.id.clone());
11577 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11578 assert_eq!(second.id, "c");
11579 tried.insert(second.id.clone());
11580 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11581 }
11582
11583 #[test]
11584 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11585 let roster = [spec("a"), spec("b")];
11586 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11587 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11588 }
11589
11590 #[test]
11591 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11592 let roster = [spec("a"), spec("b"), spec("c")];
11593 let h = SeatHistory {
11594 failed: ids(&["a"]),
11595 last_ok: Some("b".to_owned()),
11596 last_fail: None,
11597 };
11598 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11599 let gone = SeatHistory {
11601 last_ok: Some("zzz".to_owned()),
11602 ..h.clone()
11603 };
11604 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11605 let failed = SeatHistory {
11606 failed: ids(&["a", "b"]),
11607 last_ok: Some("b".to_owned()),
11608 last_fail: None,
11609 };
11610 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11611 }
11612
11613 #[test]
11614 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11615 let first = SeatState::new("review-1", "alpha", 7);
11616 let next = handover_seat("review-1", "gamma", 7);
11617 assert_ne!(first.claude_session, next.claude_session);
11618 }
11619
11620 #[test]
11621 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11622 let mut state = state_with_summary("x", "y");
11623 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11624 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11625 assert_ne!(a.claude_session, b.claude_session);
11626 }
11627
11628 #[test]
11629 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11630 let roster = [spec("a"), spec("b")];
11631 let h = SeatHistory {
11632 failed: ids(&["a", "b"]),
11633 ..SeatHistory::default()
11634 };
11635 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11636 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11637 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11638 }
11639
11640 #[test]
11647 fn failure_signature_ignores_numbers_and_paths() {
11648 assert_eq!(
11649 failure_signature("exited with Some(2) and no usable output"),
11650 failure_signature("exited with Some(137) and no usable output")
11651 );
11652 assert_eq!(
11653 failure_signature("process exited with no usable output (exit code 2)"),
11654 failure_signature("process exited with no usable output (exit code 137)")
11655 );
11656 for (old, new) in [
11657 (
11658 "exited with Some(2) and no usable output",
11659 "process exited with no usable output (exit code 2)",
11660 ),
11661 (
11662 "exited with None and no usable output",
11663 "process exited with no usable output (exit code unavailable)",
11664 ),
11665 ] {
11666 let previous = FailClass::Other(failure_signature(old));
11667 let current = FailClass::Other(failure_signature(new));
11668 assert_eq!(previous, current, "resumption retains the failure class");
11669 assert!(!should_hand_over(Some(&previous), ¤t));
11670 }
11671 assert_eq!(
11672 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11673 failure_signature("cannot open /var/x.txt: denied")
11674 );
11675 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11676 }
11677
11678 #[test]
11679 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11680 let other = FailClass::Other("x".into());
11681 assert!(should_hand_over(None, &FailClass::Quota));
11682 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11683 assert!(should_hand_over(
11684 Some(&FailClass::Timeout),
11685 &FailClass::Timeout
11686 ));
11687 assert!(should_hand_over(None, &other));
11688 assert!(!should_hand_over(Some(&other), &other));
11689 assert!(should_hand_over(
11690 Some(&other),
11691 &FailClass::Other("y".into())
11692 ));
11693 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11695 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11696 }
11697
11698 #[test]
11699 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11700 let a = SeatState::new("judge-1", "alpha", 7);
11701 let b = handover_seat("judge-1", "beta", 7);
11702 assert_ne!(a.claude_session, b.claude_session);
11703 assert_eq!(b.turns, 0);
11704 }
11705
11706 #[test]
11707 fn a_timeout_is_classified_apart_from_other_failures() {
11708 assert_eq!(
11709 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11710 Some(FailClass::Timeout)
11711 );
11712 assert!(matches!(
11713 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11714 Some(FailClass::Other(_))
11715 ));
11716 }
11717
11718 #[test]
11719 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11720 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11721 let tried = BTreeSet::from(["beta".to_owned()]);
11722 let next = next_untried_in_roster(&roster, 1, &tried);
11725 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11726 }
11727
11728 #[test]
11729 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11730 let roster = vec![spec("alpha"), spec("beta")];
11731 let tried = BTreeSet::from(["beta".to_owned()]);
11732 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11736 }
11737
11738 #[test]
11739 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11740 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11741 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11742 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11746 }
11747
11748 #[test]
11749 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11750 let roster = vec![spec("a"), spec("a"), spec("b")];
11751 let tried = BTreeSet::from(["a".to_owned()]);
11752 let next = next_untried_in_roster(&roster, 0, &tried);
11753 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11754 }
11755
11756 #[test]
11757 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11758 let roster = vec![spec("a"), spec("b")];
11759 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11760 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11761 }
11762
11763 #[test]
11764 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11765 let dir = tempfile::tempdir().unwrap();
11766 let bay = dir.path().join("ffff");
11767
11768 remove_if_empty(&bay);
11770 assert!(!bay.exists());
11771
11772 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11775 remove_if_empty(&bay);
11776 assert!(bay.exists(), "non-empty directory must survive");
11777
11778 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11780 remove_if_empty(&bay);
11781 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11782 }
11783
11784 #[test]
11793 fn a_full_panel_that_found_nothing_is_clean() {
11794 assert!(round_is_clean(
11795 0,
11796 true,
11797 2,
11798 2,
11799 0,
11800 IncompleteReviewPolicy::Block
11801 ));
11802 }
11803
11804 #[test]
11805 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11806 assert!(!round_is_clean(
11807 0,
11808 true,
11809 1,
11810 2,
11811 0,
11812 IncompleteReviewPolicy::Block
11813 ));
11814 }
11815
11816 #[test]
11817 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11818 assert!(!round_is_clean(
11819 1,
11820 true,
11821 1,
11822 2,
11823 0,
11824 IncompleteReviewPolicy::Warn
11825 ));
11826 }
11827
11828 #[test]
11829 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11830 assert!(round_is_clean(
11831 0,
11832 true,
11833 1,
11834 2,
11835 0,
11836 IncompleteReviewPolicy::Warn
11837 ));
11838 }
11839
11840 #[test]
11841 fn a_full_panel_with_an_open_finding_is_not_clean() {
11842 assert!(!round_is_clean(
11843 1,
11844 true,
11845 2,
11846 2,
11847 0,
11848 IncompleteReviewPolicy::Block
11849 ));
11850 }
11851
11852 #[test]
11853 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11854 assert!(!round_is_clean(
11855 0,
11856 false,
11857 2,
11858 2,
11859 0,
11860 IncompleteReviewPolicy::Block
11861 ));
11862 }
11863
11864 #[test]
11871 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11872 assert!(round_is_clean(
11875 0,
11876 true,
11877 1,
11878 2,
11879 1,
11880 IncompleteReviewPolicy::Block
11881 ));
11882 }
11883
11884 #[test]
11885 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11886 assert!(!round_is_clean(
11889 0,
11890 true,
11891 1,
11892 2,
11893 0,
11894 IncompleteReviewPolicy::Block
11895 ));
11896 }
11897
11898 #[test]
11899 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11900 assert!(!round_is_clean(
11901 1,
11902 true,
11903 1,
11904 2,
11905 1,
11906 IncompleteReviewPolicy::Block
11907 ));
11908 assert!(!round_is_clean(
11909 0,
11910 false,
11911 1,
11912 2,
11913 1,
11914 IncompleteReviewPolicy::Block
11915 ));
11916 }
11917
11918 #[test]
11919 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11920 assert!(!round_is_clean(
11924 0,
11925 true,
11926 0,
11927 2,
11928 2,
11929 IncompleteReviewPolicy::Block
11930 ));
11931 }
11932
11933 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11934 CommandOutcome {
11935 command: "test".to_owned(),
11936 code,
11937 output_tail: String::new(),
11938 duration_ms: 0,
11939 resource_blocked,
11940 }
11941 }
11942
11943 #[test]
11944 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11945 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11946 assert!(
11947 !verify_inconclusive(&[outcome(Some(1), false)]),
11948 "an ordinary failure is still evidence about the patch"
11949 );
11950 assert!(verify_inconclusive(&[outcome(None, true)]));
11951 assert!(
11952 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11953 "one inconclusive outcome taints the whole batch"
11954 );
11955 assert!(!verify_inconclusive(&[]));
11956 }
11957
11958 #[tokio::test]
11959 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11960 let calls = std::sync::atomic::AtomicUsize::new(0);
11964 let started = Instant::now();
11965 wait_for_pids_with(
11966 &[123],
11967 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11968 Duration::from_millis(5),
11969 Duration::from_secs(5),
11970 )
11971 .await;
11972 assert!(
11973 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11974 "must keep checking rather than deciding on the first answer"
11975 );
11976 assert!(
11977 started.elapsed() < Duration::from_secs(1),
11978 "must return the moment it is confirmed dead, not wait out the ceiling"
11979 );
11980 }
11981
11982 #[tokio::test]
11983 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11984 let started = Instant::now();
11985 wait_for_pids_with(
11986 &[123],
11987 |_| true, Duration::from_millis(5),
11989 Duration::from_millis(30),
11990 )
11991 .await;
11992 let elapsed = started.elapsed();
11993 assert!(
11994 elapsed >= Duration::from_millis(30),
11995 "must not give up before its own ceiling: {elapsed:?}"
11996 );
11997 assert!(
11998 elapsed < Duration::from_secs(1),
11999 "must not wait past its own ceiling either: {elapsed:?}"
12000 );
12001 }
12002
12003 #[tokio::test]
12004 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12005 let started = Instant::now();
12006 wait_for_pids_with(
12007 &[],
12008 |_| true,
12009 Duration::from_secs(5),
12010 Duration::from_secs(5),
12011 )
12012 .await;
12013 assert!(
12014 started.elapsed() < Duration::from_millis(200),
12015 "an empty pid list has nothing to confirm"
12016 );
12017 }
12018
12019 fn review_round(
12025 clean: bool,
12026 blocking: usize,
12027 answered: usize,
12028 expected: usize,
12029 progressed: bool,
12030 e2e_ok: bool,
12031 ) -> ReviewRound {
12032 ReviewRound {
12033 round: 1,
12034 head: "h".to_owned(),
12035 verified_head: None,
12036 verified_at: None,
12037 reviews: Vec::new(),
12038 e2e: vec![CommandOutcome {
12039 command: "test".to_owned(),
12040 code: Some(if e2e_ok { 0 } else { 1 }),
12041 output_tail: String::new(),
12042 duration_ms: 0,
12043 resource_blocked: false,
12044 }],
12045 verify_retried: false,
12046 e2e_deferred: false,
12047 e2e_defer_reason: None,
12048 fix: None,
12049 blocking,
12050 answered,
12051 expected,
12052 clean,
12053 progressed,
12054 vote_split: false,
12055 reconsideration: Vec::new(),
12056 verdict: None,
12057 }
12058 }
12059
12060 #[test]
12061 fn review_conclusion_is_none_when_nothing_has_run() {
12062 assert_eq!(review_conclusion(&[], 3), None);
12063 }
12064
12065 #[test]
12066 fn review_conclusion_is_none_while_rounds_remain() {
12067 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12068 assert_eq!(review_conclusion(&rounds, 3), None);
12069 }
12070
12071 #[test]
12072 fn review_conclusion_is_gating_once_a_round_is_clean() {
12073 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12074 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12075 }
12076
12077 #[test]
12078 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12079 let rounds = vec![
12080 review_round(false, 1, 2, 2, true, true),
12081 review_round(false, 1, 2, 2, true, true),
12082 ];
12083 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12084 }
12085
12086 #[test]
12087 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12088 let rounds = vec![
12089 review_round(false, 1, 2, 2, true, true),
12090 review_round(false, 1, 2, 2, true, false),
12091 ];
12092 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12093 }
12094
12095 #[test]
12096 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12097 let mut blocked = review_round(false, 1, 2, 2, true, false);
12104 blocked.e2e[0].resource_blocked = true;
12105 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12106 assert_eq!(review_conclusion(&rounds, 2), None);
12107 }
12108
12109 #[test]
12110 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12111 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12113 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12114 }
12115
12116 #[test]
12117 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12118 let rounds = vec![
12119 review_round(false, 1, 2, 2, false, true),
12120 review_round(false, 1, 2, 2, false, true),
12121 ];
12122 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12123 }
12124
12125 fn secs(n: u64) -> Duration {
12126 Duration::from_secs(n)
12127 }
12128
12129 #[tokio::test]
12132 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12133 ask_test_home();
12134 let tmp = tempfile::tempdir().unwrap();
12135 let repo = tmp.path().join("repo");
12136 std::fs::create_dir_all(&repo).unwrap();
12137 init_repo(&repo);
12138 let g = |args: &[&str]| {
12139 let out = std::process::Command::new("git")
12140 .args(args)
12141 .current_dir(&repo)
12142 .quiet()
12143 .output()
12144 .unwrap();
12145 assert!(
12146 out.status.success(),
12147 "{}",
12148 String::from_utf8_lossy(&out.stderr)
12149 );
12150 };
12151 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12152 g(&["checkout", "-b", "historical"]);
12153 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12154 g(&["add", "."]);
12155 g(&["commit", "-m", "old alternative"]);
12156 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12157 g(&["checkout", "-b", "magi/task-selected", &base]);
12158 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12159 g(&["add", "."]);
12160 g(&["commit", "-m", "measurement infrastructure"]);
12161 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12162 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12163 g(&["add", "."]);
12164 g(&["commit", "-m", "dependent implementation"]);
12165 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12166 g(&["checkout", "main"]);
12167 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12168 g(&["add", "."]);
12169 g(&["commit", "-m", "unrelated base change"]);
12170 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12171 let origin = tmp.path().join("origin.git");
12172 g(&[
12173 "clone",
12174 "--bare",
12175 repo.to_str().unwrap(),
12176 origin.to_str().unwrap(),
12177 ]);
12178 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12179 g(&["fetch", "origin"]);
12180 let selection = crate::pr_recovery::Selection {
12181 url: "https://example.test/project/pull/1".into(),
12182 branch: "magi/task-selected".into(),
12183 head: head.clone(),
12184 instruction: "preserve measured behavior".into(),
12185 approved_at: jiff::Timestamp::now(),
12186 };
12187 let mut template = runner_at(RunStatus::Reviewing);
12188 template.state.repo = repo.clone();
12189 template.state.base_commit = current.clone();
12190 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12191 template.state.origin = Some(crate::run::Origin::queue("selected"));
12192 template.state.pr_head = Some(selection.branch.clone());
12193 template.state.pr_recovery = Some(selection.clone());
12194 let mut runner =
12195 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12196 .await
12197 .unwrap();
12198 let wt = runner.state.winner().unwrap().worktree.clone();
12199 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12200 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12201 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12202 assert!(runner.state.seeds.is_empty());
12203 assert_eq!(runner.state.instruction, "preserve measured behavior");
12204 runner.sync_to_base().await.unwrap();
12205 assert_eq!(
12206 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12207 "prerequisite"
12208 );
12209 assert_eq!(
12210 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12211 "measured implementation"
12212 );
12213 assert_eq!(
12214 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12215 "current main change"
12216 );
12217 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12218 assert_eq!(
12219 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12220 head
12221 );
12222
12223 let mut partial = runner_at(RunStatus::Reviewing);
12225 partial.state.repo = repo.clone();
12226 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12227 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12228 run: Some("exit 1".into()),
12229 ..Default::default()
12230 }];
12231 let selection = runner.state.pr_recovery.clone().unwrap();
12232 partial.state.pr_recovery = Some(selection.clone());
12233 partial.state.pr_head = Some(selection.branch.clone());
12234 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12235 let id = partial.state.id.clone();
12236 let error = match Runner::open_selected(
12237 &repo,
12238 partial.state,
12239 partial.roles,
12240 1,
12241 runner.state.base_commit.clone(),
12242 selection,
12243 )
12244 .await
12245 {
12246 Ok(_) => panic!("setup must fail"),
12247 Err(e) => e,
12248 };
12249 assert_eq!(
12250 error
12251 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12252 .unwrap()
12253 .run,
12254 id
12255 );
12256 let failed = RunState::load(&id).unwrap();
12257 assert_eq!(failed.status, RunStatus::Blocked);
12258 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12259 assert!(Runner::resume(&id).is_err());
12260 runner.state.status = RunStatus::Ready;
12261 runner.state.driver_exited = true;
12262 runner.state.save().unwrap();
12263 let mut retry = runner_at(RunStatus::Reviewing);
12264 retry.state.repo = repo.clone();
12265 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12266 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12267 reselected.approved_at = jiff::Timestamp::now();
12268 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12269 retry.state.pr_head = Some(reselected.branch.clone());
12270 retry.state.pr_recovery = Some(reselected.clone());
12271 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12272 .await
12273 .unwrap();
12274 let restarted = Runner::open_selected(
12275 &repo,
12276 retry.state,
12277 retry.roles,
12278 1,
12279 runner.state.base_commit.clone(),
12280 reselected,
12281 )
12282 .await
12283 .unwrap();
12284 assert_eq!(
12285 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12286 .await
12287 .unwrap(),
12288 head
12289 );
12290 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12291 assert_eq!(
12292 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12293 head
12294 );
12295 }
12296
12297 #[tokio::test]
12298 #[cfg(unix)]
12299 async fn fallback_and_interrupted_restoration_require_coordination() {
12300 use std::os::fd::AsRawFd as _;
12301 ask_test_home();
12302 let tmp = tempfile::tempdir().unwrap();
12303 let mut runner = registered_runner(tmp.path()).await;
12304 runner.state.config.graph.timeout_resource_wait = 1;
12305 let winner = runner.state.candidates[0].clone();
12306 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12307 let leftover = winner.worktree.join("leftover");
12308 std::fs::write(&leftover, "untrusted edits").unwrap();
12309 let next = tmp.path().join("next-fixer");
12310 let mut failed = spec("first");
12311 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12312 let mut successor = spec("second");
12313 successor.command = vec![
12314 "sh".into(),
12315 "-c".into(),
12316 format!("touch '{}'; echo done", next.display()),
12317 ];
12318 let attempts = vec![
12319 (failed.clone(), "first".into()),
12320 (successor.clone(), "second".into()),
12321 ];
12322 let lock_path = runner.state.repo.join(".git/updater.lock");
12323 let external = std::fs::File::create(&lock_path).unwrap();
12324 assert_eq!(
12325 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12326 0
12327 );
12328 let artifacts = tmp.path().join("artifacts");
12329 let build = |spec, seat| SeatJob {
12330 spec,
12331 seat,
12332 cwd: winner.worktree.clone(),
12333 prompt: "fixture".into(),
12334 timeout: Duration::from_secs(10),
12335 allow_write: true,
12336 sessions: false,
12337 artifacts: artifacts.clone(),
12338 stem: "fix".into(),
12339 handover: None,
12340 };
12341 let (_, _, out) = runner
12342 .ask_fixer_among(
12343 attempts.clone(),
12344 "graft",
12345 None,
12346 Some((&winner.worktree, &head)),
12347 build,
12348 )
12349 .await;
12350 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12351 assert!(leftover.exists(), "contention must prevent clean");
12352 assert!(!next.exists());
12353 assert!(runner.state.handovers.is_empty());
12354 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12356 from_run: "earlier".into(),
12357 generation: 1,
12358 mode: crate::recovery::Mode::VerifyRepair,
12359 reasons: Vec::new(),
12360 reviewed_head: None,
12361 rounds: Vec::new(),
12362 open_findings: Vec::new(),
12363 evidence: None,
12364 criteria_inherited: true,
12365 repair: crate::recovery::Repair::Started {
12366 before: head.clone(),
12367 },
12368 });
12369 assert!(
12370 runner
12371 .recovery_repair(&winner)
12372 .await
12373 .unwrap_err()
12374 .is::<crate::repo_lock::ResourceBlocked>()
12375 );
12376 assert!(matches!(
12377 runner.state.recovery.as_ref().unwrap().repair,
12378 crate::recovery::Repair::Started { .. }
12379 ));
12380 assert!(leftover.exists());
12381 drop(external);
12382 runner.recovery_repair(&winner).await.unwrap();
12383 assert!(!leftover.exists());
12384 assert!(matches!(
12385 runner.state.recovery.as_ref().unwrap().repair,
12386 crate::recovery::Repair::Done { .. }
12387 ));
12388 std::fs::remove_file(&lock_path).unwrap();
12390 std::fs::create_dir(&lock_path).unwrap();
12391 std::fs::write(&leftover, "untrusted edits").unwrap();
12392 let (_, _, out) = runner
12393 .ask_fixer_among(
12394 attempts,
12395 "graft",
12396 None,
12397 Some((&winner.worktree, &head)),
12398 build,
12399 )
12400 .await;
12401 assert!(matches!(out, AgentOutcome::Failed(_)));
12402 assert!(leftover.exists());
12403 assert!(!next.exists());
12404 assert_eq!(
12405 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12406 head
12407 );
12408 }
12409
12410 #[cfg(unix)]
12411 async fn registered_runner(root: &Path) -> Runner {
12412 ask_test_home();
12413 let repo = root.join("repo");
12414 std::fs::create_dir_all(&repo).unwrap();
12415 init_repo(&repo);
12416 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12417 let mut runner = runner_at(RunStatus::Prep);
12418 runner.state.repo = repo.clone();
12419 runner.state.config.coordination = crate::config::test_coordination();
12420 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12421 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12422 runner.state.config.graph.advise = false;
12423 runner.roles.implementers = vec![spec("implement")];
12424 runner.roles.judges = vec![spec("judge")];
12425 runner.prep().await.unwrap();
12426 runner
12427 }
12428
12429 #[tokio::test]
12430 #[cfg(unix)]
12431 async fn parked_implementer_persists_completed_work() {
12432 for registered in [false, true] {
12433 let tmp = tempfile::tempdir().unwrap();
12434 let mut runner = registered_runner(tmp.path()).await;
12435 if !registered {
12436 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12437 }
12438 let marker = tmp.path().join("started");
12439 let release = tmp.path().join("release");
12440 let mut implementer = spec("implement");
12441 implementer.command = vec![
12442 "sh".into(),
12443 "-c".into(),
12444 format!(
12445 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12446 marker.display(),
12447 release.display()
12448 ),
12449 ];
12450 runner.roles.implementers = vec![implementer.clone()];
12451 runner.state.config.agents = vec![implementer];
12452 let pause = runner.pause.clone();
12453 let (result, ()) = tokio::join!(runner.implement(), async {
12454 tokio::time::timeout(Duration::from_secs(10), async {
12455 while !marker.exists() {
12456 tokio::time::sleep(Duration::from_millis(10)).await;
12457 }
12458 })
12459 .await
12460 .unwrap();
12461 pause.park();
12462 std::fs::write(&release, "go").unwrap();
12463 });
12464 result.unwrap();
12465 let candidate = &runner.state.candidates[0];
12466 assert!(candidate.commits > 0, "{candidate:?}");
12467 assert!(candidate.summary.contains("saved-result"));
12468 assert_eq!(candidate.files, 1);
12469 let before = candidate.commits;
12470 runner.implement().await.unwrap();
12471 assert_eq!(runner.state.candidates[0].commits, before);
12472 }
12473 }
12474
12475 #[tokio::test]
12476 #[cfg(unix)]
12477 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12478 let tmp = tempfile::tempdir().unwrap();
12479 let mut runner = registered_runner(tmp.path()).await;
12480 let winner = runner.state.candidates[0].clone();
12481 let cache = tmp.path().join("cache");
12482 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12483 runner.state.config.graph.timeout_resource_wait = 0;
12484 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12485 let crate::cache::AcquireOutcome::Acquired(held) =
12486 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12487 else {
12488 panic!("cache must be free")
12489 };
12490 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12491 from_run: "earlier".into(),
12492 generation: 1,
12493 mode: crate::recovery::Mode::VerifyRepair,
12494 reasons: Vec::new(),
12495 reviewed_head: None,
12496 rounds: Vec::new(),
12497 open_findings: Vec::new(),
12498 evidence: None,
12499 criteria_inherited: true,
12500 repair: crate::recovery::Repair::NotStarted,
12501 });
12502 runner.recovery_repair(&winner).await.unwrap();
12503 assert!(matches!(
12504 runner.state.recovery.as_ref().unwrap().repair,
12505 crate::recovery::Repair::WaitingForRestore { .. }
12506 ));
12507 drop(held);
12508 runner.state.resource_block = None;
12509 runner.state.config.graph.timeout_resource_wait = 1;
12510 runner.recovery_repair(&winner).await.unwrap();
12511 assert!(matches!(
12512 runner.state.recovery.as_ref().unwrap().repair,
12513 crate::recovery::Repair::Done { .. }
12514 ));
12515 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12516 }
12517
12518 #[tokio::test]
12519 #[cfg(unix)]
12520 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12521 use std::os::fd::AsRawFd as _;
12522 for graft in [false, true] {
12523 let tmp = tempfile::tempdir().unwrap();
12524 let mut runner = registered_runner(tmp.path()).await;
12525 runner.state.config.graph.timeout_resource_wait = 1;
12526 let winner = runner.state.candidates[0].clone();
12527 let marker = tmp.path().join("fixer-started");
12528 let mut fixer = spec("fixer");
12529 fixer.command = vec![
12530 "sh".into(),
12531 "-c".into(),
12532 format!("echo started >> '{}'; echo done", marker.display()),
12533 ];
12534 runner.roles.fixer = Some(vec![fixer]);
12535 if graft {
12536 let mut loser = winner.clone();
12537 loser.label = 'B';
12538 loser.index = 1;
12539 runner.state.candidates.push(loser);
12540 runner.state.config.graph.graft_runner_up = true;
12541 runner.state.tally = Some(Tally {
12542 first_choice: BTreeMap::new(),
12543 borda: BTreeMap::new(),
12544 winner: winner.label,
12545 rankings: 1,
12546 unanimous_initial: true,
12547 deliberated: false,
12548 changed_votes: 0,
12549 unanimous_final: true,
12550 tie_break: None,
12551 judges: 1,
12552 present: 1,
12553 quorum: 1,
12554 met_quorum: true,
12555 uncontested: None,
12556 });
12557 runner.state.judgements.push(crate::run::Judgement {
12558 judge: 1,
12559 seat: "judge-1".into(),
12560 agent: "judge".into(),
12561 ranking: vec![winner.label, 'B'],
12562 reasons: BTreeMap::new(),
12563 confidence: None,
12564 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12565 order: vec![0, 1],
12566 failed: None,
12567 duration_ms: 0,
12568 });
12569 } else {
12570 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12571 from_run: "earlier".into(),
12572 generation: 1,
12573 mode: crate::recovery::Mode::VerifyRepair,
12574 reasons: Vec::new(),
12575 reviewed_head: None,
12576 rounds: Vec::new(),
12577 open_findings: Vec::new(),
12578 evidence: None,
12579 criteria_inherited: true,
12580 repair: crate::recovery::Repair::NotStarted,
12581 });
12582 }
12583 let reader = crate::repo_lock::SeatLease::for_checkout(
12584 &winner.worktree,
12585 false,
12586 crate::config::test_coordination().as_ref(),
12587 )
12588 .await
12589 .unwrap();
12590 let pause = runner.pause.clone();
12591 let (result, ()) = tokio::join!(
12592 async {
12593 if graft {
12594 runner.graft().await
12595 } else {
12596 runner.recovery_repair(&winner).await
12597 }
12598 },
12599 async {
12600 tokio::time::sleep(Duration::from_millis(100)).await;
12601 pause.park();
12602 }
12603 );
12604 result.unwrap();
12605 assert!(!marker.exists(), "a blocked seat must not spawn");
12606 runner.state = RunState::load(&runner.state.id).unwrap();
12608 if graft {
12609 assert_eq!(
12610 runner.state.graft.as_ref().unwrap().outcome,
12611 GraftOutcome::WaitingForRestore
12612 );
12613 } else {
12614 assert!(matches!(
12615 runner.state.recovery.as_ref().unwrap().repair,
12616 crate::recovery::Repair::WaitingForRestore { .. }
12617 ));
12618 }
12619 drop(reader);
12620 runner.pause = Pause::new();
12621 runner.state.resource_block = None;
12622 let external =
12624 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12625 assert_eq!(
12626 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12627 0
12628 );
12629 let leftover = winner.worktree.join("leftover");
12630 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12631 let result = if graft {
12632 runner.graft().await
12633 } else {
12634 runner.recovery_repair(&winner).await
12635 };
12636 assert!(
12637 result
12638 .unwrap_err()
12639 .is::<crate::repo_lock::ResourceBlocked>()
12640 );
12641 assert!(leftover.exists());
12642 assert!(!marker.exists());
12643 drop(external);
12644 runner.state.resource_block = None;
12645 let result = if graft {
12646 runner.graft().await
12647 } else {
12648 runner.recovery_repair(&winner).await
12649 };
12650 result.unwrap();
12651 assert!(!leftover.exists(), "restore must precede retry");
12652 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12653 if graft {
12654 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12655 } else {
12656 assert!(matches!(
12657 runner.state.recovery.as_ref().unwrap().repair,
12658 crate::recovery::Repair::Done { .. }
12659 ));
12660 }
12661 }
12662 }
12663
12664 #[tokio::test]
12665 #[cfg(unix)]
12666 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12667 use std::os::fd::AsRawFd as _;
12668 let tmp = tempfile::tempdir().unwrap();
12669 let mut runner = registered_runner(tmp.path()).await;
12670 runner.state.config.graph.timeout_resource_wait = 1;
12671 let winner = runner.state.candidates[0].clone();
12672 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12673 runner.state.tally = Some(Tally {
12674 first_choice: BTreeMap::new(),
12675 borda: BTreeMap::new(),
12676 winner: winner.label,
12677 rankings: 0,
12678 unanimous_initial: false,
12679 deliberated: false,
12680 changed_votes: 0,
12681 unanimous_final: false,
12682 tie_break: None,
12683 judges: 0,
12684 present: 0,
12685 quorum: 0,
12686 met_quorum: true,
12687 uncontested: Some("fixture".into()),
12688 });
12689 runner.state.graft = Some(GraftRecord {
12690 pre: pre.clone(),
12691 post: None,
12692 lines: None,
12693 limit: None,
12694 agent: None,
12695 outcome: GraftOutcome::Running,
12696 requested: Vec::new(),
12697 adoption: None,
12698 adoption_error: None,
12699 });
12700 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12701 git::commit_all(&winner.worktree, "unfinished graft")
12702 .await
12703 .unwrap();
12704 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12705 let leftover = winner.worktree.join("leftover");
12706 std::fs::write(&leftover, "untrusted edits").unwrap();
12707 let path = runner.state.repo.join(".git/updater.lock");
12708 let external = std::fs::File::create(&path).unwrap();
12709 assert_eq!(
12710 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12711 0
12712 );
12713 assert!(
12714 runner
12715 .graft()
12716 .await
12717 .unwrap_err()
12718 .is::<crate::repo_lock::ResourceBlocked>()
12719 );
12720 assert!(matches!(
12721 runner.state.graft.as_ref().unwrap().outcome,
12722 GraftOutcome::Running
12723 ));
12724 assert_eq!(
12725 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12726 unfinished
12727 );
12728 assert!(leftover.exists());
12729 drop(external);
12730 std::fs::remove_file(&path).unwrap();
12731 std::fs::create_dir(&path).unwrap();
12732 assert!(runner.graft().await.is_err());
12733 assert!(matches!(
12734 runner.state.graft.as_ref().unwrap().outcome,
12735 GraftOutcome::Running
12736 ));
12737 assert_eq!(
12738 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12739 unfinished
12740 );
12741 assert!(leftover.exists());
12742 std::fs::remove_dir(&path).unwrap();
12743 runner.graft().await.unwrap();
12744 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12745 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12746 assert!(!leftover.exists());
12747 }
12748
12749 #[tokio::test]
12750 #[cfg(unix)]
12751 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12752 use std::os::fd::AsRawFd as _;
12753 let tmp = tempfile::tempdir().unwrap();
12754 let mut runner = registered_runner(tmp.path()).await;
12755 runner.state.prep_complete = Some(false);
12756 runner.state.config.graph.timeout_resource_wait = 1;
12757 runner.state.save().unwrap();
12758 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12759 assert_eq!(
12760 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12761 0
12762 );
12763 runner.execute().await.unwrap();
12764 let disk = RunState::load(&runner.state.id).unwrap();
12765 assert!(disk.driver_exited);
12766 assert_eq!(disk.driver_pid, None);
12767 assert_eq!(runner.state.status, RunStatus::Blocked);
12768 assert_eq!(
12769 runner.state.resource_block.as_ref().unwrap().kind,
12770 "repository coordination"
12771 );
12772 assert!(runner.state.prep_failure.is_none());
12773 assert!(runner.state.seats.is_empty());
12774 runner.pause.park();
12775 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12778 .await
12779 .unwrap()
12780 .unwrap();
12781 assert!(runner.state.parked);
12782 assert!(runner.state.prep_failure.is_none());
12783 let disk = RunState::load(&runner.state.id).unwrap();
12784 assert!(disk.driver_exited);
12785 assert_eq!(disk.driver_pid, None);
12786 }
12787
12788 #[tokio::test]
12789 #[cfg(unix)]
12790 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12791 use std::os::fd::AsRawFd as _;
12792 let tmp = tempfile::tempdir().unwrap();
12793 let mut runner = registered_runner(tmp.path()).await;
12794 runner.state.prep_complete = Some(false);
12795 runner.state.config.graph.timeout_resource_wait = 30;
12796 runner.state.save().unwrap();
12797 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12798 assert_eq!(
12799 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12800 0
12801 );
12802 assert!(
12803 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12804 .await
12805 .is_err()
12806 );
12807 let disk = RunState::load(&runner.state.id).unwrap();
12808 assert!(disk.driver_exited);
12809 assert_eq!(disk.driver_pid, None);
12810 assert!(disk.status.resumable());
12811 assert!(runner.state.driver_exited);
12812 assert_eq!(runner.state.driver_pid, None);
12813 runner.state.save().unwrap();
12814 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12815 }
12816
12817 #[tokio::test]
12818 #[cfg(unix)]
12819 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12820 let tmp = tempfile::tempdir().unwrap();
12821 let mut runner = registered_runner(tmp.path()).await;
12822 let winner = runner.state.candidates[0].clone();
12823 let cache = tmp.path().join("cache");
12824 let started = tmp.path().join("pre-gate-started");
12825 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12826 runner.state.config.verify.gate =
12827 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12828 let home = crate::run::home();
12829 let owner =
12830 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12831 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12832 crate::cache::AcquireOutcome::Acquired(g) => g,
12833 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12834 };
12835 let mut gate = Box::pin(runner.run_gate(&winner));
12836 assert!(
12837 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12838 .await
12839 .is_err()
12840 );
12841 assert!(!started.exists(), "pre-gate must wait for the cache too");
12842 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12846 let editor = crate::repo_lock::SeatLease::checkout_with(
12847 &winner.worktree,
12848 true,
12849 crate::config::test_coordination().as_ref(),
12850 &wait,
12851 )
12852 .await
12853 .unwrap();
12854 drop(editor);
12855 guard.release();
12856 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12857 .await
12858 .unwrap()
12859 .unwrap();
12860 drop(gate);
12861 assert!(outcomes.iter().all(CommandOutcome::ok));
12862 assert!(started.exists());
12863 }
12864
12865 #[tokio::test]
12866 #[cfg(unix)]
12867 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12868 use std::os::fd::AsRawFd as _;
12869
12870 ask_test_home();
12871 let tmp = tempfile::tempdir().unwrap();
12872 let repo = tmp.path().join("repo");
12873 std::fs::create_dir_all(&repo).unwrap();
12874 init_repo(&repo);
12875 let common = repo.join(".git");
12876 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12877 let mut runner = runner_at(RunStatus::Prep);
12878 runner.state.repo = repo.clone();
12879 runner.state.config.coordination = crate::config::test_coordination();
12880 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12881 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12882 runner.state.config.graph.advise = false;
12883 runner.roles.implementers = vec![spec("implement")];
12884 runner.roles.judges = vec![spec("judge")];
12885 runner.prep().await.unwrap();
12886 let winner = runner.state.candidates[0].clone();
12887 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12890 let completed = tmp.path().join("seat-completed");
12891 runner.roles.implementers[0].command = vec![
12892 "sh".into(),
12893 "-c".into(),
12894 format!("touch '{}'; echo completed", completed.display()),
12895 ];
12896 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12897 assert_eq!(
12898 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12899 0
12900 );
12901 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12902 let mut implement = Box::pin(runner.implement());
12903 tokio::select! {
12904 result = &mut implement => panic!("rescue must wait: {result:?}"),
12905 result = tokio::time::timeout(Duration::from_secs(10), async {
12906 while !completed.exists() {
12907 tokio::time::sleep(Duration::from_millis(20)).await;
12908 }
12909 }) => result.unwrap(),
12910 }
12911 assert!(
12912 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12913 .await
12914 .is_err()
12915 );
12916 assert_eq!(
12917 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12918 before
12919 );
12920 assert_eq!(
12921 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12922 0
12923 );
12924 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12925 .await
12926 .unwrap()
12927 .unwrap();
12928 drop(implement);
12929 assert_ne!(
12930 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12931 before
12932 );
12933 assert!(git::is_clean(&winner.worktree).await.unwrap());
12934
12935 let started = tmp.path().join("pre-gate-started");
12936 let proceed = tmp.path().join("pre-gate-proceed");
12937 let checked = tmp.path().join("gate-ran");
12938 runner.state.config.verify.pre_gate = vec![format!(
12939 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12940 started.display(),
12941 proceed.display()
12942 )];
12943 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12944 assert_eq!(
12945 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12946 0
12947 );
12948 let mut gate = Box::pin(runner.run_gate(&winner));
12949 assert!(
12950 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12951 .await
12952 .is_err()
12953 );
12954 assert!(!started.exists());
12955 assert!(!checked.exists());
12956 assert!(git::is_clean(&winner.worktree).await.unwrap());
12957 assert_eq!(
12958 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12959 0
12960 );
12961 tokio::select! {
12962 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12963 result = tokio::time::timeout(Duration::from_secs(10), async {
12964 while !started.exists() {
12965 tokio::time::sleep(Duration::from_millis(20)).await;
12966 }
12967 }) => result.unwrap(),
12968 }
12969 assert_eq!(
12970 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12971 -1,
12972 "mechanical edits must hold the real lock"
12973 );
12974 std::fs::write(proceed, "ready").unwrap();
12975 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12976 .await
12977 .unwrap()
12978 .unwrap();
12979 drop(gate);
12980 assert!(outcomes.iter().all(CommandOutcome::ok));
12981 assert!(checked.exists());
12982 assert!(runner.state.pre_gate_commit.is_some());
12983 assert!(git::is_clean(&winner.worktree).await.unwrap());
12984 assert_eq!(
12985 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12986 0,
12987 "gate completion must release coordination"
12988 );
12989 drop(outsider);
12990 std::fs::remove_file(common.join("updater.lock")).unwrap();
12991 std::fs::create_dir(common.join("updater.lock")).unwrap();
12992 std::fs::remove_file(&started).unwrap();
12993 std::fs::remove_file(&checked).unwrap();
12994 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12995 let error = runner.run_gate(&winner).await.unwrap_err();
12996 assert!(
12997 error
12998 .to_string()
12999 .contains("acquire mandatory repository lock")
13000 );
13001 assert!(
13002 !started.exists(),
13003 "failed acquisition must prevent pre-gate"
13004 );
13005 assert!(!checked.exists(), "failed acquisition must prevent gate");
13006 assert!(git::is_clean(&winner.worktree).await.unwrap());
13007 assert_eq!(
13008 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13009 head
13010 );
13011 }
13012
13013 #[tokio::test]
13014 #[cfg(unix)]
13015 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13016 use std::os::fd::AsRawFd as _;
13017 use std::time::Duration;
13018
13019 ask_test_home();
13020 let tmp = tempfile::tempdir().unwrap();
13021 let repo = tmp.path().join("repo");
13022 std::fs::create_dir_all(&repo).unwrap();
13023 init_repo(&repo);
13024 let common = repo.join(".git");
13025 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13026 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13027 assert_eq!(
13028 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13029 0
13030 );
13031 let mut runner = runner_at(RunStatus::Prep);
13032 runner.state.repo = repo.clone();
13033 runner.state.config.coordination = crate::config::test_coordination();
13034 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13035 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13036 runner.state.config.graph.advise = true;
13037 runner.state.config.graph.advisors = 1;
13038 runner.roles.implementers = vec![spec("implement")];
13039 runner.roles.judges = vec![spec("judge")];
13040 let root = runner.state.worktree_root();
13041 let started = tmp.path().join("setup-started");
13042 let proceed = tmp.path().join("setup-proceed");
13043 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13044 run: Some(format!(
13045 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13046 started.display(),
13047 proceed.display()
13048 )),
13049 ..Default::default()
13050 }];
13051 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13052 let config = git::git(&repo, &["config", "--local", "--list"])
13053 .await
13054 .unwrap();
13055 let mut prep = Box::pin(runner.prep());
13056 assert!(
13057 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13058 .await
13059 .is_err()
13060 );
13061 assert!(
13062 !root.exists(),
13063 "contention must prevent even checkout creation"
13064 );
13065 assert!(!started.exists(), "contention must prevent setup");
13066 assert_eq!(
13067 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13068 branches
13069 );
13070 assert_eq!(
13071 git::git(&repo, &["config", "--local", "--list"])
13072 .await
13073 .unwrap(),
13074 config
13075 );
13076 drop(lock);
13077 tokio::select! {
13078 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13079 result = tokio::time::timeout(Duration::from_secs(10), async {
13080 while !started.exists() {
13081 tokio::time::sleep(Duration::from_millis(20)).await;
13082 }
13083 }) => result.unwrap(),
13084 }
13085 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13086 assert_eq!(
13087 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13088 -1
13089 );
13090 assert!(
13091 tokio::time::timeout(
13092 Duration::from_millis(100),
13093 crate::repo_lock::SeatLease::for_checkout(
13094 &repo,
13095 true,
13096 crate::config::test_coordination().as_ref()
13097 )
13098 )
13099 .await
13100 .is_err(),
13101 "preparation must also exclude local editors"
13102 );
13103 std::fs::write(proceed, "ready").unwrap();
13104 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13105 .await
13106 .unwrap()
13107 .unwrap();
13108 drop(prep);
13109 assert_eq!(runner.state.prep_complete, Some(true));
13110 assert!(root.join("judge-1").exists());
13111 assert!(root.join("advisor-1").exists());
13112 assert_eq!(
13113 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13114 0
13115 );
13116 }
13117
13118 #[tokio::test]
13119 #[cfg(unix)]
13120 async fn preparation_lock_failure_prevents_repository_changes() {
13121 ask_test_home();
13122 let tmp = tempfile::tempdir().unwrap();
13123 let repo = tmp.path().join("repo");
13124 std::fs::create_dir_all(&repo).unwrap();
13125 init_repo(&repo);
13126 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13127 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13130 let mut runner = runner_at(RunStatus::Prep);
13131 runner.state.repo = repo.clone();
13132 runner.state.config.coordination = crate::config::test_coordination();
13133 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13134 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13135 runner.roles.implementers = vec![spec("implement")];
13136 let error = runner.prep().await.unwrap_err();
13137 assert!(
13138 error
13139 .to_string()
13140 .contains("acquire mandatory repository lock")
13141 );
13142 assert!(!runner.state.worktree_root().exists());
13143 assert!(runner.state.candidates.is_empty());
13144 assert!(!repo.join(".git/config.worktree").exists());
13145 assert_eq!(
13146 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13147 "* main"
13148 );
13149 }
13150
13151 #[tokio::test]
13152 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13153 ask_test_home();
13154 let tmp = tempfile::tempdir().unwrap();
13155 let repo = tmp.path().join("repo");
13156 std::fs::create_dir_all(&repo).unwrap();
13157 init_repo(&repo);
13158 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13159 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13160 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13161 git::git(&repo, &["add", "."]).await.unwrap();
13162 git::git(&repo, &["commit", "-m", "required seed"])
13163 .await
13164 .unwrap();
13165 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13166 git::git(&repo, &["checkout", "main"]).await.unwrap();
13167 let mut runner = runner_at(RunStatus::Prep);
13168 runner.state.repo = repo.clone();
13169 runner.state.base_commit = base.clone();
13170 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13171 runner.state.config.graph.advise = false;
13172 runner.state.config.blind.commit_msg_hook = false;
13173 runner.state.seeds = vec![refs::Seed {
13174 token: seed.clone(),
13175 sha: seed.clone(),
13176 kind: refs::SeedKind::Unmerged,
13177 branch: false,
13178 detail: String::new(),
13179 }];
13180 runner.roles.implementers = vec![spec("one"), spec("two")];
13181 runner.roles.judges = vec![spec("judge")];
13182 let label = blind::assign_labels(2, runner.state.seed)[0];
13183 let branch = runner.state.branch_for(label);
13184 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13185 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13186 .await
13187 .unwrap();
13188 runner.state.candidates.push(Candidate {
13189 index: 0,
13190 label,
13191 agent: "one".into(),
13192 branch,
13193 worktree,
13194 summary: String::new(),
13195 stat: String::new(),
13196 files: 0,
13197 commits: 0,
13198 empty: false,
13199 failed: None,
13200 verified_noop: None,
13201 duration_ms: 0,
13202 folded: false,
13203 });
13204 runner.state.save().unwrap();
13205 runner.state = RunState::load(&runner.state.id).unwrap();
13207 assert_eq!(runner.state.prep_complete, Some(false));
13208 runner.prep().await.unwrap();
13209 assert_eq!(runner.state.prep_complete, Some(true));
13210 assert_eq!(runner.state.candidates.len(), 2);
13211 for candidate in &runner.state.candidates {
13212 assert_eq!(
13213 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13214 git::tree_of(&repo, &seed).await.unwrap()
13215 );
13216 assert_eq!(
13217 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13218 "required commit"
13219 );
13220 }
13221 assert!(runner.state.worktree_root().join("judge-1").exists());
13222 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13223 runner.prep().await.unwrap();
13224 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13225
13226 runner.state.prep_complete = Some(false);
13227 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13228 run: Some(format!(
13229 "test -f '{}'",
13230 tmp.path().join("dependency-ready").display()
13231 )),
13232 ..Default::default()
13233 }];
13234 assert!(runner.prep().await.is_err());
13235 assert!(runner.state.prep_failure.is_some());
13236 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13237 runner.state = RunState::load(&runner.state.id).unwrap();
13238 assert!(
13239 runner.prep().await.is_err(),
13240 "correcting the cause alone must not auto-retry"
13241 );
13242 let failed_at = runner.state.events.last().unwrap().at;
13243 assert!(
13244 runner.retry_preparation(failed_at).is_err(),
13245 "old approvals cannot unlock the hold"
13246 );
13247 let mut task = crate::queue::Task::new(
13248 "retry".into(),
13249 "retry setup".into(),
13250 repo.clone(),
13251 crate::queue::Source::Human,
13252 );
13253 task.start(runner.state.id.clone());
13254 task.hold_machine(Some("setup failed".into()));
13255 task.release_operator("cli", None);
13256 runner
13257 .retry_preparation(task.resumption.as_ref().unwrap().at)
13258 .unwrap();
13259 runner.prep().await.unwrap();
13260 assert!(runner.state.prep_failure.is_none());
13261 assert_eq!(runner.state.prep_complete, Some(true));
13262 assert_eq!(runner.state.candidates.len(), 2);
13263 assert!(runner.state.worktree_root().join("judge-1").exists());
13264 for candidate in &runner.state.candidates {
13265 assert_eq!(
13266 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13267 git::tree_of(&repo, &seed).await.unwrap()
13268 );
13269 }
13270 }
13271
13272 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13273 let mut heads = Vec::new();
13274 for candidate in candidates {
13275 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13276 }
13277 heads
13278 }
13279
13280 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13281 ask_test_home();
13282 let tmp = tempfile::tempdir().unwrap();
13283 let repo = tmp.path().join("repo");
13284 std::fs::create_dir(&repo).unwrap();
13285 init_repo(&repo);
13286 let mut runner = runner_at(RunStatus::Reviewing);
13287 runner.state.repo = repo.clone();
13288 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13289 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13290 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13291 let branch = format!("candidate-{label}");
13292 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13293 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13294 .await
13295 .unwrap();
13296 if label == 'B' {
13297 std::fs::write(worktree.join("change"), "winner").unwrap();
13298 git::rescue_commit(&worktree, "candidate change")
13299 .await
13300 .unwrap();
13301 }
13302 runner.state.candidates.push(Candidate {
13303 index,
13304 label,
13305 agent: "alpha".into(),
13306 branch,
13307 worktree,
13308 summary: "completed reply".into(),
13309 stat: String::new(),
13310 files: usize::from(label == 'B'),
13311 commits: usize::from(label == 'B'),
13312 empty: label == 'A',
13313 failed: None,
13314 verified_noop: None,
13315 duration_ms: 1,
13316 folded: false,
13317 });
13318 }
13319 runner.state.judge_skipped = true;
13320 runner.state.tally = Some(Tally {
13321 first_choice: BTreeMap::new(),
13322 borda: BTreeMap::new(),
13323 winner: 'B',
13324 rankings: 0,
13325 unanimous_initial: false,
13326 deliberated: false,
13327 changed_votes: 0,
13328 unanimous_final: false,
13329 tie_break: None,
13330 judges: 0,
13331 present: 0,
13332 quorum: 0,
13333 met_quorum: true,
13334 uncontested: Some("only B produced a change".into()),
13335 });
13336 runner.state.save().unwrap();
13337 (tmp, runner)
13338 }
13339
13340 #[tokio::test]
13341 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13342 let (_tmp, mut runner) = preservation_resume_fixture().await;
13343 runner.implement().await.unwrap();
13345 assert!(runner.state.candidate_preservation.is_empty());
13346 assert!(
13347 !runner
13348 .state
13349 .dir()
13350 .join("artifacts/candidate-A-recovery")
13351 .exists()
13352 );
13353 runner.fold_losers().await.unwrap();
13354 assert!(!runner.state.candidates[0].worktree.exists());
13355 for schema in [14, crate::run::SCHEMA] {
13356 runner.state.schema = schema;
13357 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13358 runner.state.save().unwrap();
13359 runner.state = RunState::load(&runner.state.id).unwrap();
13360 runner.implement().await.unwrap();
13361 assert!(runner.state.candidate_preservation.is_empty());
13362 assert!(runner.state.candidates[0].folded);
13363 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13364 }
13365 }
13366
13367 #[tokio::test]
13368 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13369 let (_tmp, mut runner) = preservation_resume_fixture().await;
13370 let wt = runner.state.candidates[0].worktree.clone();
13371 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13372 runner.state.schema = 14;
13373 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13374 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13375 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13376 for _ in 0..2 {
13377 assert!(runner.implement().await.is_err());
13378 assert_eq!(runner.state.status, RunStatus::Blocked);
13379 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13380 assert!(
13381 runner.state.candidates[0].empty,
13382 "old verdict must remain unusable"
13383 );
13384 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13385 assert!(runner.state.judge_skipped);
13386 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13387 assert!(
13388 crate::run::read_artifact(&runner.state, "cand-A.patch")
13389 .unwrap()
13390 .contains("owned edits")
13391 );
13392 assert!(wt.exists());
13393 assert_eq!(
13394 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13395 .await
13396 .unwrap(),
13397 1
13398 );
13399 runner.state = RunState::load(&runner.state.id).unwrap();
13400 }
13401 }
13402
13403 #[tokio::test]
13404 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13405 let (_tmp, mut runner) = preservation_resume_fixture().await;
13406 let wt = runner.state.candidates[0].worktree.clone();
13407 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13408 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13409 let index_lock = PathBuf::from(
13410 git::git(
13411 &wt,
13412 &[
13413 "rev-parse",
13414 "--path-format=absolute",
13415 "--git-path",
13416 "index.lock",
13417 ],
13418 )
13419 .await
13420 .unwrap(),
13421 );
13422 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13423 runner.state.schema = 14;
13424 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13425 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13426 assert!(runner.implement().await.is_err());
13427 assert!(runner.after_implement().is_err());
13428 assert!(runner.fold_losers().await.is_err());
13429 assert_eq!(
13430 std::fs::read_to_string(wt.join("recovered")).unwrap(),
13431 "failed seat edits"
13432 );
13433 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13434 runner.state = RunState::load(&runner.state.id).unwrap();
13435 std::fs::remove_file(index_lock).unwrap();
13436 assert!(runner.implement().await.is_err());
13438 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13439 assert!(runner.state.candidates[0].empty);
13440 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13441 assert!(
13442 crate::run::read_artifact(&runner.state, "cand-A.patch")
13443 .unwrap()
13444 .contains("failed seat edits")
13445 );
13446 assert!(wt.exists());
13447 }
13448
13449 #[tokio::test]
13450 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13451 let (_tmp, mut runner) = preservation_resume_fixture().await;
13452 let repo = runner.state.repo.clone();
13453 let common = git::git(
13454 &repo,
13455 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13456 )
13457 .await
13458 .unwrap();
13459 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13460 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13461 assert!(runner.fold_losers().await.is_err());
13462 assert!(!runner.state.candidates[0].worktree.exists());
13463 assert!(!runner.state.candidates[0].folded);
13464 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13465 runner.state = RunState::load(&runner.state.id).unwrap();
13466 std::fs::remove_file(branch_lock).unwrap();
13467 let judge = runner.state.worktree_root().join("judge-1");
13468 git::worktree_add_detached(&repo, &judge, "main")
13469 .await
13470 .unwrap();
13471 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13472 runner.roles.judges = vec![spec("judge")];
13473 assert!(runner.fold_losers().await.is_err());
13474 runner.state = RunState::load(&runner.state.id).unwrap();
13475 assert!(
13476 runner.state.candidates[0].folded,
13477 "candidate progress saved before judge failure"
13478 );
13479 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13480 assert!(judge.join("uncommitted").exists());
13481 std::fs::remove_file(judge.join("uncommitted")).unwrap();
13482 runner.fold_losers().await.unwrap();
13483 assert!(!judge.exists());
13484 runner.fold_losers().await.unwrap();
13485 }
13486
13487 #[tokio::test]
13488 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13489 let (_tmp, mut runner) = preservation_resume_fixture().await;
13490 runner.state.candidates.retain(|c| c.label == 'B');
13491 runner.state.tally = None;
13492 runner.state.judge_skipped = false;
13493 runner.state.prep_complete = Some(true);
13494 runner.state.config.graph.advise = false;
13495 runner.state.config.graph.selected_implementer = Some("alpha".into());
13496 let wt = runner.state.candidates[0].worktree.clone();
13497 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13498 runner
13499 .state
13500 .candidate_preservation
13501 .insert('B', "awaiting persistence".into());
13502 let index_lock = git::git(
13503 &wt,
13504 &[
13505 "rev-parse",
13506 "--path-format=absolute",
13507 "--git-path",
13508 "index.lock",
13509 ],
13510 )
13511 .await
13512 .unwrap();
13513 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13514 runner.state.save().unwrap();
13515 assert!(runner.execute_graph().await.is_err());
13516 assert_eq!(runner.state.status, RunStatus::Blocked);
13517 assert!(runner.state.candidate_preservation.contains_key(&'B'));
13518 std::fs::remove_file(index_lock).unwrap();
13519 for _ in 0..2 {
13522 runner.state = RunState::load(&runner.state.id).unwrap();
13523 runner.pause.park();
13524 runner.execute_graph().await.unwrap();
13525 assert_eq!(runner.state.status, RunStatus::Implementing);
13526 assert!(runner.state.candidate_preservation.is_empty());
13527 assert_eq!(runner.state.candidates[0].commits, 2);
13528 assert!(runner.state.tally.is_none());
13529 }
13530 runner.implement().await.unwrap();
13531 assert_eq!(runner.state.status, RunStatus::Judging);
13532 runner.state.status = RunStatus::Blocked;
13534 runner.state.candidates[0].failed = Some("implementation failed".into());
13535 runner.state.save().unwrap();
13536 let error = runner.execute_graph().await.unwrap_err();
13537 assert!(
13538 error
13539 .to_string()
13540 .contains("selected implementation is held")
13541 );
13542 }
13543
13544 #[tokio::test]
13545 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13546 let (_tmp, mut runner) = preservation_resume_fixture().await;
13547 let repo = runner.state.repo.clone();
13548 let candidate = runner.state.candidates[0].clone();
13549 let registered = git::worktree_holding(&repo, &candidate.branch)
13552 .await
13553 .unwrap()
13554 .unwrap();
13555 assert_eq!(
13556 registered.canonicalize().unwrap(),
13557 candidate.worktree.canonicalize().unwrap()
13558 );
13559 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13560 assert_eq!(
13561 git::worktree_holding(&repo, &candidate.branch)
13562 .await
13563 .unwrap(),
13564 Some(registered)
13565 );
13566 assert!(
13567 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13568 "the stale registration initially keeps the branch checked out"
13569 );
13570 runner.fold_losers().await.unwrap();
13571 assert!(runner.state.candidates[0].folded);
13572 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13573 assert_eq!(
13574 git::worktree_holding(&repo, &candidate.branch)
13575 .await
13576 .unwrap(),
13577 None
13578 );
13579 runner.state = RunState::load(&runner.state.id).unwrap();
13580 runner.fold_losers().await.unwrap();
13581 }
13582
13583 #[tokio::test]
13584 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13585 let (_tmp, mut runner) = preservation_resume_fixture().await;
13586 let repo = runner.state.repo.clone();
13587 let candidate = runner.state.candidates[0].clone();
13588 let winner = runner.state.candidates[1].clone();
13589 let registered = git::worktree_holding(&repo, &candidate.branch)
13590 .await
13591 .unwrap()
13592 .unwrap();
13593 assert_eq!(
13594 registered.canonicalize().unwrap(),
13595 candidate.worktree.canonicalize().unwrap()
13596 );
13597 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13598 assert_eq!(
13599 git::worktree_holding(&repo, &candidate.branch)
13600 .await
13601 .unwrap(),
13602 Some(registered)
13603 );
13604 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13605
13606 for _ in 0..2 {
13607 fold_run(&mut runner.state, false, &crate::run::home())
13608 .await
13609 .unwrap();
13610 runner.state = RunState::load(&runner.state.id).unwrap();
13611 assert!(runner.state.candidates[0].folded);
13612 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13613 assert_eq!(
13614 git::worktree_holding(&repo, &candidate.branch)
13615 .await
13616 .unwrap(),
13617 None
13618 );
13619 assert!(!runner.state.candidates[1].folded);
13620 assert!(winner.worktree.exists());
13621 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13622 }
13623 }
13624
13625 #[tokio::test]
13626 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13627 ask_test_home();
13628 let tmp = tempfile::tempdir().unwrap();
13629 let repo = tmp.path().join("repo");
13630 std::fs::create_dir(&repo).unwrap();
13631 init_repo(&repo);
13632 let mut runner = runner_at(RunStatus::Implementing);
13633 runner.state.repo = repo.clone();
13634 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13635 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13636 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13637 let branch = format!("candidate-{label}");
13638 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13639 git::worktree_add_branch(&repo, &wt, &branch, "main")
13640 .await
13641 .unwrap();
13642 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13643 git::git(&wt, &["add", "README.md"]).await.unwrap();
13644 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13645 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13646 runner.state.candidates.push(Candidate {
13647 index,
13648 label,
13649 agent: "alpha".to_owned(),
13650 branch,
13651 worktree: wt,
13652 summary: "completed implementation".to_owned(),
13653 stat: String::new(),
13654 files: 0,
13655 commits: 0,
13656 empty: false,
13657 failed: None,
13658 verified_noop: None,
13659 duration_ms: 42,
13660 folded: false,
13661 });
13662 runner
13663 .state
13664 .candidate_preservation
13665 .insert(label, "awaiting persistence".to_owned());
13666 }
13667 let wt = runner.state.candidates[0].worktree.clone();
13668 let index_lock = git::git(
13669 &wt,
13670 &[
13671 "rev-parse",
13672 "--path-format=absolute",
13673 "--git-path",
13674 "index.lock",
13675 ],
13676 )
13677 .await
13678 .unwrap();
13679 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13680 runner.state.save().unwrap();
13681 assert!(runner.persist_candidates().await.is_err());
13682 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13683 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13684 assert_eq!(
13685 runner.state.candidates[1].commits, 1,
13686 "other results are still collected"
13687 );
13688 assert!(
13689 !runner.state.candidates[0].empty,
13690 "failed rescue is not no change"
13691 );
13692 assert_eq!(
13693 runner.state.candidates[0].summary,
13694 "completed implementation"
13695 );
13696 assert!(runner.after_implement().is_err());
13697 let home = tmp.path().join("fold-home");
13698 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13699 assert!(wt.join("new-file").exists());
13700 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13701 assert!(recovery.join("staged.patch").exists());
13702 assert!(recovery.join("unstaged.patch").exists());
13703 assert_eq!(
13704 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13705 "untracked edits\n"
13706 );
13707 runner.state.begin_pass();
13708 runner.state.end_pass_parked(ParkCause::Upgrade {
13709 from: "1.0".into(),
13710 to: Some("1.1".into()),
13711 });
13712 runner.state.save().unwrap();
13713 runner.state = RunState::load(&runner.state.id).unwrap();
13714 assert!(matches!(
13715 runner.state.passes.last().unwrap().end,
13716 Some(crate::run::PassEnd::Parked {
13717 cause: ParkCause::Upgrade { .. }
13718 })
13719 ));
13720 std::fs::remove_file(index_lock).unwrap();
13721 runner.implement().await.unwrap();
13723 assert!(runner.state.candidate_preservation.is_empty());
13724 assert!(
13725 runner
13726 .state
13727 .candidates
13728 .iter()
13729 .all(|c| c.commits == 1 && !c.empty)
13730 );
13731 assert!(git::is_clean(&wt).await.unwrap());
13732 assert!(
13733 crate::run::read_artifact(&runner.state, "cand-A.patch")
13734 .unwrap()
13735 .contains("later tracked edits")
13736 );
13737 }
13738
13739 #[tokio::test]
13740 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13741 ask_test_home();
13742 let tmp = tempfile::tempdir().unwrap();
13743 let repo = tmp.path().join("repo");
13744 std::fs::create_dir(&repo).unwrap();
13745 init_repo(&repo);
13746 let wt = tmp.path().join("candidate");
13747 git::worktree_add_branch(&repo, &wt, "candidate", "main")
13748 .await
13749 .unwrap();
13750 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13751 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13752 assert!(wt.join("uncommitted").exists());
13753 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13754 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13755 let mut runner = runner_at(RunStatus::Implementing);
13756 runner.state.repo = repo.clone();
13757 runner.state.base_commit = "missing-base".to_owned();
13758 runner.state.candidates.push(Candidate {
13759 index: 0,
13760 label: 'A',
13761 agent: "alpha".into(),
13762 branch: "candidate".into(),
13763 worktree: wt.clone(),
13764 summary: "completed".into(),
13765 stat: String::new(),
13766 files: 0,
13767 commits: 0,
13768 empty: false,
13769 failed: None,
13770 verified_noop: None,
13771 duration_ms: 1,
13772 folded: false,
13773 });
13774 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13775 assert!(
13776 fold_run(&mut runner.state, true, &tmp.path().join("home"))
13777 .await
13778 .is_err()
13779 );
13780 assert!(!runner.state.candidates[0].folded);
13781 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13782 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13783 runner
13784 .state
13785 .candidate_preservation
13786 .insert('A', "pending".into());
13787 assert!(runner.persist_candidates().await.is_err());
13788 assert!(!runner.state.candidates[0].empty);
13789 assert!(!runner.state.candidates[0].folded);
13790 assert!(wt.exists());
13791 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13792 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13793 std::fs::create_dir_all(&patch_path).unwrap();
13794 assert!(
13795 runner.persist_candidates().await.is_err(),
13796 "artifact failure stays pending"
13797 );
13798 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13799 assert!(!runner.state.candidates[0].empty);
13800 std::fs::remove_dir(patch_path).unwrap();
13801 runner.persist_candidates().await.unwrap();
13802 assert!(
13803 runner.state.candidates[0].empty,
13804 "only successful inspection confirms no change"
13805 );
13806 }
13807
13808 fn init_repo(dir: &Path) {
13809 let run = |args: &[&str]| {
13810 let out = std::process::Command::new("git")
13811 .args(args)
13812 .current_dir(dir)
13813 .quiet()
13814 .output()
13815 .expect("spawn git");
13816 assert!(
13817 out.status.success(),
13818 "git {args:?} failed: {}",
13819 String::from_utf8_lossy(&out.stderr)
13820 );
13821 };
13822 run(&["init", "-b", "main"]);
13823 run(&["config", "user.name", "magi test"]);
13824 run(&["config", "user.email", "magi@example.com"]);
13825 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13826 run(&["add", "-A"]);
13827 run(&["commit", "-m", "init"]);
13828 }
13829
13830 fn ask_test_home() {
13838 crate::run::pin_test_home();
13839 }
13840
13841 fn runner_at(status: RunStatus) -> Runner {
13844 let mut state = RunState::new(
13845 PathBuf::from("/nonexistent/repo"),
13846 "main".to_owned(),
13847 "deadbeef".to_owned(),
13848 "task".to_owned(),
13849 Config::default(),
13850 );
13851 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13854 state.id = format!(
13855 "20000101-000000-{:04x}",
13856 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13857 );
13858 state.status = status;
13859 Runner {
13860 state,
13861 roles: ResolvedRoles {
13862 implementers: Vec::new(),
13863 judges: Vec::new(),
13864 reviewers: Vec::new(),
13865 fixer: None,
13866 conductor: conductor(),
13867 implementer_roster: Vec::new(),
13868 judge_roster: Vec::new(),
13869 reviewer_roster: Vec::new(),
13870 },
13871 sem: Arc::new(Semaphore::new(1)),
13872 pause: Pause::new(),
13873 interrupt: Pause::new(),
13874 }
13875 }
13876
13877 #[test]
13878 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13879 crate::run::pin_test_home();
13880 let mut runner = runner_at(RunStatus::Gating);
13881 runner.state.save().unwrap();
13882 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13885 assert!(runner.claim_driver().is_err());
13886 assert!(runner.state.driver_exited);
13887 assert_eq!(runner.state.driver_pid, None);
13888 let disk = RunState::load(&runner.state.id).unwrap();
13889 assert!(disk.driver_exited);
13890 assert_eq!(disk.driver_pid, None);
13891 assert_eq!(disk.status, RunStatus::Gating);
13892 }
13893
13894 #[tokio::test]
13895 async fn an_early_graph_error_releases_its_driver() {
13896 crate::run::pin_test_home();
13897 let mut runner = runner_at(RunStatus::Blocked);
13898 runner.state.config.graph.selected_implementer = Some("selected".into());
13899 runner.state.save().unwrap();
13900 assert!(runner.execute().await.is_err());
13901 let disk = RunState::load(&runner.state.id).unwrap();
13902 assert!(disk.driver_exited);
13903 assert_eq!(disk.driver_pid, None);
13904 assert_eq!(disk.status, RunStatus::Blocked);
13905 }
13906
13907 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13908 crate::run::MergeRepair {
13909 question: "repair-question".into(),
13910 head: "old-head".into(),
13911 pr: "https://github.com/o/r/pull/42".into(),
13912 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13913 gate: vec![outcome(Some(0), false)],
13914 gate_head: Some("old-head".into()),
13915 gate_fixes: Vec::new(),
13916 pre_gate: Vec::new(),
13917 phase,
13918 failure: None,
13919 }
13920 }
13921
13922 #[tokio::test]
13923 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13924 crate::run::pin_test_home();
13925 let mut runner = runner_at(RunStatus::Landing);
13926 runner.state.merge_repairs.push(merge_repair_request(
13927 crate::run::MergeRepairPhase::Requested,
13928 ));
13929 runner.state.save().unwrap();
13930 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13931 assert!(
13932 runner
13933 .execute()
13934 .await
13935 .unwrap_err()
13936 .to_string()
13937 .contains("already running")
13938 );
13939 assert_eq!(
13940 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13941 crate::run::MergeRepairPhase::Requested
13942 );
13943 drop(claim);
13944 let mut disk = runner.state.clone();
13945 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13946 disk.save().unwrap();
13947 assert!(
13948 runner
13949 .execute()
13950 .await
13951 .unwrap_err()
13952 .to_string()
13953 .contains("decision changed")
13954 );
13955 assert_eq!(
13956 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13957 crate::run::MergeRepairPhase::Complete
13958 );
13959 }
13960
13961 #[test]
13962 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13963 crate::run::pin_test_home();
13964 let mut runner = runner_at(RunStatus::Landing);
13965 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13966 runner.state.reviews = request.reviews.clone();
13967 runner.state.gate = request.gate.clone();
13968 runner.state.gate_ran = true;
13969 runner.state.land_approval = Some(crate::run::LandApproval {
13970 question: "old".into(),
13971 head: "old-head".into(),
13972 });
13973 runner.state.merge_repairs.push(request);
13974 runner.begin_merge_repair_review(0).unwrap();
13975 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13976 assert!(!runner.state.gate_status().ok());
13977 assert!(runner.state.land_approval.is_none());
13978 let disk = RunState::load(&runner.state.id).unwrap();
13979 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13980 assert!(disk.merge_repairs[0].gate[0].ok());
13981 assert_eq!(
13982 disk.merge_repairs[0].phase,
13983 crate::run::MergeRepairPhase::Reviewing
13984 );
13985 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13986 assert!(disk.reviews.is_empty());
13987 }
13988
13989 #[tokio::test]
13990 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13991 crate::run::pin_test_home();
13992 for phase in [
13993 crate::run::MergeRepairPhase::Fixing,
13994 crate::run::MergeRepairPhase::Requested,
13995 ] {
13996 let mut runner = runner_at(RunStatus::Landing);
13997 runner.state.config.graph.land_rounds = 0;
13998 runner.state.merge_repairs.push(merge_repair_request(phase));
13999 runner.state.save().unwrap();
14000 assert!(!runner.repair_merge_review().await.unwrap());
14001 assert_eq!(runner.state.status, RunStatus::Blocked);
14002 assert_eq!(
14003 runner.state.merge_repairs[0].phase,
14004 crate::run::MergeRepairPhase::Failed
14005 );
14006 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14007 assert!(
14008 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14009 "interrupted"
14010 } else {
14011 "budget exhausted"
14012 })
14013 );
14014 assert!(runner.state.merge.is_none());
14015 }
14016 }
14017
14018 #[test]
14022 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14023 crate::run::pin_test_home();
14024 let mut runner = runner_at(RunStatus::Implementing);
14025 let interrupt = Pause::new();
14026 runner.watch_interrupt(interrupt.clone());
14027
14028 interrupt.park_because("task a1b2 asked to run first");
14029
14030 assert!(runner.park_here().expect("park_here"));
14031 assert!(runner.state.parked);
14032 let last = runner.state.events.last().expect("a park event");
14033 assert_eq!(last.node, "park");
14034 assert!(
14035 last.message.contains("task a1b2 asked to run first"),
14036 "expected the interrupt reason in {:?}",
14037 last.message
14038 );
14039 }
14040
14041 #[test]
14044 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14045 crate::run::pin_test_home();
14046 let mut runner = runner_at(RunStatus::Implementing);
14047 runner.state.begin_pass();
14048 let shutdown = Pause::new();
14049 runner.on_pause(shutdown.clone());
14050 let upgrade = ParkCause::Upgrade {
14051 from: "0.121.0".to_owned(),
14052 to: Some("0.121.1".to_owned()),
14053 };
14054 shutdown.park_for(upgrade.clone());
14055 shutdown.park_for(ParkCause::Operator);
14056
14057 assert!(runner.park_here().expect("park_here"));
14058 assert_eq!(
14059 runner.state.passes[0].end,
14060 Some(crate::run::PassEnd::Parked { cause: upgrade })
14061 );
14062 let last = runner.state.events.last().expect("a park event");
14063 assert!(last.message.contains("0.121.0"), "{}", last.message);
14064 }
14065
14066 #[test]
14074 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14075 crate::run::pin_test_home();
14076 let mut runner = runner_at(RunStatus::Implementing);
14077 let shutdown = Pause::new();
14078 runner.on_pause(shutdown.clone());
14079 let interrupt = Pause::new();
14080 runner.watch_interrupt(interrupt.clone());
14081
14082 assert!(!runner.park_here().expect("park_here"));
14084 assert!(!runner.state.parked);
14085
14086 interrupt.park_because("test");
14088 assert!(!shutdown.parked());
14089 assert!(runner.park_here().expect("park_here"));
14090 }
14091
14092 #[tokio::test]
14106 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14107 crate::run::pin_test_home();
14108 let mut runner = runner_at(RunStatus::Implementing);
14109 let interrupt = Pause::new();
14110 runner.watch_interrupt(interrupt.clone());
14111
14112 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14113 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14114
14115 let node = async move {
14119 started_tx.send(()).expect("send started");
14120 finish_rx.await.expect("recv finish");
14121 "node finished"
14122 };
14123
14124 let interrupter = async move {
14125 started_rx.await.expect("recv started");
14126 interrupt.park_because("higher-priority task waiting");
14128 tokio::task::yield_now().await;
14132 finish_tx.send(()).expect("send finish");
14133 };
14134
14135 let (node_result, ()) = tokio::join!(node, interrupter);
14136 assert_eq!(
14137 node_result, "node finished",
14138 "the in-flight call ran to completion"
14139 );
14140
14141 assert!(runner.park_here().expect("park_here"));
14144 assert!(runner.state.parked);
14145 }
14146
14147 #[test]
14153 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14154 crate::run::pin_test_home();
14155 let mut runner = runner_at(RunStatus::Judging);
14156 runner.state.config.agents = vec![conductor()];
14160 runner.state.candidates = vec![Candidate {
14161 index: 0,
14162 label: 'A',
14163 agent: "alpha".to_owned(),
14164 branch: "magi/x/A".to_owned(),
14165 worktree: PathBuf::from("/nonexistent/worktree"),
14166 summary: "did the thing".to_owned(),
14167 stat: "1 file changed".to_owned(),
14168 files: 1,
14169 commits: 1,
14170 empty: false,
14171 failed: None,
14172 verified_noop: None,
14173 duration_ms: 1234,
14174 folded: false,
14175 }];
14176 let run_id = runner.state.id.clone();
14177
14178 let interrupt = Pause::new();
14179 runner.watch_interrupt(interrupt.clone());
14180 interrupt.park_because("task c3d4 asked to run first");
14181 assert!(runner.park_here().expect("park_here"));
14182
14183 let resumed = Runner::resume(&run_id).expect("resume");
14184 assert_eq!(resumed.state.candidates.len(), 1);
14185 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14186 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14187 assert_eq!(resumed.state.status, runner.state.status);
14188 assert!(
14189 resumed.state.parked,
14190 "still parked until `execute` actually walks the graph again"
14191 );
14192 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14193 }
14194
14195 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14197 let mut q = ask::Question::new(
14198 run.to_owned(),
14199 "implement".to_owned(),
14200 "impl-A".to_owned(),
14201 "Which storage backend should the cache use?".to_owned(),
14202 String::new(),
14203 vec!["SQLite".to_owned(), "Redis".to_owned()],
14204 );
14205 store.put(&mut q).unwrap();
14206 q
14207 }
14208
14209 #[test]
14210 fn a_failed_runs_open_question_is_abandoned() {
14211 ask_test_home();
14212 let store = ask::Questions::open();
14213 let mut runner = runner_at(RunStatus::Failed);
14214 let run = runner.state.id.clone();
14215 let q = ask_open_question(&store, &run);
14216
14217 runner.settle_questions();
14218
14219 let back = store.get(&q.id).unwrap();
14220 assert!(
14221 !back.status.open(),
14222 "the seat that asked died with the run; nobody is left to read an answer"
14223 );
14224 assert!(
14225 back.detail.contains(&run) && back.detail.contains("failed"),
14226 "the reason names what the run became, not just that it is gone: {}",
14227 back.detail
14228 );
14229 }
14230
14231 #[test]
14232 fn a_merged_runs_open_question_is_abandoned_too() {
14233 ask_test_home();
14234 let store = ask::Questions::open();
14235 for status in [RunStatus::Merged, RunStatus::Ready] {
14238 let mut runner = runner_at(status);
14239 let run = runner.state.id.clone();
14240 let q = ask_open_question(&store, &run);
14241
14242 runner.settle_questions();
14243
14244 let back = store.get(&q.id).unwrap();
14245 assert!(
14246 !back.status.open(),
14247 "{status:?} run's question must not outlive the run"
14248 );
14249 }
14250 }
14251
14252 #[test]
14253 fn a_still_resumable_runs_open_question_is_left_alone() {
14254 ask_test_home();
14255 let store = ask::Questions::open();
14256 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14262 let mut runner = runner_at(status);
14263 let run = runner.state.id.clone();
14264 let q = ask_open_question(&store, &run);
14265
14266 runner.settle_questions();
14267
14268 let back = store.get(&q.id).unwrap();
14269 assert!(
14270 back.status.open(),
14271 "{status:?} is still alive; the question must still be waiting"
14272 );
14273 }
14274 }
14275
14276 #[test]
14277 fn settle_questions_never_touches_an_already_answered_question() {
14278 ask_test_home();
14279 let store = ask::Questions::open();
14280 let mut runner = runner_at(RunStatus::Failed);
14281 let run = runner.state.id.clone();
14282 let mut q = ask_open_question(&store, &run);
14283 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14284 .unwrap();
14285 store.put(&mut q).unwrap();
14286
14287 runner.settle_questions();
14292 runner.settle_questions();
14293
14294 let back = store.get(&q.id).unwrap();
14295 assert_eq!(
14296 back.status,
14297 ask::QuestionStatus::Answered,
14298 "a real answer is a decision on record, never overwritten by a sweep"
14299 );
14300 }
14301
14302 #[tokio::test]
14313 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14314 crate::run::pin_test_home();
14315 let tmp = tempfile::tempdir().expect("tempdir");
14316 let repo = tmp.path().join("repo");
14317 std::fs::create_dir_all(&repo).unwrap();
14318 init_repo(&repo);
14319
14320 let mut config = Config::default();
14321 config.graph.worktree_root = Some(tmp.path().join("wt"));
14322
14323 let mut state = RunState::new(
14324 repo.clone(),
14325 "main".to_owned(),
14326 "deadbeef".to_owned(),
14327 "task".to_owned(),
14328 config,
14329 );
14330 let root = state.worktree_root();
14331 let wt_a = root.join("cand-A");
14332 let wt_b = root.join("cand-B");
14333 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14334 .await
14335 .expect("worktree A");
14336 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14337 .await
14338 .expect("worktree B");
14339
14340 state.candidates = vec![
14341 Candidate {
14342 index: 0,
14343 label: 'A',
14344 agent: "alpha".to_owned(),
14345 branch: "magi/x/A".to_owned(),
14346 worktree: wt_a.clone(),
14347 summary: String::new(),
14348 stat: String::new(),
14349 files: 0,
14350 commits: 0,
14351 empty: false,
14352 failed: None,
14353 verified_noop: None,
14354 duration_ms: 0,
14355 folded: false,
14356 },
14357 Candidate {
14358 index: 1,
14359 label: 'B',
14360 agent: "beta".to_owned(),
14361 branch: "magi/x/B".to_owned(),
14362 worktree: wt_b.clone(),
14363 summary: String::new(),
14364 stat: String::new(),
14365 files: 0,
14366 commits: 0,
14367 empty: false,
14368 failed: None,
14369 verified_noop: None,
14370 duration_ms: 0,
14371 folded: false,
14372 },
14373 ];
14374 state.tally = Some(Tally {
14375 first_choice: BTreeMap::from([('A', 1)]),
14376 borda: BTreeMap::new(),
14377 winner: 'A',
14378 rankings: 1,
14379 unanimous_initial: true,
14380 deliberated: false,
14381 changed_votes: 0,
14382 unanimous_final: true,
14383 tie_break: None,
14384 judges: 1,
14385 present: 1,
14386 quorum: 1,
14387 met_quorum: true,
14388 uncontested: None,
14389 });
14390 state.status = RunStatus::Ready;
14391
14392 fold_run(&mut state, false, &crate::run::home())
14393 .await
14394 .expect("fold_run");
14395
14396 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14397 assert!(
14398 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14399 "the unmerged winner's branch survives"
14400 );
14401 assert!(
14402 !state.candidates[0].folded,
14403 "the winner is not marked folded"
14404 );
14405
14406 assert!(!wt_b.exists(), "the loser's worktree is removed");
14407 assert!(
14408 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14409 "the loser's branch is removed"
14410 );
14411 assert!(state.candidates[1].folded, "the loser is marked folded");
14412 }
14413
14414 #[tokio::test]
14415 #[cfg(unix)]
14416 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14417 use std::os::fd::AsRawFd as _;
14418 use std::time::Duration;
14419
14420 let tmp = tempfile::tempdir().unwrap();
14421 let repo = tmp.path().join("repo");
14422 std::fs::create_dir_all(&repo).unwrap();
14423 init_repo(&repo);
14424 let common = repo.join(".git");
14425 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14426 let home = tmp.path().join("home");
14427 let mut config = Config::default();
14428 config.coordination = crate::config::test_coordination();
14429 config.graph.worktree_root = Some(tmp.path().join("wt"));
14430 config.graph.timeout_resource_wait = 1;
14431 let mut state = RunState::new(
14432 repo.clone(),
14433 "main".into(),
14434 "deadbeef".into(),
14435 "task".into(),
14436 config,
14437 );
14438 let root = state.worktree_root();
14439 let candidate = root.join("cand-A");
14440 let reviewer = root.join("review-1");
14441 let branch = state.branch_for('A');
14442 git::worktree_add_branch(&repo, &candidate, &branch, "main")
14443 .await
14444 .unwrap();
14445 git::worktree_add_detached(&repo, &reviewer, "main")
14446 .await
14447 .unwrap();
14448 git::acquire_worktree_config(&repo).await.unwrap();
14449 state.enabled_worktree_config = true;
14450 state.status = RunStatus::Ready;
14451 state.candidates.push(Candidate {
14452 index: 0,
14453 label: 'A',
14454 agent: "implement".into(),
14455 branch: branch.clone(),
14456 worktree: candidate.clone(),
14457 summary: String::new(),
14458 stat: String::new(),
14459 files: 0,
14460 commits: 0,
14461 empty: false,
14462 failed: None,
14463 verified_noop: None,
14464 duration_ms: 0,
14465 folded: false,
14466 });
14467 state.save_under(&home).unwrap();
14468 let config_before = std::fs::read(common.join("config")).unwrap();
14469 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14470 .await
14471 .unwrap();
14472 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14475 assert_eq!(
14476 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14477 0
14478 );
14479 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14480 assert!(
14481 tokio::time::timeout(Duration::from_millis(200), &mut fold)
14482 .await
14483 .is_err()
14484 );
14485 assert!(candidate.exists());
14486 assert!(reviewer.exists());
14487 assert!(git::branch_exists(&repo, &branch).await.unwrap());
14488 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14489 assert_eq!(
14490 git::git(&repo, &["worktree", "list", "--porcelain"])
14491 .await
14492 .unwrap(),
14493 worktrees_before
14494 );
14495 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14496 .await
14497 .unwrap()
14498 .unwrap_err();
14499 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14500 drop(fold);
14501 assert!(candidate.exists() && reviewer.exists());
14502 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14503 assert_eq!(
14504 state.resource_block.as_ref().unwrap().kind,
14505 "repository coordination"
14506 );
14507 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14508 drop(external);
14509 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14510 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14511 .await
14512 .unwrap()
14513 .unwrap();
14514 drop(fold);
14515 assert!(removed.contains(&branch));
14516 assert!(!root.exists());
14517 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14518 assert!(state.candidates[0].folded);
14519 assert!(!state.enabled_worktree_config);
14520 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14521 let enabled = git::git_raw(
14522 &repo,
14523 &["config", "--local", "--get", "extensions.worktreeConfig"],
14524 )
14525 .await
14526 .unwrap();
14527 assert!(!enabled.ok(), "the last config reference was released");
14528 }
14529
14530 #[tokio::test]
14533 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14534 let tmp = tempfile::tempdir().expect("tempdir");
14535 let repo = tmp.path().join("repo");
14536 std::fs::create_dir_all(&repo).unwrap();
14537 init_repo(&repo);
14538 let home = tmp.path().join("home");
14539
14540 let mut config = Config::default();
14541 config.graph.worktree_root = Some(tmp.path().join("wt"));
14542 let mut state = RunState::new(
14543 repo.clone(),
14544 "main".to_owned(),
14545 "deadbeef".to_owned(),
14546 "task".to_owned(),
14547 config,
14548 );
14549 git::git(&repo, &["branch", "magi/x/A", "main"])
14551 .await
14552 .expect("branch");
14553 state.candidates = vec![Candidate {
14554 index: 0,
14555 label: 'A',
14556 agent: "alpha".to_owned(),
14557 branch: "magi/x/A".to_owned(),
14558 worktree: state.worktree_root().join("cand-A"),
14559 summary: String::new(),
14560 stat: String::new(),
14561 files: 0,
14562 commits: 0,
14563 empty: false,
14564 failed: None,
14565 verified_noop: None,
14566 duration_ms: 0,
14567 folded: true,
14568 }];
14569 state.released_to = Some("20260901-000000-new1".to_owned());
14570 state.released_branches = vec!["magi/x/A".to_owned()];
14571
14572 fold_run(&mut state, true, &home).await.expect("fold_run");
14573
14574 assert!(
14575 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14576 "the handed-over branch survives a fold"
14577 );
14578 }
14579
14580 #[tokio::test]
14584 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14585 let tmp = tempfile::tempdir().expect("tempdir");
14586 let repo = tmp.path().join("repo");
14587 std::fs::create_dir_all(&repo).unwrap();
14588 init_repo(&repo);
14589 let run = |args: &[&str]| {
14590 let out = std::process::Command::new("git")
14591 .quiet()
14592 .args(args)
14593 .current_dir(&repo)
14594 .output()
14595 .expect("spawn git");
14596 assert!(out.status.success(), "git {args:?}");
14597 };
14598 run(&["branch", "magi/x/A"]);
14599 run(&["checkout", "-q", "-b", "magi/x/B"]);
14600 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14601 run(&["add", "-A"]);
14602 run(&["commit", "-q", "-m", "work"]);
14603 run(&["checkout", "-q", "main"]);
14604
14605 let probe = RunState::new(
14609 repo.clone(),
14610 "main".to_owned(),
14611 "deadbeef".to_owned(),
14612 "task".to_owned(),
14613 Config::default(),
14614 );
14615 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14616 let bare = tmp.path().join("origin.git");
14617 let out = std::process::Command::new("git")
14618 .quiet()
14619 .args(["init", "-q", "--bare"])
14620 .arg(&bare)
14621 .output()
14622 .expect("spawn git");
14623 assert!(out.status.success(), "git init --bare");
14624 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14625 run(&["push", "-q", "origin", "main"]);
14626
14627 let mut state = RunState::new(
14628 repo.clone(),
14629 "main".to_owned(),
14630 "deadbeef".to_owned(),
14631 "task".to_owned(),
14632 Config::default(),
14633 );
14634 state.seeds = vec![refs::Seed {
14635 token: "magi/27b2/A".to_owned(),
14636 kind: refs::SeedKind::Unresolved,
14637 sha: String::new(),
14638 branch: true,
14639 detail: "no branch or commit named magi/27b2/A".to_owned(),
14640 }];
14641
14642 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14643 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14644 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14645 let detail = empty_candidate_detail(&state, "main");
14646 assert!(detail.starts_with("empty candidate"), "{detail}");
14647 assert!(detail.contains("magi/27b2/A"), "{detail}");
14648 }
14649
14650 #[tokio::test]
14659 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14660 let tmp = tempfile::tempdir().expect("tempdir");
14661 let repo = tmp.path().join("repo");
14662 std::fs::create_dir_all(&repo).unwrap();
14663 init_repo(&repo);
14664
14665 let mut config = Config::default();
14666 config.merge.mode = MergeMode::Local;
14667
14668 let mut state = RunState::new(
14669 repo.clone(),
14670 "main".to_owned(),
14671 "deadbeef".to_owned(),
14672 "task".to_owned(),
14673 config,
14674 );
14675 state.candidates = vec![Candidate {
14676 index: 0,
14677 label: 'A',
14678 agent: "alpha".to_owned(),
14679 branch: "does-not-exist".to_owned(),
14680 worktree: repo.clone(),
14681 summary: String::new(),
14682 stat: String::new(),
14683 files: 0,
14684 commits: 0,
14685 empty: false,
14686 failed: None,
14687 verified_noop: None,
14688 duration_ms: 0,
14689 folded: false,
14690 }];
14691 state.tally = Some(Tally {
14692 first_choice: BTreeMap::from([('A', 1)]),
14693 borda: BTreeMap::new(),
14694 winner: 'A',
14695 rankings: 1,
14696 unanimous_initial: true,
14697 deliberated: false,
14698 changed_votes: 0,
14699 unanimous_final: true,
14700 tie_break: None,
14701 judges: 0,
14702 present: 0,
14703 quorum: 0,
14704 met_quorum: true,
14705 uncontested: Some("only candidate A produced a change".to_owned()),
14706 });
14707 state.reviews = vec![ReviewRound {
14708 round: 1,
14709 head: "deadbeef".to_owned(),
14710 verified_head: None,
14711 verified_at: None,
14712 reviews: Vec::new(),
14713 e2e: Vec::new(),
14714 fix: None,
14715 blocking: 0,
14716 answered: 0,
14717 expected: 0,
14718 clean: true,
14719 verify_retried: false,
14720 e2e_deferred: false,
14721 e2e_defer_reason: None,
14722 progressed: false,
14723 vote_split: false,
14724 reconsideration: Vec::new(),
14725 verdict: None,
14726 }];
14727 state.gate = vec![CommandOutcome {
14728 command: "test".to_owned(),
14729 code: Some(0),
14730 output_tail: String::new(),
14731 duration_ms: 0,
14732 resource_blocked: false,
14733 }];
14734 state.gate_ran = true;
14735 state.status = RunStatus::Ready;
14740 state.merge = Some(MergeOutcome {
14741 mode: MergeMode::Local,
14742 ok: false,
14743 detail: "already concluded".to_owned(),
14744 empty: false,
14745 });
14746
14747 let mut runner = Runner {
14748 state,
14749 roles: ResolvedRoles {
14750 implementers: Vec::new(),
14751 judges: Vec::new(),
14752 reviewers: Vec::new(),
14753 fixer: None,
14754 conductor: conductor(),
14755 implementer_roster: Vec::new(),
14756 judge_roster: Vec::new(),
14757 reviewer_roster: Vec::new(),
14758 },
14759 sem: Arc::new(Semaphore::new(1)),
14760 pause: Pause::new(),
14761 interrupt: Pause::new(),
14762 };
14763
14764 runner.merge().await.expect("merge");
14765
14766 assert_eq!(
14767 runner.state.status,
14768 RunStatus::Ready,
14769 "a concluded run's status must not change on reentry"
14770 );
14771 assert_eq!(
14772 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14773 Some("already concluded"),
14774 "merge must not run again once the node already recorded an outcome"
14775 );
14776 }
14777
14778 #[tokio::test]
14787 async fn merge_refuses_a_gate_that_has_not_actually_run() {
14788 let tmp = tempfile::tempdir().expect("tempdir");
14789 let repo = tmp.path().join("repo");
14790 std::fs::create_dir_all(&repo).unwrap();
14791 init_repo(&repo);
14792
14793 let mut config = Config::default();
14794 config.merge.mode = MergeMode::Local;
14795
14796 let mut state = RunState::new(
14797 repo.clone(),
14798 "main".to_owned(),
14799 "deadbeef".to_owned(),
14800 "task".to_owned(),
14801 config,
14802 );
14803 state.candidates = vec![Candidate {
14804 index: 0,
14805 label: 'A',
14806 agent: "alpha".to_owned(),
14807 branch: "does-not-exist".to_owned(),
14808 worktree: repo.clone(),
14809 summary: String::new(),
14810 stat: String::new(),
14811 files: 0,
14812 commits: 0,
14813 empty: false,
14814 failed: None,
14815 verified_noop: None,
14816 duration_ms: 0,
14817 folded: false,
14818 }];
14819 state.tally = Some(Tally {
14820 first_choice: BTreeMap::from([('A', 1)]),
14821 borda: BTreeMap::new(),
14822 winner: 'A',
14823 rankings: 1,
14824 unanimous_initial: true,
14825 deliberated: false,
14826 changed_votes: 0,
14827 unanimous_final: true,
14828 tie_break: None,
14829 judges: 0,
14830 present: 0,
14831 quorum: 0,
14832 met_quorum: true,
14833 uncontested: Some("only candidate A produced a change".to_owned()),
14834 });
14835 state.reviews = vec![ReviewRound {
14836 round: 1,
14837 head: "deadbeef".to_owned(),
14838 verified_head: None,
14839 verified_at: None,
14840 reviews: Vec::new(),
14841 e2e: Vec::new(),
14842 fix: None,
14843 blocking: 0,
14844 answered: 0,
14845 expected: 0,
14846 clean: true,
14847 verify_retried: false,
14848 e2e_deferred: false,
14849 e2e_defer_reason: None,
14850 progressed: false,
14851 vote_split: false,
14852 reconsideration: Vec::new(),
14853 verdict: None,
14854 }];
14855 state.gate = Vec::new();
14857 state.gate_ran = false;
14858 state.status = RunStatus::Gating;
14859
14860 let mut runner = Runner {
14861 state,
14862 roles: ResolvedRoles {
14863 implementers: Vec::new(),
14864 judges: Vec::new(),
14865 reviewers: Vec::new(),
14866 fixer: None,
14867 conductor: conductor(),
14868 implementer_roster: Vec::new(),
14869 judge_roster: Vec::new(),
14870 reviewer_roster: Vec::new(),
14871 },
14872 sem: Arc::new(Semaphore::new(1)),
14873 pause: Pause::new(),
14874 interrupt: Pause::new(),
14875 };
14876
14877 runner.merge().await.expect("merge");
14878
14879 assert!(
14880 runner.state.merge.is_none(),
14881 "an empty gate must never be read as a passing one: {:?}",
14882 runner.state.merge
14883 );
14884 }
14885
14886 #[tokio::test]
14893 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14894 let tmp = tempfile::tempdir().expect("tempdir");
14895 let repo = tmp.path().join("repo");
14896 std::fs::create_dir_all(&repo).unwrap();
14897 init_repo(&repo);
14898
14899 let config = Config::default();
14901
14902 let mut state = RunState::new(
14903 repo.clone(),
14904 "main".to_owned(),
14905 "deadbeef".to_owned(),
14906 "task".to_owned(),
14907 config,
14908 );
14909 state.candidates = vec![Candidate {
14910 index: 0,
14911 label: 'A',
14912 agent: "alpha".to_owned(),
14913 branch: "does-not-exist".to_owned(),
14914 worktree: repo.clone(),
14915 summary: String::new(),
14916 stat: String::new(),
14917 files: 0,
14918 commits: 0,
14919 empty: false,
14920 failed: None,
14921 verified_noop: None,
14922 duration_ms: 0,
14923 folded: false,
14924 }];
14925 state.tally = Some(Tally {
14926 first_choice: BTreeMap::from([('A', 1)]),
14927 borda: BTreeMap::new(),
14928 winner: 'A',
14929 rankings: 1,
14930 unanimous_initial: true,
14931 deliberated: false,
14932 changed_votes: 0,
14933 unanimous_final: true,
14934 tie_break: None,
14935 judges: 0,
14936 present: 0,
14937 quorum: 0,
14938 met_quorum: true,
14939 uncontested: Some("only candidate A produced a change".to_owned()),
14940 });
14941 state.reviews = vec![ReviewRound {
14942 round: 1,
14943 head: "deadbeef".to_owned(),
14944 verified_head: None,
14945 verified_at: None,
14946 reviews: Vec::new(),
14947 e2e: Vec::new(),
14948 fix: None,
14949 blocking: 0,
14950 answered: 0,
14951 expected: 0,
14952 clean: true,
14953 verify_retried: false,
14954 e2e_deferred: false,
14955 e2e_defer_reason: None,
14956 progressed: false,
14957 vote_split: false,
14958 reconsideration: Vec::new(),
14959 verdict: None,
14960 }];
14961
14962 let mut runner = Runner {
14963 state,
14964 roles: ResolvedRoles {
14965 implementers: Vec::new(),
14966 judges: Vec::new(),
14967 reviewers: Vec::new(),
14968 fixer: None,
14969 conductor: conductor(),
14970 implementer_roster: Vec::new(),
14971 judge_roster: Vec::new(),
14972 reviewer_roster: Vec::new(),
14973 },
14974 sem: Arc::new(Semaphore::new(1)),
14975 pause: Pause::new(),
14976 interrupt: Pause::new(),
14977 };
14978
14979 runner.gate().await.expect("gate");
14980 assert!(
14981 runner.state.gate_ran,
14982 "zero configured commands is still a real attempt, not an unrun gate"
14983 );
14984 assert!(runner.state.gate.is_empty());
14985 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
14986 assert_ne!(
14987 runner.state.status,
14988 RunStatus::Blocked,
14989 "a gate with nothing to check must not read as failed"
14990 );
14991
14992 runner.merge().await.expect("merge");
14993 assert_eq!(
14994 runner.state.status,
14995 RunStatus::Ready,
14996 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14997 );
14998 }
14999
15000 #[tokio::test]
15011 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15012 crate::run::pin_test_home();
15013 let home = crate::run::home();
15014
15015 let tmp = tempfile::tempdir().expect("tempdir");
15016 let repo = tmp.path().join("repo");
15017 std::fs::create_dir_all(&repo).unwrap();
15018 init_repo(&repo);
15019 let cache_dir = tmp.path().join("target");
15022
15023 let mut config = Config::default();
15024 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15025 config.graph.timeout_verify = Some(2);
15028 config.graph.timeout_resource_wait = 2;
15029
15030 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15031 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15032 .expect("no io error acquiring directly")
15033 {
15034 crate::cache::AcquireOutcome::Acquired(g) => g,
15035 crate::cache::AcquireOutcome::Busy(b) => {
15036 panic!("expected the direct acquire to win the lease first: {b:?}")
15037 }
15038 };
15039
15040 let mut state = RunState::new(
15041 repo.clone(),
15042 "main".to_owned(),
15043 "deadbeef".to_owned(),
15044 "task".to_owned(),
15045 config,
15046 );
15047 state.candidates = vec![Candidate {
15048 index: 0,
15049 label: 'A',
15050 agent: "alpha".to_owned(),
15051 branch: "does-not-exist".to_owned(),
15052 worktree: repo.clone(),
15053 summary: String::new(),
15054 stat: String::new(),
15055 files: 0,
15056 commits: 0,
15057 empty: false,
15058 failed: None,
15059 verified_noop: None,
15060 duration_ms: 0,
15061 folded: false,
15062 }];
15063 state.tally = Some(Tally {
15064 first_choice: BTreeMap::from([('A', 1)]),
15065 borda: BTreeMap::new(),
15066 winner: 'A',
15067 rankings: 1,
15068 unanimous_initial: true,
15069 deliberated: false,
15070 changed_votes: 0,
15071 unanimous_final: true,
15072 tie_break: None,
15073 judges: 0,
15074 present: 0,
15075 quorum: 0,
15076 met_quorum: true,
15077 uncontested: Some("only candidate A produced a change".to_owned()),
15078 });
15079 state.reviews = vec![ReviewRound {
15080 round: 1,
15081 head: "deadbeef".to_owned(),
15082 verified_head: None,
15083 verified_at: None,
15084 reviews: Vec::new(),
15085 e2e: Vec::new(),
15086 fix: None,
15087 blocking: 0,
15088 answered: 0,
15089 expected: 0,
15090 clean: true,
15091 verify_retried: false,
15092 e2e_deferred: false,
15093 e2e_defer_reason: None,
15094 progressed: false,
15095 vote_split: false,
15096 reconsideration: Vec::new(),
15097 verdict: None,
15098 }];
15099
15100 let mut runner = Runner {
15101 state,
15102 roles: ResolvedRoles {
15103 implementers: Vec::new(),
15104 judges: Vec::new(),
15105 reviewers: Vec::new(),
15106 fixer: None,
15107 conductor: conductor(),
15108 implementer_roster: Vec::new(),
15109 judge_roster: Vec::new(),
15110 reviewer_roster: Vec::new(),
15111 },
15112 sem: Arc::new(Semaphore::new(1)),
15113 pause: Pause::new(),
15114 interrupt: Pause::new(),
15115 };
15116
15117 let started = std::time::Instant::now();
15118 runner.gate().await.expect("gate");
15119 assert!(
15120 started.elapsed() < Duration::from_secs(1),
15121 "a gate with nothing to run must never wait on a lease it never needed"
15122 );
15123 assert!(
15124 runner.state.gate_ran,
15125 "zero commands is still a real, immediate attempt"
15126 );
15127 assert!(runner.state.gate.is_empty());
15128 assert_ne!(
15129 runner.state.status,
15130 RunStatus::Blocked,
15131 "must not read as resource-blocked on a lease it never asked for"
15132 );
15133 }
15134
15135 #[tokio::test]
15145 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15146 crate::run::pin_test_home();
15147
15148 let tmp = tempfile::tempdir().expect("tempdir");
15149 let repo = tmp.path().join("repo");
15150 std::fs::create_dir_all(&repo).unwrap();
15151 init_repo(&repo);
15152
15153 let mut config = Config::default();
15154 config.verify.gate = vec![
15155 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15156 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15157 .to_owned(),
15158 ];
15159
15160 let mut state = RunState::new(
15161 repo.clone(),
15162 "main".to_owned(),
15163 "deadbeef".to_owned(),
15164 "task".to_owned(),
15165 config,
15166 );
15167 let run_id = state.id.clone();
15168 state.candidates = vec![Candidate {
15169 index: 0,
15170 label: 'A',
15171 agent: "alpha".to_owned(),
15172 branch: "does-not-exist".to_owned(),
15173 worktree: repo.clone(),
15174 summary: String::new(),
15175 stat: String::new(),
15176 files: 0,
15177 commits: 0,
15178 empty: false,
15179 failed: None,
15180 verified_noop: None,
15181 duration_ms: 0,
15182 folded: false,
15183 }];
15184 state.tally = Some(Tally {
15185 first_choice: BTreeMap::from([('A', 1)]),
15186 borda: BTreeMap::new(),
15187 winner: 'A',
15188 rankings: 1,
15189 unanimous_initial: true,
15190 deliberated: false,
15191 changed_votes: 0,
15192 unanimous_final: true,
15193 tie_break: None,
15194 judges: 0,
15195 present: 0,
15196 quorum: 0,
15197 met_quorum: true,
15198 uncontested: Some("only candidate A produced a change".to_owned()),
15199 });
15200 state.reviews = vec![ReviewRound {
15201 round: 1,
15202 head: "deadbeef".to_owned(),
15203 verified_head: None,
15204 verified_at: None,
15205 reviews: Vec::new(),
15206 e2e: Vec::new(),
15207 fix: None,
15208 blocking: 0,
15209 answered: 0,
15210 expected: 0,
15211 clean: true,
15212 verify_retried: false,
15213 e2e_deferred: false,
15214 e2e_defer_reason: None,
15215 progressed: false,
15216 vote_split: false,
15217 reconsideration: Vec::new(),
15218 verdict: None,
15219 }];
15220
15221 let mut runner = Runner {
15222 state,
15223 roles: ResolvedRoles {
15224 implementers: Vec::new(),
15225 judges: Vec::new(),
15226 reviewers: Vec::new(),
15227 fixer: None,
15228 conductor: conductor(),
15229 implementer_roster: Vec::new(),
15230 judge_roster: Vec::new(),
15231 reviewer_roster: Vec::new(),
15232 },
15233 sem: Arc::new(Semaphore::new(1)),
15234 pause: Pause::new(),
15235 interrupt: Pause::new(),
15236 };
15237
15238 let started_marker = repo.join("started.marker");
15239 let release_marker = repo.join("release.marker");
15240 let poller = tokio::spawn(async move {
15241 for _ in 0..100 {
15246 if started_marker.exists()
15247 && let Ok(s) = crate::run::RunState::load(&run_id)
15248 && let Some(a) = s.active.get("gate")
15249 {
15250 std::fs::write(&release_marker, b"go").expect("release marker");
15251 return Some(a.clone());
15252 }
15253 tokio::time::sleep(Duration::from_millis(50)).await;
15254 }
15255 None
15256 });
15257
15258 runner.gate().await.expect("gate");
15259 let captured = poller.await.expect("poller task");
15260 let captured = captured.expect(
15261 "the poller never saw a `gate` task entry in run.json while the command was \
15262 still blocked on its own release marker",
15263 );
15264
15265 assert_eq!(captured.task.as_deref(), Some("gate"));
15266 assert_eq!(captured.node, "gate");
15267 assert_eq!(captured.index, Some(1));
15268 assert_eq!(captured.total, Some(1));
15269 assert!(
15270 captured
15271 .command
15272 .as_deref()
15273 .is_some_and(|c| c.contains("started.marker")),
15274 "{captured:?}"
15275 );
15276
15277 assert!(
15278 runner.state.active.is_empty(),
15279 "the entry must be cleared once the command actually finished: {:?}",
15280 runner.state.active
15281 );
15282 assert!(runner.state.gate_ran);
15283 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15284 }
15285
15286 #[tokio::test]
15289 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15290 use crate::verdict::{Finding, ReviewVote, Severity};
15291 crate::run::pin_test_home();
15292 let tmp = tempfile::tempdir().expect("tempdir");
15293 let repo = tmp.path().join("repo");
15294 std::fs::create_dir_all(&repo).unwrap();
15295 init_repo(&repo);
15296
15297 for (severity, vote, expect) in [
15298 (Severity::Major, ReviewVote::Reject, true),
15299 (Severity::Minor, ReviewVote::Reject, false),
15300 (Severity::Major, ReviewVote::Approve, false),
15301 ] {
15302 let mut round = review_round(false, 1, 1, 1, false, true);
15303 round.reviews = vec![ReviewRecord {
15304 reviewer: 1,
15305 agent: "alpha".to_owned(),
15306 summary: String::new(),
15307 findings: vec![Finding {
15308 id: "R1-1-1".to_owned(),
15309 severity,
15310 file: None,
15311 line: None,
15312 title: "t".to_owned(),
15313 detail: String::new(),
15314 }],
15315 vote: Some(vote),
15316 failed: None,
15317 duration_ms: 0,
15318 attempts: 0,
15319 }];
15320 let mut state = RunState::new(
15321 repo.clone(),
15322 "main".to_owned(),
15323 "deadbeef".to_owned(),
15324 "task".to_owned(),
15325 Config::default(),
15326 );
15327 state.reviews = vec![round];
15328 let mut runner = Runner {
15329 state,
15330 roles: ResolvedRoles {
15331 implementers: Vec::new(),
15332 judges: Vec::new(),
15333 reviewers: Vec::new(),
15334 fixer: None,
15335 conductor: conductor(),
15336 implementer_roster: Vec::new(),
15337 judge_roster: Vec::new(),
15338 reviewer_roster: Vec::new(),
15339 },
15340 sem: Arc::new(Semaphore::new(1)),
15341 pause: Pause::new(),
15342 interrupt: Pause::new(),
15343 };
15344 let shell = runner.state.config.shell();
15345 runner
15346 .stop_reviewing("round budget spent", &shell, &repo)
15347 .await
15348 .expect("stop_reviewing");
15349 assert_eq!(runner.state.status, RunStatus::Gating);
15350 assert_eq!(
15351 runner.state.contested_handoff.is_some(),
15352 expect,
15353 "{severity:?} + {vote:?}"
15354 );
15355 }
15356 }
15357
15358 #[tokio::test]
15371 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15372 crate::run::pin_test_home();
15373 let home = crate::run::home();
15374
15375 let tmp = tempfile::tempdir().expect("tempdir");
15376 let repo = tmp.path().join("repo");
15377 std::fs::create_dir_all(&repo).unwrap();
15378 init_repo(&repo);
15379 let head = crate::git::rev_parse(&repo, "HEAD")
15380 .await
15381 .expect("rev-parse");
15382 let cache_dir = tmp.path().join("target");
15385
15386 let mut config = Config::default();
15387 config.verify.e2e = vec![format!(
15388 "CARGO_TARGET_DIR='{}' test -f README.md",
15389 cache_dir.display()
15390 )];
15391 config.graph.review_rounds = 1;
15392 config.graph.timeout_verify = Some(2);
15395 config.graph.timeout_resource_wait = 2;
15396
15397 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15398 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15399 .expect("no io error acquiring directly")
15400 {
15401 crate::cache::AcquireOutcome::Acquired(g) => g,
15402 crate::cache::AcquireOutcome::Busy(b) => {
15403 panic!("expected the direct acquire to win the lease first: {b:?}")
15404 }
15405 };
15406
15407 let mut state = RunState::new(
15408 repo.clone(),
15409 "main".to_owned(),
15410 head.clone(),
15411 "task".to_owned(),
15412 config,
15413 );
15414 state.candidates = vec![Candidate {
15415 index: 0,
15416 label: 'A',
15417 agent: "alpha".to_owned(),
15418 branch: "does-not-exist".to_owned(),
15419 worktree: repo.clone(),
15420 summary: String::new(),
15421 stat: String::new(),
15422 files: 0,
15423 commits: 0,
15424 empty: false,
15425 failed: None,
15426 verified_noop: None,
15427 duration_ms: 0,
15428 folded: false,
15429 }];
15430 state.tally = Some(Tally {
15431 first_choice: BTreeMap::from([('A', 1)]),
15432 borda: BTreeMap::new(),
15433 winner: 'A',
15434 rankings: 1,
15435 unanimous_initial: true,
15436 deliberated: false,
15437 changed_votes: 0,
15438 unanimous_final: true,
15439 tie_break: None,
15440 judges: 0,
15441 present: 0,
15442 quorum: 0,
15443 met_quorum: true,
15444 uncontested: Some("only candidate A produced a change".to_owned()),
15445 });
15446 state.reviews = vec![ReviewRound {
15450 round: 1,
15451 head: head.clone(),
15452 verified_head: None,
15453 verified_at: None,
15454 reviews: Vec::new(),
15455 e2e: Vec::new(),
15456 fix: None,
15457 blocking: 1,
15458 answered: 1,
15459 expected: 1,
15460 clean: false,
15461 verify_retried: false,
15462 e2e_deferred: true,
15463 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15464 progressed: false,
15465 vote_split: false,
15466 reconsideration: Vec::new(),
15467 verdict: None,
15468 }];
15469
15470 let mut runner = Runner {
15471 state,
15472 roles: ResolvedRoles {
15473 implementers: Vec::new(),
15474 judges: Vec::new(),
15475 reviewers: Vec::new(),
15476 fixer: None,
15477 conductor: conductor(),
15478 implementer_roster: Vec::new(),
15479 judge_roster: Vec::new(),
15480 reviewer_roster: Vec::new(),
15481 },
15482 sem: Arc::new(Semaphore::new(1)),
15483 pause: Pause::new(),
15484 interrupt: Pause::new(),
15485 };
15486
15487 let shell = runner.state.config.shell();
15488 runner
15489 .stop_reviewing("round budget spent", &shell, &repo)
15490 .await
15491 .expect("stop_reviewing");
15492
15493 let last = runner.state.reviews.last().expect("round record");
15494 assert_eq!(
15495 last.e2e_status(),
15496 E2eStatus::ResourceBlocked,
15497 "the shared cache is still held; the attempt must read as blocked, not deferred or \
15498 failed: {last:?}"
15499 );
15500 assert_eq!(
15501 last.verified_head.as_deref(),
15502 Some(head.as_str()),
15503 "which commit this attempt targeted is known even though nothing finished checking \
15504 it"
15505 );
15506 let first_attempt_at = last
15507 .verified_at
15508 .expect("when this attempt ran is known too");
15509 assert_ne!(
15510 runner.state.status,
15511 RunStatus::Blocked,
15512 "contention is evidence about the machine, not the patch — it must not settle the \
15513 run as blocked: {:?}",
15514 runner.state.status
15515 );
15516 assert!(
15517 !runner
15518 .state
15519 .events
15520 .iter()
15521 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15522 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15523 runner.state.events
15524 );
15525
15526 runner
15531 .stop_reviewing("round budget spent", &shell, &repo)
15532 .await
15533 .expect("stop_reviewing retry");
15534 assert_eq!(
15535 runner.state.reviews.len(),
15536 1,
15537 "no new round was started: {:?}",
15538 runner.state.reviews
15539 );
15540 let last = runner.state.reviews.last().expect("round record");
15541 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15542 assert!(
15543 last.verified_at.expect("still known") > first_attempt_at,
15544 "a second reentry must be a fresh attempt, not a stale copy of the first"
15545 );
15546 assert_ne!(runner.state.status, RunStatus::Blocked);
15547
15548 held.release();
15549 }
15550
15551 #[tokio::test]
15563 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15564 crate::run::pin_test_home();
15565 let home = crate::run::home();
15566
15567 let tmp = tempfile::tempdir().expect("tempdir");
15568 let repo = tmp.path().join("repo");
15569 std::fs::create_dir_all(&repo).unwrap();
15570 init_repo(&repo);
15571 let head = crate::git::rev_parse(&repo, "HEAD")
15572 .await
15573 .expect("rev-parse");
15574 let cache_dir = tmp.path().join("target");
15575
15576 let mut config = Config::default();
15577 config.verify.e2e = vec![format!(
15578 "CARGO_TARGET_DIR='{}' test -f README.md",
15579 cache_dir.display()
15580 )];
15581 config.graph.review_rounds = 1;
15582 config.graph.timeout_verify = Some(2);
15583 config.graph.timeout_resource_wait = 2;
15584
15585 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15586 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15587 .expect("no io error acquiring directly")
15588 {
15589 crate::cache::AcquireOutcome::Acquired(g) => g,
15590 crate::cache::AcquireOutcome::Busy(b) => {
15591 panic!("expected the direct acquire to win the lease first: {b:?}")
15592 }
15593 };
15594
15595 let mut state = RunState::new(
15596 repo.clone(),
15597 "main".to_owned(),
15598 head.clone(),
15599 "task".to_owned(),
15600 config,
15601 );
15602 state.candidates = vec![Candidate {
15603 index: 0,
15604 label: 'A',
15605 agent: "alpha".to_owned(),
15606 branch: "does-not-exist".to_owned(),
15607 worktree: repo.clone(),
15608 summary: String::new(),
15609 stat: String::new(),
15610 files: 0,
15611 commits: 0,
15612 empty: false,
15613 failed: None,
15614 verified_noop: None,
15615 duration_ms: 0,
15616 folded: false,
15617 }];
15618 state.tally = Some(Tally {
15619 first_choice: BTreeMap::from([('A', 1)]),
15620 borda: BTreeMap::new(),
15621 winner: 'A',
15622 rankings: 1,
15623 unanimous_initial: true,
15624 deliberated: false,
15625 changed_votes: 0,
15626 unanimous_final: true,
15627 tie_break: None,
15628 judges: 0,
15629 present: 0,
15630 quorum: 0,
15631 met_quorum: true,
15632 uncontested: Some("only candidate A produced a change".to_owned()),
15633 });
15634 state.reviews = vec![ReviewRound {
15638 round: 1,
15639 head: head.clone(),
15640 verified_head: Some(head.clone()),
15641 verified_at: Some(jiff::Timestamp::now()),
15642 reviews: Vec::new(),
15643 e2e: vec![CommandOutcome {
15644 command: format!(
15645 "CARGO_TARGET_DIR='{}' test -f README.md",
15646 cache_dir.display()
15647 ),
15648 code: None,
15649 output_tail: "waiting for the shared build cache".to_owned(),
15650 duration_ms: 0,
15651 resource_blocked: true,
15652 }],
15653 fix: None,
15654 blocking: 1,
15655 answered: 1,
15656 expected: 1,
15657 clean: false,
15658 verify_retried: false,
15659 e2e_deferred: false,
15660 e2e_defer_reason: None,
15661 progressed: false,
15662 vote_split: false,
15663 reconsideration: Vec::new(),
15664 verdict: None,
15665 }];
15666
15667 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15668 let mut runner = Runner {
15669 state,
15670 roles: ResolvedRoles {
15671 implementers: Vec::new(),
15672 judges: Vec::new(),
15673 reviewers: Vec::new(),
15674 fixer: None,
15675 conductor: conductor(),
15676 implementer_roster: Vec::new(),
15677 judge_roster: Vec::new(),
15678 reviewer_roster: Vec::new(),
15679 },
15680 sem: Arc::new(Semaphore::new(1)),
15681 pause: Pause::new(),
15682 interrupt: Pause::new(),
15683 };
15684
15685 runner.review_loop().await.expect("review_loop");
15690
15691 assert_eq!(
15692 runner.state.reviews.len(),
15693 1,
15694 "no new round was started on top of the unresolved one: {:?}",
15695 runner.state.reviews
15696 );
15697 let last = &runner.state.reviews[0];
15698 assert_eq!(
15699 last.e2e_status(),
15700 E2eStatus::ResourceBlocked,
15701 "still contended: {last:?}"
15702 );
15703 assert!(
15704 last.verified_at.expect("still known") > first_attempt_at,
15705 "review_loop must have actually retried the check, not left it exactly as found"
15706 );
15707 assert_ne!(
15708 runner.state.status,
15709 RunStatus::Blocked,
15710 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15711 runner.state.status
15712 );
15713
15714 held.release();
15715 }
15716
15717 #[tokio::test]
15718 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15719 crate::run::pin_test_home();
15720 let tmp = tempfile::tempdir().expect("tempdir");
15721 let repo = tmp.path().join("repo");
15722 std::fs::create_dir_all(&repo).unwrap();
15723 init_repo(&repo);
15724
15725 let mut config = Config::default();
15726 config.merge.mode = MergeMode::Pr;
15727 config.graph.land = true;
15728 config.graph.land_approval = false;
15729
15730 let mut state = RunState::new(
15731 repo.clone(),
15732 "main".to_owned(),
15733 "deadbeef".to_owned(),
15734 "task".to_owned(),
15735 config,
15736 );
15737 state.candidates = vec![Candidate {
15738 index: 0,
15739 label: 'A',
15740 agent: "alpha".to_owned(),
15741 branch: "does-not-exist".to_owned(),
15742 worktree: repo.clone(),
15743 summary: String::new(),
15744 stat: String::new(),
15745 files: 0,
15746 commits: 0,
15747 empty: false,
15748 failed: None,
15749 verified_noop: None,
15750 duration_ms: 0,
15751 folded: false,
15752 }];
15753 state.tally = Some(Tally {
15754 first_choice: BTreeMap::from([('A', 1)]),
15755 borda: BTreeMap::new(),
15756 winner: 'A',
15757 rankings: 1,
15758 unanimous_initial: true,
15759 deliberated: false,
15760 changed_votes: 0,
15761 unanimous_final: true,
15762 tie_break: None,
15763 judges: 0,
15764 present: 0,
15765 quorum: 0,
15766 met_quorum: true,
15767 uncontested: Some("only candidate A produced a change".to_owned()),
15768 });
15769 state.reviews = vec![ReviewRound {
15770 round: 1,
15771 head: "deadbeef".to_owned(),
15772 verified_head: None,
15773 verified_at: None,
15774 reviews: Vec::new(),
15775 e2e: Vec::new(),
15776 fix: None,
15777 blocking: 0,
15778 answered: 0,
15779 expected: 0,
15780 clean: true,
15781 verify_retried: false,
15782 e2e_deferred: false,
15783 e2e_defer_reason: None,
15784 progressed: false,
15785 vote_split: false,
15786 reconsideration: Vec::new(),
15787 verdict: None,
15788 }];
15789 state.gate = vec![CommandOutcome {
15790 command: "test".to_owned(),
15791 code: Some(0),
15792 output_tail: String::new(),
15793 duration_ms: 0,
15794 resource_blocked: false,
15795 }];
15796 state.gate_ran = true;
15797 state.status = RunStatus::Landing;
15801 state.merge = Some(MergeOutcome {
15802 mode: MergeMode::Pr,
15803 ok: true,
15804 detail: "https://example.invalid/x/y/pull/1".to_owned(),
15805 empty: false,
15806 });
15807
15808 ask_test_home();
15812 state.save().expect("save landing run");
15815 let store = ask::Questions::open();
15816 let q = ask_open_question(&store, &state.id);
15817
15818 let mut runner = Runner {
15819 state,
15820 roles: ResolvedRoles {
15821 implementers: Vec::new(),
15822 judges: Vec::new(),
15823 reviewers: Vec::new(),
15824 fixer: None,
15825 conductor: conductor(),
15826 implementer_roster: Vec::new(),
15827 judge_roster: Vec::new(),
15828 reviewer_roster: Vec::new(),
15829 },
15830 sem: Arc::new(Semaphore::new(1)),
15831 pause: Pause::new(),
15832 interrupt: Pause::new(),
15833 };
15834
15835 runner.execute().await.expect("execute");
15840
15841 assert_eq!(
15842 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15843 Some("https://example.invalid/x/y/pull/1"),
15844 "reentry must not push again or open a second pull request over the \
15845 one `land` is already watching"
15846 );
15847 assert_ne!(
15848 runner.state.status,
15849 RunStatus::Landing,
15850 "land could not actually reach the fake pull request, so it must \
15851 have given up rather than left the run silently parked forever"
15852 );
15853 assert_eq!(runner.state.status, RunStatus::Blocked);
15857 assert!(
15858 store.get(&q.id).unwrap().status.open(),
15859 "Blocked is still alive; settle_questions must have been a no-op here"
15860 );
15861 }
15862
15863 fn state_with_round(round: ReviewRound) -> RunState {
15864 let mut s = RunState::new(
15865 PathBuf::from("/repo"),
15866 "main".to_owned(),
15867 "abc1234".to_owned(),
15868 "add retries".to_owned(),
15869 Config::default(),
15870 );
15871 s.reviews = vec![round];
15872 s
15873 }
15874
15875 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15876 crate::verdict::Finding {
15877 id: id.to_owned(),
15878 severity,
15879 file: None,
15880 line: None,
15881 title: title.to_owned(),
15882 detail: String::new(),
15883 }
15884 }
15885
15886 #[test]
15887 fn pr_body_names_open_findings_and_declined_ones() {
15888 let round = ReviewRound {
15889 round: 2,
15890 head: "deadbee".to_owned(),
15891 verified_head: None,
15892 verified_at: None,
15893 reviews: vec![ReviewRecord {
15894 attempts: 0,
15895 reviewer: 1,
15896 agent: "alpha".to_owned(),
15897 summary: String::new(),
15898 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15899 vote: None,
15900 failed: None,
15901 duration_ms: 0,
15902 }],
15903 e2e: vec![CommandOutcome {
15904 command: "cargo test".to_owned(),
15905 code: Some(0),
15906 output_tail: String::new(),
15907 duration_ms: 0,
15908 resource_blocked: false,
15909 }],
15910 verify_retried: false,
15911 e2e_deferred: false,
15912 e2e_defer_reason: None,
15913 fix: Some(FixRecord {
15914 agent: "alpha".to_owned(),
15915 addressed: Vec::new(),
15916 rejected: vec![crate::verdict::Rejection {
15917 id: "R1-1-1".to_owned(),
15918 why: "not reachable from any caller".to_owned(),
15919 }],
15920 notes: String::new(),
15921 committed: true,
15922 failed: None,
15923 duration_ms: 0,
15924 continuation: None,
15925 }),
15926 blocking: 0,
15927 answered: 1,
15928 expected: 1,
15929 clean: false,
15930 progressed: true,
15931 vote_split: false,
15932 reconsideration: Vec::new(),
15933 verdict: None,
15934 };
15935 let state = state_with_round(round);
15936 let body = pr_message(&state, 'A').body;
15937
15938 assert!(body.contains("add retries"), "the task must still be there");
15939 assert!(body.contains("R2-1-1"), "{body}");
15940 assert!(!body.contains("unused import"), "{body}");
15941 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15942 assert!(
15943 !body.contains("not reachable from any caller"),
15944 "the reason it was declined: {body}"
15945 );
15946 }
15947
15948 #[test]
15949 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15950 let round = ReviewRound {
15951 round: 1,
15952 head: "deadbee".to_owned(),
15953 verified_head: None,
15954 verified_at: None,
15955 reviews: vec![ReviewRecord {
15956 attempts: 0,
15957 reviewer: 1,
15958 agent: "alpha".to_owned(),
15959 summary: String::new(),
15960 findings: Vec::new(),
15961 vote: None,
15962 failed: None,
15963 duration_ms: 0,
15964 }],
15965 e2e: Vec::new(),
15966 verify_retried: false,
15967 e2e_deferred: false,
15968 e2e_defer_reason: None,
15969 fix: None,
15970 blocking: 0,
15971 answered: 1,
15972 expected: 1,
15973 clean: true,
15974 progressed: false,
15975 vote_split: false,
15976 reconsideration: Vec::new(),
15977 verdict: None,
15978 };
15979 let state = state_with_round(round);
15980 let body = pr_message(&state, 'A').body;
15981 assert!(!body.contains("Open review findings"), "{body}");
15982 assert!(!body.contains("Declined"), "{body}");
15983 }
15984
15985 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15986 let mut state = RunState::new(
15987 PathBuf::from("/repo"),
15988 "main".to_owned(),
15989 "abc1234".to_owned(),
15990 instruction.to_owned(),
15991 Config::default(),
15992 );
15993 state.candidates.push(Candidate {
15994 index: 0,
15995 label: 'A',
15996 agent: "alpha".to_owned(),
15997 branch: "magi/x/A".to_owned(),
15998 worktree: PathBuf::from("/wt"),
15999 summary: summary.to_owned(),
16000 stat: String::new(),
16001 files: 1,
16002 commits: 1,
16003 empty: false,
16004 failed: None,
16005 verified_noop: None,
16006 folded: false,
16007 duration_ms: 0,
16008 });
16009 state
16010 }
16011
16012 fn review_state(subjects: &[&str]) -> RunState {
16013 let mut state = state_with_summary(
16014 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16015 "",
16016 );
16017 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16018 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16019 state
16020 }
16021
16022 #[tokio::test]
16025 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16026 ask_test_home();
16027 let tmp = tempfile::tempdir().unwrap();
16028 let repo = tmp.path().join("repo");
16029 std::fs::create_dir_all(&repo).unwrap();
16030 init_repo(&repo);
16031 let origin = tmp.path().join("origin.git");
16032 let g = |dir: &Path, args: &[&str]| {
16033 let out = std::process::Command::new("git")
16034 .args(args)
16035 .current_dir(dir)
16036 .quiet()
16037 .output()
16038 .expect("spawn git");
16039 assert!(
16040 out.status.success(),
16041 "git {args:?}: {}",
16042 String::from_utf8_lossy(&out.stderr)
16043 );
16044 };
16045 g(
16046 tmp.path(),
16047 &[
16048 "clone",
16049 "--bare",
16050 "-q",
16051 repo.to_str().unwrap(),
16052 origin.to_str().unwrap(),
16053 ],
16054 );
16055 g(
16056 &repo,
16057 &["remote", "add", "origin", origin.to_str().unwrap()],
16058 );
16059 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16060
16061 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16063 g(&repo, &["add", "-A"]);
16064 g(
16065 &repo,
16066 &["commit", "-q", "-m", "chore(deps): update a crate"],
16067 );
16068 g(&repo, &["push", "-q", "origin", "main"]);
16069 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16070 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16071 g(&repo, &["add", "-A"]);
16072 g(
16073 &repo,
16074 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16075 );
16076 g(&repo, &["checkout", "-q", "main"]);
16077
16078 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16079 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16080 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16082 assert_eq!(fallback, c1);
16083
16084 let mut state = review_state(&[
16085 "chore(deps): update a crate",
16086 "fix(daemon): apply a chosen action",
16087 ]);
16088 state.repo = repo.clone();
16089 state.base_branch = "main".to_owned();
16090 state.base_commit = c1;
16091 refresh_reviewed_commits(&mut state, "feat/own").await;
16092 assert_eq!(
16093 state.reviewed_commits,
16094 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16095 );
16096 assert_eq!(
16097 review_title(&state).as_deref(),
16098 Some("fix(daemon): apply a chosen action")
16099 );
16100 assert_eq!(
16101 leaked_subjects(&state, "feat/own").await,
16102 Some(vec!["chore(deps): update a crate".to_owned()])
16103 );
16104 state.config.merge.remote = "nowhere".to_owned();
16107 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16108 }
16109
16110 struct SummaryFixture {
16114 _tmp: tempfile::TempDir,
16115 repo: PathBuf,
16116 reply: PathBuf,
16117 runner: Runner,
16118 }
16119
16120 fn summary_fixture() -> SummaryFixture {
16121 ask_test_home();
16122 let tmp = tempfile::tempdir().unwrap();
16123 let repo = tmp.path().join("repo");
16124 std::fs::create_dir_all(&repo).unwrap();
16125 init_repo(&repo);
16126 let origin = tmp.path().join("origin.git");
16127 let g = |args: &[&str]| {
16128 let out = std::process::Command::new("git")
16129 .args(args)
16130 .current_dir(&repo)
16131 .quiet()
16132 .output()
16133 .expect("spawn git");
16134 assert!(
16135 out.status.success(),
16136 "git {args:?}: {}",
16137 String::from_utf8_lossy(&out.stderr)
16138 );
16139 };
16140 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16141 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16142 g(&["push", "-q", "origin", "main"]);
16143 g(&["checkout", "-q", "-b", "feat/own"]);
16144 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16145 g(&["add", "-A"]);
16146 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16147 let reply = tmp.path().join("reply.txt");
16148 std::fs::write(
16149 &reply,
16150 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16151 )
16152 .unwrap();
16153 let mut runner = runner_at(RunStatus::Gating);
16154 let mut state = review_state(&["feat(own): add the own file"]);
16155 state.repo = repo.clone();
16156 state.base_branch = "main".to_owned();
16157 state.candidates[0].branch = "feat/own".to_owned();
16158 state.candidates[0].worktree = repo.clone();
16159 let mut author = spec("alpha");
16160 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16161 author
16162 .env
16163 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16164 state.config.agents = vec![author];
16165 runner.state = state;
16166 SummaryFixture {
16167 _tmp: tmp,
16168 repo,
16169 reply,
16170 runner,
16171 }
16172 }
16173
16174 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16175 let winner = f.runner.state.candidates[0].clone();
16176 f.runner
16177 .guarded_pr_message(&winner, None, true)
16178 .await
16179 .unwrap()
16180 .expect("an English description never parks")
16181 }
16182
16183 fn clean_round_with_minor_findings() -> ReviewRound {
16184 let mut round = passing_round(2, None);
16185 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16186 round.reviews = vec![ReviewRecord {
16187 attempts: 0,
16188 reviewer: 1,
16189 agent: "alpha".into(),
16190 summary: "内部レビュー".into(),
16191 findings: vec![
16192 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16193 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16194 ],
16195 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16196 failed: None,
16197 duration_ms: 0,
16198 }];
16199 round
16200 }
16201
16202 #[tokio::test]
16203 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16204 for review_only in [false, true] {
16205 let mut f = summary_fixture();
16206 if !review_only {
16207 f.runner.state.candidates[0].agent = "alpha".into();
16208 }
16209 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16210 assert!(f.runner.state.reviews[0].clean);
16211 assert!(f.runner.state.reviews[0].fix.is_none());
16212 assert!(f.runner.state.open_findings().is_empty());
16213 assert_eq!(
16214 crate::pr_outcome::required_finding_ids(&f.runner.state),
16215 ["R2-1-1", "R2-1-2"]
16216 );
16217 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16218 let evidence = crate::pr_outcome::summary_evidence(
16219 &f.runner.state,
16220 crate::pr_outcome::FinalHead::At(&head),
16221 );
16222 assert!(
16223 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16224 );
16225 let omitted = posted(&mut f).await;
16227 assert!(omitted.body.contains("No description"));
16228 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16229 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16230 assert!(f.runner.state.change_summary.is_none());
16231 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16232 let described = posted(&mut f).await;
16233 assert!(described.body.contains("improve documentation for callers"));
16234 assert!(described.body.contains("remaining edge case"));
16235 assert!(described.body.contains("## Open review findings"));
16236 assert!(!described.body.contains("未修正"));
16237 }
16238 }
16239
16240 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16241 let mut f = summary_fixture();
16242 if !review_only {
16243 f.runner.state.candidates[0].agent = "alpha".into();
16244 }
16245 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16246 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16247 fixed
16248 .fix
16249 .as_mut()
16250 .unwrap()
16251 .rejected
16252 .push(crate::verdict::Rejection {
16253 id: "R1-1-1".into(),
16254 why: "呼び出されないコードのため辞退".into(),
16255 });
16256 let mut clean = clean_round_with_minor_findings();
16257 clean.verified_head = Some(head.clone());
16258 f.runner.state.reviews = vec![fixed, clean];
16259 ok_gate(&mut f.runner.state, Some(&head));
16260 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16261 f.runner.state.config.agents.push(scripted("beta", reply));
16262 f.runner.state.config.roles.language_judge =
16263 Some(crate::config::AgentChoice::One("beta".into()));
16264 f
16265 }
16266
16267 #[tokio::test]
16268 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16269 for review_only in [false, true] {
16270 let mut f = rewrite_fixture_with_findings(review_only,
16271 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16272 let m = posted(&mut f).await;
16273 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16274 let expected =
16275 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16276 assert!(m.body.contains(&expected));
16277 assert!(m.body.contains("## Review rounds"));
16278 assert!(
16279 m.body
16280 .contains("Gate (`verify.gate`): passed on the final commit")
16281 );
16282 assert!(
16283 m.body
16284 .contains("Tests (`verify.e2e`): passed on the final commit")
16285 );
16286 let c = f.runner.state.change_summary.as_ref().unwrap();
16287 assert!(
16288 m.body
16289 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16290 );
16291 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16292 assert!(crate::pr_outcome::covers_required_findings(
16293 &f.runner.state,
16294 &m.body
16295 ));
16296 assert!(m.body.contains("documentation remains incomplete"));
16297 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16298 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16299 assert!(f.runner.state.github_text.is_none());
16300 }
16301 }
16302
16303 #[tokio::test]
16304 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16305 for review_only in [false, true] {
16306 for reply in [
16307 "TITLE: fix: public description\nA short English description.",
16308 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16309 ] {
16310 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16311 let winner = f.runner.state.candidates[0].clone();
16312 let posted = f
16313 .runner
16314 .guarded_pr_message(&winner, None, true)
16315 .await
16316 .unwrap();
16317 assert!(posted.is_none());
16318 assert!(f.runner.state.parked);
16319 let withheld = f.runner.state.github_text.as_ref().unwrap();
16320 assert!(
16321 withheld.rewrite.is_none(),
16322 "an incomplete rewrite must not be selectable"
16323 );
16324 let q = ask::Questions::open()
16325 .get(withheld.question.as_ref().unwrap())
16326 .unwrap();
16327 assert!(
16328 !q.choices
16329 .contains(&crate::github_text::USE_REWRITE.to_owned())
16330 );
16331 assert!(
16332 q.detail.contains("R1-1-1")
16333 && q.detail.contains("R2-1-1")
16334 && q.detail.contains("R2-1-2")
16335 );
16336 }
16337 }
16338 }
16339
16340 #[tokio::test]
16341 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16342 for review_only in [false, true] {
16343 let mut f = summary_fixture();
16344 if !review_only {
16345 f.runner.state.candidates[0].agent = "alpha".into();
16346 }
16347 f.runner.state.instruction = "内部タスクの説明".into();
16348 f.runner.state.candidates[0].summary = "内部の候補報告".into();
16349 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16350 f.runner.state.reviews = vec![
16351 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16352 round_with_notes(2, Some("二回目の内部修正報告です。")),
16353 passing_round(3, Some(&head)),
16354 ];
16355 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16356 ok_gate(&mut f.runner.state, Some(&head));
16357 let captured = f.repo.join("public-prompt.txt");
16358 let author = &mut f.runner.state.config.agents[0];
16359 author.command = vec![
16360 "sh".into(),
16361 "-c".into(),
16362 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16363 ];
16364 author.env.insert(
16365 "PROMPT_FILE".into(),
16366 captured.to_string_lossy().into_owned(),
16367 );
16368 let m = posted(&mut f).await;
16369 assert!(m.body.contains("Adds `own.txt`"));
16370 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16371 assert!(m.body.contains("R1-1-1"));
16372 assert!(
16373 m.body
16374 .contains("Gate (`verify.gate`): passed on the final commit")
16375 );
16376 assert!(
16377 m.body
16378 .contains("Tests (`verify.e2e`): passed on the final commit")
16379 );
16380 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16381 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16382 let prompt = std::fs::read_to_string(captured).unwrap();
16383 assert!(prompt.contains(&format!(
16384 "git diff {} {head}",
16385 f.runner.state.change_summary.as_ref().unwrap().base
16386 )));
16387 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16388 assert_eq!(
16389 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16390 "二回目の内部修正報告です。"
16391 );
16392 }
16393 }
16394
16395 #[tokio::test]
16396 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16397 for review_only in [false, true] {
16398 let mut f = summary_fixture();
16399 if !review_only {
16400 f.runner.state.candidates[0].agent = "alpha".into();
16401 }
16402 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16403 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16404 std::fs::write(&f.reply, "no usable description").unwrap();
16405 let failed = posted(&mut f).await;
16406 assert!(
16407 failed
16408 .body
16409 .contains("No description of what this change does could be generated")
16410 );
16411 assert!(failed.body.is_ascii());
16412 assert!(f.runner.state.github_text.is_none());
16413 assert!(f.runner.state.change_summary.is_none());
16414 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16415 let evidence = crate::pr_outcome::summary_evidence(
16416 &f.runner.state,
16417 crate::pr_outcome::FinalHead::At(&head),
16418 );
16419 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16420 assert!(
16421 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16422 .exists()
16423 );
16424 f.runner.state =
16425 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16426 std::fs::write(
16427 &f.reply,
16428 "TITLE: feat(own): add the own file\nAdds the committed own file.",
16429 )
16430 .unwrap();
16431 let retried = posted(&mut f).await;
16432 assert!(retried.body.contains("Adds the committed own file."));
16433 assert_eq!(
16434 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16435 "日本語の修正報告"
16436 );
16437 }
16438 }
16439
16440 #[tokio::test]
16441 async fn same_head_verification_and_findings_invalidate_public_prose() {
16442 let mut f = summary_fixture();
16443 posted(&mut f).await;
16444 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16445 let original = f
16446 .runner
16447 .state
16448 .change_summary
16449 .as_ref()
16450 .unwrap()
16451 .evidence
16452 .clone();
16453 ok_gate(&mut f.runner.state, Some(&head));
16454 std::fs::write(
16455 &f.reply,
16456 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16457 )
16458 .unwrap();
16459 let verified = posted(&mut f).await;
16460 assert!(verified.body.contains("now verified"));
16461 assert_ne!(
16462 f.runner.state.change_summary.as_ref().unwrap().evidence,
16463 original
16464 );
16465 let mut round = round_with_notes(1, Some("内部報告"));
16466 round.reviews.push(ReviewRecord {
16467 attempts: 0,
16468 reviewer: 1,
16469 agent: "alpha".into(),
16470 summary: "内部レビュー".into(),
16471 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16472 vote: None,
16473 failed: None,
16474 duration_ms: 0,
16475 });
16476 round
16477 .fix
16478 .as_mut()
16479 .unwrap()
16480 .rejected
16481 .push(crate::verdict::Rejection {
16482 id: "R1-1-1".into(),
16483 why: "呼び出し元から到達できない".into(),
16484 });
16485 f.runner.state.reviews = vec![round];
16486 let omitted = posted(&mut f).await;
16488 assert!(omitted.body.contains("No description"));
16489 assert!(omitted.body.contains("R1-1-1"));
16490 assert!(!omitted.body.contains("到達"));
16491 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16492 let described = posted(&mut f).await;
16493 assert!(described.body.contains("unreachable from callers"));
16494 assert!(described.body.contains("R1-1-1"));
16495 assert!(!described.body.contains("到達"));
16496 let evidence = f
16497 .runner
16498 .state
16499 .change_summary
16500 .as_ref()
16501 .unwrap()
16502 .evidence
16503 .clone();
16504 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16505 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16506 assert!(posted(&mut f).await.body.contains("rationale updated"));
16507 assert_ne!(
16508 f.runner.state.change_summary.as_ref().unwrap().evidence,
16509 evidence
16510 );
16511 f.runner
16513 .state
16514 .change_summary
16515 .as_mut()
16516 .unwrap()
16517 .evidence
16518 .clear();
16519 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16520 assert!(posted(&mut f).await.body.contains("current specification"));
16521 }
16522
16523 #[tokio::test]
16524 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16525 let mut f = summary_fixture();
16526 let m = posted(&mut f).await;
16527 assert_eq!(m.title, "feat(own): add the own file");
16528 assert!(
16529 m.body
16530 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16531 "{}",
16532 m.body
16533 );
16534 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16535 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16536 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16537 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16538 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16539 let kept = f.runner.state.change_summary.as_ref().unwrap();
16540 assert_eq!(
16541 (kept.head.as_str(), kept.base.as_str()),
16542 (head.as_str(), base.as_str())
16543 );
16544 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16545 }
16546
16547 #[tokio::test]
16548 async fn review_only_summary_receives_the_configured_github_language() {
16549 let mut f = summary_fixture();
16550 f.runner.state.config.graph.github_language = "ja".into();
16551 let captured = f.repo.join("summary-prompt.txt");
16552 let author = &mut f.runner.state.config.agents[0];
16553 author.command = vec![
16554 "sh".into(),
16555 "-c".into(),
16556 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16557 ];
16558 author.env.insert(
16559 "PROMPT_FILE".into(),
16560 captured.to_string_lossy().into_owned(),
16561 );
16562 std::fs::write(
16563 &f.reply,
16564 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16565 )
16566 .unwrap();
16567 let m = posted(&mut f).await;
16568 let prompt = std::fs::read_to_string(captured).unwrap();
16569 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16570 assert!(!prompt.contains("in English"));
16571 assert_eq!(m.title, "feat(own): ファイルを追加");
16572 assert!(m.body.contains("own.txt を追加します。"));
16573 }
16574
16575 #[tokio::test]
16576 async fn review_fixes_come_after_the_whole_change_summary() {
16577 let mut f = summary_fixture();
16578 f.runner.state.reviews = vec![
16579 round_with_notes(1, Some("handled the PATH case")),
16580 round_with_notes(2, None),
16581 ];
16582 let m = posted(&mut f).await;
16583 let summary = m.body.find("## Summary").expect("summary");
16584 let fixes = m.body.find("## Review rounds").expect("review rounds");
16585 assert!(summary < fixes, "{}", m.body);
16586 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16587 assert!(m.body.contains("## Final verification"), "{}", m.body);
16588 }
16589
16590 #[tokio::test]
16591 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16592 let mut f = summary_fixture();
16593 let first = posted(&mut f).await;
16594 let first_head = f.runner.state.change_summary.clone().unwrap().head;
16595 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16597 let again = posted(&mut f).await;
16598 assert_eq!((again.title, again.body), (first.title, first.body));
16599
16600 let g = |args: &[&str]| {
16601 let out = std::process::Command::new("git")
16602 .args(args)
16603 .current_dir(&f.repo)
16604 .quiet()
16605 .output()
16606 .expect("spawn git");
16607 assert!(out.status.success(), "git {args:?}");
16608 };
16609 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16610 g(&["add", "-A"]);
16611 g(&["commit", "-q", "-m", "fix(own): handle more"]);
16612 let second = posted(&mut f).await;
16613 let kept = f.runner.state.change_summary.clone().unwrap();
16614 assert_ne!(kept.head, first_head);
16615 assert!(second.body.contains("Not read yet."), "{}", second.body);
16616 assert!(!second.body.contains("so the first summary is read"));
16617 assert_eq!(second.title, "feat(own): other");
16618 }
16619
16620 #[tokio::test]
16621 async fn an_unavailable_summary_is_stated_and_never_invented() {
16622 let mut f = summary_fixture();
16623 std::fs::write(&f.reply, "no title line here").unwrap();
16624 let m = posted(&mut f).await;
16625 assert!(
16626 m.body
16627 .contains("No description of what this change does could be generated"),
16628 "{}",
16629 m.body
16630 );
16631 assert!(m.body.contains("internal reports remain in local records"));
16632 assert!(f.runner.state.change_summary.is_none());
16633 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16634 let winner = f.runner.state.candidates[0].clone();
16636 let preview = f
16637 .runner
16638 .guarded_pr_message(&winner, None, false)
16639 .await
16640 .unwrap()
16641 .unwrap();
16642 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16643 }
16644
16645 #[test]
16646 fn pr_message_review_single_commit_uses_its_subject() {
16647 let state = review_state(&["feat(nats): per-role user"]);
16648 let m = pr_message(&state, 'A');
16649 assert_eq!(m.title, "feat(nats): per-role user");
16650 assert!(!m.body.contains("Review the work already"), "{}", m.body);
16651 assert!(m.body.contains("## Commits under review"), "{}", m.body);
16652 }
16653
16654 #[test]
16655 fn pr_message_review_multi_commit_takes_the_oldest() {
16656 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16657 let m = pr_message(&state, 'A');
16658 assert_eq!(m.title, "feat: the change");
16659 for s in ["feat: the change", "fix: typo", "fix: again"] {
16660 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16661 }
16662 }
16663
16664 #[test]
16665 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16666 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16667 let mut state = review_state(&[first, "fix: later fixup"]);
16668 state.candidates[0].branch = "機能/ログイン".to_owned();
16669 let m = pr_message(&state, 'A');
16670 assert!(
16671 m.title.starts_with("chore: land candidate A of run"),
16672 "{}",
16673 m.title
16674 );
16675 }
16676 }
16677
16678 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16679 BranchFacts {
16680 commits: commits
16681 .iter()
16682 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16683 .collect(),
16684 stat: stat.to_owned(),
16685 }
16686 }
16687
16688 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16689 let mut r = review_round(true, 0, 1, 1, true, true);
16690 r.round = round;
16691 r.fix = notes.map(|n| FixRecord {
16692 agent: "fixer".to_owned(),
16693 addressed: Vec::new(),
16694 rejected: Vec::new(),
16695 notes: n.to_owned(),
16696 committed: true,
16697 failed: None,
16698 duration_ms: 0,
16699 continuation: None,
16700 });
16701 r
16702 }
16703
16704 #[test]
16705 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16706 let state = review_state(&["ignored"]);
16707 let f = facts(
16708 &[
16709 (
16710 "fix(login): resolve PATH on macOS",
16711 "Login shells skip rc files.",
16712 ),
16713 ("fix: address review", ""),
16714 ],
16715 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16716 );
16717 let m = pr_message_with(&state, 'A', Some(&f));
16718 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16719 assert!(
16720 m.body
16721 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
16722 "{}",
16723 m.body
16724 );
16725 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16726 assert!(m.body.contains("## Diff stat"), "{}", m.body);
16727 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16728 for banned in [
16729 "Review the work already",
16730 "no task statement",
16731 "Original task",
16732 ] {
16733 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16734 }
16735 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16736 }
16737
16738 #[test]
16739 fn pr_message_review_truncates_a_huge_first_commit_body() {
16740 let state = review_state(&["ignored"]);
16741 let f = facts(
16742 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16743 "s",
16744 );
16745 let m = pr_message_with(&state, 'A', Some(&f));
16746 assert!(m.body.len() < 30_000, "{}", m.body.len());
16747 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16748 assert!(
16749 m.body.contains("- ... 1 more commit(s)"),
16750 "{}",
16751 m.body.len()
16752 );
16753 assert!(m.body.ends_with("magi:candidate-a\n"));
16754 }
16755
16756 #[test]
16757 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16758 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16759 assert_eq!(m.title, "feat: x");
16760 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16761 assert!(!m.body.contains("Diff stat"), "{}", m.body);
16762 assert!(!m.body.contains("no task statement"), "{}", m.body);
16763 }
16764
16765 #[test]
16766 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16767 let mut state = review_state(&["日本語の件名"]);
16768 state.candidates[0].branch = "fix/macos-login-path".to_owned();
16769 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16770 state.candidates[0].branch = "機能/ログイン".to_owned();
16771 assert!(
16772 pr_message(&state, 'A')
16773 .title
16774 .starts_with("chore: land candidate A")
16775 );
16776 }
16777
16778 #[test]
16779 fn pr_message_review_fixes_survive_a_clean_final_round() {
16780 let mut state = review_state(&["feat: x"]);
16781 state.reviews = vec![
16782 round_with_notes(1, Some("handled the PATH case")),
16783 round_with_notes(2, None),
16784 ];
16785 let body = pr_message(&state, 'A').body;
16786 assert!(
16787 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16788 "{body}"
16789 );
16790 assert!(body.contains("### Round 1"), "{body}");
16791
16792 state.reviews = vec![
16793 round_with_notes(1, Some("first fix")),
16794 round_with_notes(2, Some("")),
16795 round_with_notes(3, Some("second fix")),
16796 round_with_notes(4, None),
16797 ];
16798 let body = pr_message(&state, 'A').body;
16799 assert!(
16800 body.contains("### Round 1") && !body.contains("first fix"),
16801 "{body}"
16802 );
16803 assert!(
16804 body.contains("### Round 3") && !body.contains("second fix"),
16805 "{body}"
16806 );
16807 assert!(body.contains("### Round 2"), "{body}");
16808 }
16809
16810 const FINAL: &str = "abcdef1234567890";
16811
16812 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16813 let mut r = round_with_notes(round, None);
16814 r.verified_head = verified.map(str::to_owned);
16815 r
16816 }
16817
16818 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16819 state.gate = vec![CommandOutcome {
16820 command: "gate".to_owned(),
16821 code: Some(0),
16822 output_tail: String::new(),
16823 duration_ms: 0,
16824 resource_blocked: false,
16825 }];
16826 state.gate_ran = true;
16827 state.gate_head = head.map(str::to_owned);
16828 }
16829
16830 fn body_at(state: &RunState) -> String {
16831 pr_message_raw(
16832 state,
16833 'A',
16834 None,
16835 crate::pr_outcome::FinalHead::At(FINAL),
16836 ChangeNote::NotAsked,
16837 )
16838 .body
16839 }
16840
16841 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16842
16843 #[test]
16844 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16845 let mut state = review_state(&["feat: x"]);
16846 state.reviews = vec![
16847 round_with_notes(1, Some(SANDBOXED)),
16848 passing_round(2, Some(FINAL)),
16849 ];
16850 ok_gate(&mut state, Some(FINAL));
16851 let body = body_at(&state);
16852 assert!(!body.contains("Handled the PATH case."), "{body}");
16853 assert!(!body.contains("could not commit"), "{body}");
16854 assert!(!body.contains("could not run"), "{body}");
16855 assert!(
16856 body.contains("The fix is committed on the branch"),
16857 "{body}"
16858 );
16859 assert!(!body.contains("sandbox"), "{body}");
16860 assert!(
16861 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16862 "{body}"
16863 );
16864 assert!(
16865 body.contains("Gate (`verify.gate`): passed on the final commit"),
16866 "{body}"
16867 );
16868 assert!(
16870 state.reviews[0]
16871 .fix
16872 .as_ref()
16873 .unwrap()
16874 .notes
16875 .contains("could not commit")
16876 );
16877 }
16878
16879 #[test]
16880 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16881 let mut state = review_state(&["feat: x"]);
16882 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16883 r1.fix.as_mut().unwrap().committed = false;
16884 let mut r2 = passing_round(2, Some(FINAL));
16885 r2.e2e[0].code = Some(1);
16886 state.reviews = vec![r1, r2];
16887 let body = body_at(&state);
16888 assert!(!body.contains("could not commit"), "{body}");
16889 assert!(!body.contains("could not run"), "{body}");
16890 assert!(body.contains("This round produced no commit"), "{body}");
16891 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16892 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16893 assert!(!body.contains("passed"), "{body}");
16894 }
16895
16896 #[test]
16897 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16898 let mut state = review_state(&["feat: x"]);
16899 state.reviews = vec![
16900 round_with_notes(1, Some(SANDBOXED)),
16901 passing_round(2, Some("0123456789abcdef")),
16902 ];
16903 ok_gate(&mut state, Some("0123456789abcdef"));
16904 let body = body_at(&state);
16905 assert!(!body.contains("HTTP tests could not run"), "{body}");
16906 assert!(!body.contains("could not commit"), "{body}");
16908 assert!(
16909 body.contains("passed on an earlier commit (`0123456`)"),
16910 "{body}"
16911 );
16912 assert!(!body.contains("passed on the final commit"), "{body}");
16913
16914 ok_gate(&mut state, None);
16916 let body = body_at(&state);
16917 assert!(
16918 body.contains("passed on a commit that was not recorded"),
16919 "{body}"
16920 );
16921 }
16922
16923 #[test]
16924 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16925 let state = state_with_summary(
16926 "add retries to the client",
16927 "TITLE: feat: retries\n\nDid it.",
16928 );
16929 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16930 assert_eq!(m.title, "feat: retries");
16931 assert!(
16932 m.body.contains("<summary>Original task</summary>"),
16933 "{}",
16934 m.body
16935 );
16936 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16937 assert!(
16938 m.body
16939 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16940 "{}",
16941 m.body
16942 );
16943 }
16944
16945 #[test]
16946 fn pr_message_review_bounds_a_long_english_subject() {
16947 let long = format!("feat: {}", "word ".repeat(100));
16948 let m = pr_message(&review_state(&[&long]), 'A');
16949 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16950 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16951 }
16952
16953 #[test]
16954 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16955 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16956 let m = pr_message(&state, 'A');
16957 assert_eq!(m.title, "add retries");
16958 assert!(m.body.contains("<summary>Original task</summary>"));
16959 assert!(!m.body.contains("Commits under review"));
16960 assert_eq!(landing_subject_source(&state), state.instruction);
16961 }
16962
16963 #[test]
16964 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16965 let state = review_state(&["feat: the change", "fix: typo"]);
16966 let source = landing_subject_source(&state);
16967 assert_eq!(land::merge_subject("", &source), "feat: the change");
16968 assert_eq!(
16969 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16970 "feat: the change"
16971 );
16972 assert_eq!(
16974 land::merge_subject("feat: renamed by hand", &source),
16975 "feat: renamed by hand"
16976 );
16977 let blank = review_state(&["日本語"]);
16978 assert!(
16979 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16980 );
16981 }
16982
16983 #[test]
16984 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16985 let state = review_state(&["feat: the change"]);
16986 let source = landing_subject_source(&state);
16987 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16988 assert_eq!(
16989 land::merge_subject(landing_title(&state, old), &source),
16990 "feat: the change"
16991 );
16992 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16993 let task = state_with_summary("add retries", "");
16994 assert_eq!(landing_title(&task, old), old);
16995 }
16996
16997 #[test]
16998 fn pr_message_describes_the_change_not_the_task() {
16999 let state = state_with_summary(
17000 "今回やってほしいこと: results projector を直す",
17001 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17002 );
17003 let m = pr_message(&state, 'A');
17004 assert_eq!(m.title, "fix(web): batch the runs list reads");
17005 assert!(
17006 m.body.starts_with("## Summary\n\n- reads run.json once"),
17007 "{}",
17008 m.body
17009 );
17010 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17011 let task_at = m.body.find("今回やってほしいこと").unwrap();
17012 let details_at = m.body.find("<details>").unwrap();
17013 assert!(
17014 details_at < task_at,
17015 "the task lives inside <details>: {}",
17016 m.body
17017 );
17018 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17019 assert!(m.body.contains("magi:candidate-a"));
17020 }
17021
17022 #[test]
17023 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17024 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17025 let m = pr_message(&state, 'A');
17026 assert_eq!(m.title, "add retries");
17027 assert!(
17028 m.body.contains("## Summary\n\n- did some things"),
17029 "{}",
17030 m.body
17031 );
17032
17033 let none = RunState::new(
17034 PathBuf::from("/repo"),
17035 "main".to_owned(),
17036 "abc1234".to_owned(),
17037 "add retries".to_owned(),
17038 Config::default(),
17039 );
17040 let m = pr_message(&none, 'A');
17041 assert_eq!(m.title, "add retries");
17042 assert!(!m.body.contains("## Summary"), "{}", m.body);
17043 }
17044
17045 #[test]
17046 fn pr_message_refuses_the_candidate_commit_subject() {
17047 for bad in [
17048 "TITLE: magi: candidate A (uncommitted work)",
17049 "TITLE: chore: stuff (uncommitted work)",
17050 "TITLE: ",
17051 ] {
17052 let state = state_with_summary("add retries", bad);
17053 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17054 }
17055 }
17056
17057 #[test]
17058 fn pr_message_bounds_a_very_long_task_and_title() {
17059 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17060 let state = state_with_summary(&long, "- nothing");
17061 let m = pr_message(&state, 'A');
17062 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17063 assert!(!m.title.contains('\n'));
17064
17065 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17066 let m = pr_message(&state, 'A');
17067 assert!(m.title.starts_with("feat: "));
17068 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17069 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17070 }
17071
17072 fn long_title_of(instruction: &str) -> String {
17073 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17074 }
17075
17076 #[test]
17077 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17078 let first = "Make the landing path keep a readable title for long tasks";
17079 let line = format!(
17080 "{first}. {}",
17081 "And then keep going with more words ".repeat(20)
17082 );
17083 let t = long_title_of(&line);
17084 assert_eq!(t, first);
17085 assert!(!t.starts_with("chore: land"));
17086 }
17087
17088 #[test]
17089 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17090 let line = "word ".repeat(200);
17091 let t = long_title_of(&line);
17092 assert!(t.ends_with("word..."), "{t}");
17093 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17094 }
17095
17096 #[test]
17097 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17098 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17099 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17100 }
17101 }
17102
17103 #[test]
17104 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17105 let mut state = state_with_summary("再試行を追加する", "- nothing");
17106 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17107 state.config.graph.github_language = "ja".into();
17108 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17109 }
17110
17111 #[test]
17112 fn pr_message_title_limit_is_exact() {
17113 let at = "a".repeat(PR_TITLE_MAX);
17114 assert_eq!(long_title_of(&at), at);
17115 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17116 assert!(over.ends_with("..."), "{over}");
17117 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17118 }
17119
17120 #[test]
17121 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17122 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17123 let t = long_title_of(&line);
17124 assert!(t.ends_with("..."), "{t}");
17125 assert!(t.is_ascii(), "{t}");
17126 }
17127
17128 #[test]
17129 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17130 let line = format!(
17131 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17132 "plus more filler words ".repeat(20)
17133 );
17134 let t = long_title_of(&line);
17135 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17136 }
17137
17138 #[test]
17139 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17140 let line = format!("\n\n# {}", "title words ".repeat(40));
17141 let state = state_with_summary(&line, "- nothing");
17142 let m = pr_message(&state, 'A');
17143 assert!(m.title.starts_with("title words"), "{}", m.title);
17144 assert_eq!(
17145 land::merge_subject(&m.title, &landing_subject_source(&state)),
17146 m.title
17147 );
17148 assert_eq!(
17150 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17151 "feat: renamed by hand"
17152 );
17153 }
17154
17155 #[tokio::test]
17156 async fn github_text_rewrite_is_bounded_and_falls_back() {
17157 crate::run::pin_test_home();
17158 for (reply, accepted) in [
17159 (
17160 "TITLE: fix: retries\nAdd retries for failed requests.",
17161 true,
17162 ),
17163 (
17164 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17165 false,
17166 ),
17167 (
17168 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17169 true,
17170 ),
17171 (
17172 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17173 true,
17174 ),
17175 (
17176 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17177 true,
17178 ),
17179 (
17180 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17181 false,
17182 ),
17183 ("TITLE: fix: retries\n", false),
17184 ] {
17185 let dir = tempfile::tempdir().unwrap();
17186 let mut runner = runner_at(RunStatus::Gating);
17187 runner.state = state_with_summary(
17188 "add retries",
17189 "TITLE: fix: retries\n日本語の説明を書きます。",
17190 );
17191 runner.state.repo = dir.path().to_owned();
17192 runner.state.candidates[0].worktree = dir.path().to_owned();
17193 let mut author = spec("alpha");
17194 author.command = vec![
17195 "sh".into(),
17196 "-c".into(),
17197 "printf '%s' \"$REWRITE_REPLY\"".into(),
17198 ];
17199 author.env.insert("REWRITE_REPLY".into(), reply.into());
17200 runner.state.config.agents = vec![author];
17201 seed_public_summary(&mut runner);
17202 let winner = runner.state.candidates[0].clone();
17203 let first = runner
17204 .guarded_pr_message(&winner, None, true)
17205 .await
17206 .unwrap();
17207 let message = if accepted {
17208 first.expect("a passing rewrite never asks")
17209 } else {
17210 assert!(first.is_none());
17212 assert!(runner.state.parked);
17213 let store = ask::Questions::open();
17214 let asked: Vec<_> = store
17215 .list()
17216 .into_iter()
17217 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17218 .collect();
17219 assert_eq!(asked.len(), 1);
17220 assert!(!asked[0].detail.contains("secret"));
17221 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17222 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17223 assert_eq!(
17224 asked[0]
17225 .choices
17226 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17227 rewrite.is_some()
17228 );
17229 store
17232 .update(&asked[0].id, |q| {
17233 q.abandon("test");
17234 Ok(())
17235 })
17236 .unwrap();
17237 let again = runner
17238 .guarded_pr_message(&winner, None, true)
17239 .await
17240 .unwrap()
17241 .expect("an abandoned question falls back");
17242 let n = store
17243 .list()
17244 .into_iter()
17245 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17246 .count();
17247 assert_eq!(n, 1, "asked once per rejected text");
17248 again
17249 };
17250 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17251 assert!(!message.body.contains("secret"));
17252 assert!(!message.body.contains("prefix"));
17253 assert!(!message.title.contains("secret"));
17254 assert_eq!(
17255 message.body.contains("Add retries for failed requests."),
17256 accepted
17257 );
17258 assert_eq!(
17259 runner.state.seats["github-text-rewrite"].turns, 1,
17260 "only one rewrite invocation"
17261 );
17262 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17263 if accepted {
17264 assert!(runner.state.github_text.is_none());
17265 assert!(!runner.state.parked);
17266 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17267 assert!(message.body.contains("## Final verification"));
17268 assert!(message.body.contains("Evidence: final diff"));
17269 assert!(
17270 message
17271 .body
17272 .contains(&format!("magi:run/{}", runner.state.id))
17273 );
17274 }
17275 if !accepted {
17276 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17277 assert!(
17278 message
17279 .body
17280 .contains(&format!("magi:run/{}", runner.state.id))
17281 );
17282 }
17283 }
17284 }
17285
17286 fn scripted(id: &str, reply: &str) -> AgentSpec {
17287 let mut a = spec(id);
17288 a.command = vec![
17289 "sh".into(),
17290 "-c".into(),
17291 "printf '%s' \"$REWRITE_REPLY\"".into(),
17292 ];
17293 a.env.insert("REWRITE_REPLY".into(), reply.into());
17294 a
17295 }
17296
17297 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17298 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17299
17300 fn seed_public_summary(runner: &mut Runner) {
17302 let repo = &runner.state.repo;
17303 let g = |args: &[&str]| {
17304 let out = std::process::Command::new("git")
17305 .args(args)
17306 .current_dir(repo)
17307 .quiet()
17308 .output()
17309 .unwrap();
17310 assert!(out.status.success(), "git {args:?}");
17311 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17312 };
17313 g(&["init", "-q", "-b", "main"]);
17314 g(&[
17315 "-c",
17316 "user.name=Test",
17317 "-c",
17318 "user.email=test@example.invalid",
17319 "commit",
17320 "-q",
17321 "--allow-empty",
17322 "-m",
17323 "feat: fixture",
17324 ]);
17325 let head = g(&["rev-parse", "HEAD"]);
17326 g(&["update-ref", "refs/remotes/origin/main", &head]);
17327 let evidence = crate::pr_outcome::summary_evidence(
17328 &runner.state,
17329 crate::pr_outcome::FinalHead::At(&head),
17330 );
17331 let generated = &runner.state.candidates[0].summary;
17332 runner.state.change_summary = Some(crate::run::ChangeSummary {
17333 base: head.clone(),
17334 head,
17335 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17336 title: summary_title(generated).unwrap(),
17337 body: summary_without_title(generated),
17338 });
17339 }
17340
17341 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17342 let mut runner = runner_at(RunStatus::Gating);
17343 runner.state = state_with_summary("add retries", JA_SUMMARY);
17344 if review_only {
17345 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17346 }
17347 runner.state.repo = dir.to_owned();
17348 runner.state.candidates[0].worktree = dir.to_owned();
17349 runner.state.config.agents = agents;
17350 seed_public_summary(&mut runner);
17351 runner
17352 }
17353
17354 fn gate_questions(runner: &Runner) -> usize {
17355 ask::Questions::open()
17356 .list()
17357 .into_iter()
17358 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17359 .count()
17360 }
17361
17362 #[tokio::test]
17363 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17364 crate::run::pin_test_home();
17365 let dir = tempfile::tempdir().unwrap();
17366 let mut runner = rewrite_runner(
17367 dir.path(),
17368 true,
17369 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17370 );
17371 runner.state.config.roles.language_judge =
17372 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17373 let winner = runner.state.candidates[0].clone();
17374 let msg = runner
17375 .guarded_pr_message(&winner, None, true)
17376 .await
17377 .unwrap()
17378 .expect("an English rewrite posts without asking");
17379 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17380 assert!(msg.body.contains("<details>kept</details>"));
17381 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17382 assert_eq!(gate_questions(&runner), 0);
17383 assert!(runner.state.github_text.is_none());
17384 assert!(!runner.state.seats.contains_key("impl-A"));
17385 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17386 }
17387
17388 #[tokio::test]
17389 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17390 crate::run::pin_test_home();
17391 let dir = tempfile::tempdir().unwrap();
17392 let mut runner = rewrite_runner(
17393 dir.path(),
17394 false,
17395 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17396 );
17397 runner.state.config.roles.language_judge =
17398 Some(crate::config::AgentChoice::One("beta".into()));
17399 runner.state.seats.insert(
17400 "impl-A".into(),
17401 SeatState::new("impl-A", "alpha", runner.state.seed),
17402 );
17403 let winner = runner.state.candidates[0].clone();
17404 let msg = runner
17405 .guarded_pr_message(&winner, None, true)
17406 .await
17407 .unwrap()
17408 .expect("rewritten");
17409 assert!(msg.body.contains("<details>kept</details>"));
17410 assert_eq!(runner.state.seats["impl-A"].turns, 0);
17411 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17412 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17413 }
17414
17415 #[tokio::test]
17416 async fn rewrite_reuses_a_resumable_implementer_session() {
17417 crate::run::pin_test_home();
17418 let dir = tempfile::tempdir().unwrap();
17419 let mut runner = rewrite_runner(
17420 dir.path(),
17421 false,
17422 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17423 );
17424 runner.state.config.roles.language_judge =
17425 Some(crate::config::AgentChoice::One("beta".into()));
17426 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17427 seat.turns = 1;
17428 runner.state.seats.insert("impl-A".into(), seat);
17429 let winner = runner.state.candidates[0].clone();
17430 runner
17431 .guarded_pr_message(&winner, None, true)
17432 .await
17433 .unwrap()
17434 .expect("rewritten");
17435 assert_eq!(runner.state.seats["impl-A"].turns, 2);
17436 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17437 }
17438
17439 #[tokio::test]
17440 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17441 crate::run::pin_test_home();
17442 let dir = tempfile::tempdir().unwrap();
17443 let mut broken = spec("bad");
17444 broken.command = vec!["false".into()];
17445 let mut runner = rewrite_runner(
17446 dir.path(),
17447 true,
17448 vec![
17449 broken,
17450 scripted("noformat", "Add retries."),
17451 scripted("good", EN_REWRITE),
17452 ],
17453 );
17454 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17455 "bad".into(),
17456 "noformat".into(),
17457 "good".into(),
17458 ]));
17459 let winner = runner.state.candidates[0].clone();
17460 let msg = runner
17461 .guarded_pr_message(&winner, None, true)
17462 .await
17463 .unwrap()
17464 .expect("the third agent rewrites");
17465 assert!(msg.body.contains("`src/net.rs`"));
17466 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17467 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17468 }
17469
17470 #[tokio::test]
17471 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17472 crate::run::pin_test_home();
17473 for (agents, role, saved) in [
17475 (
17476 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17477 Some("beta"),
17478 true,
17479 ),
17480 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17481 (Vec::new(), None, false),
17482 ] {
17483 let dir = tempfile::tempdir().unwrap();
17484 let mut runner = rewrite_runner(dir.path(), true, agents);
17485 runner.state.config.roles.language_judge =
17486 role.map(|r| crate::config::AgentChoice::One(r.into()));
17487 let winner = runner.state.candidates[0].clone();
17488 let first = runner
17489 .guarded_pr_message(&winner, None, true)
17490 .await
17491 .unwrap();
17492 assert!(first.is_none(), "withheld text parks the run");
17493 assert!(runner.state.parked);
17494 assert_eq!(gate_questions(&runner), 1);
17495 let g = runner.state.github_text.as_ref().unwrap();
17496 assert_eq!(g.rewrite.is_some(), saved);
17497 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17498 }
17499 }
17500
17501 #[tokio::test]
17502 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17503 crate::run::pin_test_home();
17504 let dir = tempfile::tempdir().unwrap();
17505 let mut runner = runner_at(RunStatus::Gating);
17506 runner.state =
17507 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17508 runner.state.repo = dir.path().to_owned();
17509 runner.state.candidates[0].worktree = dir.path().to_owned();
17510 let mut author = spec("alpha");
17511 author.command = vec![
17512 "sh".into(),
17513 "-c".into(),
17514 "printf '%s' \"$REWRITE_REPLY\"".into(),
17515 ];
17516 author.env.insert(
17517 "REWRITE_REPLY".into(),
17518 format!(
17519 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17520 "English sentence. ".repeat(600)
17521 ),
17522 );
17523 runner.state.config.agents = vec![author];
17524 seed_public_summary(&mut runner);
17525 let winner = runner.state.candidates[0].clone();
17526 assert!(
17527 runner
17528 .guarded_pr_message(&winner, None, true)
17529 .await
17530 .unwrap()
17531 .is_none()
17532 );
17533 let record = runner.state.github_text.clone().unwrap();
17534 let rewrite = record.rewrite.clone().unwrap();
17535 assert_eq!(rewrite.failed, ["body-language"]);
17536 assert!(
17537 rewrite
17538 .body
17539 .ends_with(&format!("magi:run/{}", runner.state.id))
17540 );
17541 let store = ask::Questions::open();
17542 let q = store.get(record.question.as_ref().unwrap()).unwrap();
17543 assert!(
17544 q.choices
17545 .contains(&crate::github_text::USE_REWRITE.to_owned())
17546 );
17547 for label in [
17548 "ORIGINAL",
17549 "FALLBACK",
17550 "REWRITE",
17551 "Rules still failed: body-language",
17552 "truncated",
17553 ] {
17554 assert!(q.detail.contains(label), "{label}");
17555 }
17556 let artifact = std::fs::read_to_string(crate::run::artifact_path(
17557 &runner.state,
17558 &crate::github_text::artifact_name(&record.fingerprint),
17559 ))
17560 .unwrap();
17561 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17562 runner.state =
17564 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17565 store
17566 .update(&q.id, |q| {
17567 q.say("まだ日本語")?;
17568 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17569 })
17570 .unwrap();
17571 assert!(
17572 runner
17573 .guarded_pr_message(&winner, None, true)
17574 .await
17575 .unwrap()
17576 .is_none()
17577 );
17578 let retry = runner.state.github_text.as_ref().unwrap();
17579 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17580 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17581 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17582 assert!(
17583 q.posting_comparison
17584 .as_ref()
17585 .unwrap()
17586 .matches_detail(&q.detail)
17587 );
17588 store
17589 .update(&q.id, |q| {
17590 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17591 })
17592 .unwrap();
17593 let message = runner
17594 .guarded_pr_message(&winner, None, true)
17595 .await
17596 .unwrap()
17597 .unwrap();
17598 assert_eq!(message.title, rewrite.title);
17599 assert_eq!(message.body, rewrite.body);
17600 assert_eq!(
17601 runner.state.github_text.as_ref().unwrap().choice,
17602 Some(GithubTextChoice::Rewrite)
17603 );
17604 let again = runner
17605 .guarded_pr_message(&winner, None, true)
17606 .await
17607 .unwrap()
17608 .unwrap();
17609 assert_eq!(again.title, message.title);
17610 assert_eq!(again.body, message.body);
17611 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17612 assert!(
17613 !runner
17614 .state
17615 .events
17616 .iter()
17617 .any(|e| e.message.contains(&rewrite.title)
17618 || e.message.contains("English sentence."))
17619 );
17620 }
17621
17622 #[tokio::test]
17623 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17624 crate::run::pin_test_home();
17625 let dir = tempfile::tempdir().unwrap();
17626 let mut runner = runner_at(RunStatus::Gating);
17627 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17628 runner.state.repo = dir.path().to_owned();
17629 runner.state.candidates[0].worktree = dir.path().to_owned();
17630 seed_public_summary(&mut runner);
17631 let winner = runner.state.candidates[0].clone();
17632 assert!(
17633 runner
17634 .guarded_pr_message(&winner, None, true)
17635 .await
17636 .unwrap()
17637 .is_none()
17638 );
17639 let record = runner.state.github_text.as_ref().unwrap();
17640 assert!(record.rewrite.is_none());
17641 let mut old = serde_json::to_value(record).unwrap();
17643 old.as_object_mut().unwrap().remove("rewrite");
17644 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17645 assert!(old.rewrite.is_none());
17646 let id = record.question.clone().unwrap();
17647 ask::Questions::open()
17648 .update(&id, |q| {
17649 q.choices.push(crate::github_text::USE_REWRITE.into());
17650 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17651 })
17652 .unwrap();
17653 let message = runner
17654 .guarded_pr_message(&winner, None, true)
17655 .await
17656 .unwrap()
17657 .unwrap();
17658 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17659 assert_eq!(
17660 runner.state.github_text.as_ref().unwrap().choice,
17661 Some(GithubTextChoice::Fallback)
17662 );
17663 }
17664
17665 #[tokio::test]
17666 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17667 crate::run::pin_test_home();
17668 let dir = tempfile::tempdir().unwrap();
17669 let mut runner = runner_at(RunStatus::Gating);
17670 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17671 runner.state.repo = dir.path().to_owned();
17672 runner.state.candidates[0].worktree = dir.path().to_owned();
17673 seed_public_summary(&mut runner);
17674 let winner = runner.state.candidates[0].clone();
17675 assert!(
17676 runner
17677 .guarded_pr_message(&winner, None, true)
17678 .await
17679 .unwrap()
17680 .is_none()
17681 );
17682 let store = ask::Questions::open();
17683 let find = |run: &str| {
17684 store
17685 .list()
17686 .into_iter()
17687 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17688 .unwrap()
17689 };
17690 let run = runner.state.id.clone();
17691 let q = find(&run);
17692 let fp = runner
17693 .state
17694 .github_text
17695 .as_ref()
17696 .unwrap()
17697 .fingerprint
17698 .clone();
17699 let name = crate::github_text::artifact_name(&fp);
17700 let path = crate::run::artifact_path(&runner.state, &name);
17701 assert!(q.detail.contains(&path.display().to_string()));
17702 assert!(q.detail.contains("title-language: title"));
17703 let saved = std::fs::read_to_string(&path).expect("artifact written");
17704 assert!(saved.contains("日本語のタイトル"));
17705 assert!(q.cwd.is_none());
17706 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17707 let say = |id: &str, text: &str, choice: &str| {
17709 store
17710 .update(id, |q| {
17711 q.thread.push(ask::Turn {
17712 who: ask::Who::Operator,
17713 body: text.to_owned(),
17714 at: jiff::Timestamp::now(),
17715 note: None,
17716 });
17717 q.answer(ask::Answer::Choice(choice.to_owned()))
17718 })
17719 .unwrap();
17720 };
17721 say(&q.id, "まだ日本語", "use my text");
17722 assert!(
17723 runner
17724 .guarded_pr_message(&winner, None, true)
17725 .await
17726 .unwrap()
17727 .is_none()
17728 );
17729 let second = find(&run);
17730 assert_ne!(second.id, q.id);
17731 assert!(second.detail.contains("did not pass"));
17732 assert!(second.detail.contains("日本語のタイトル"));
17735 assert!(!second.detail.contains("まだ日本語"));
17736 assert!(second.detail.contains(&path.display().to_string()));
17737 say(&second.id, "fix: retry failed requests", "use my text");
17738 let message = runner
17739 .guarded_pr_message(&winner, None, true)
17740 .await
17741 .unwrap()
17742 .expect("a vetted replacement is posted");
17743 assert_eq!(message.title, "fix: retry failed requests");
17744 assert_eq!(
17745 runner
17746 .state
17747 .github_text
17748 .as_ref()
17749 .unwrap()
17750 .chosen_title
17751 .as_deref(),
17752 Some("fix: retry failed requests")
17753 );
17754 assert!(runner.state.github_text.as_ref().unwrap().resolved);
17755 let again = runner
17757 .guarded_pr_message(&winner, None, true)
17758 .await
17759 .unwrap()
17760 .unwrap();
17761 assert_eq!(again.title, message.title);
17762 assert_eq!(
17763 store
17764 .list()
17765 .into_iter()
17766 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17767 .count(),
17768 2
17769 );
17770 }
17771
17772 #[test]
17773 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17774 let mut state = state_with_summary(
17778 "add retries",
17779 "TITLE: fix(web): batch reads\n- reads run.json once",
17780 );
17781 state.config.graph.language = "ja".to_owned();
17782 let m = pr_message(&state, 'A');
17783 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17784 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17785
17786 let task = "今回やってほしいこと: results projector を直す";
17789 let mut state = state_with_summary(task, "- no title line");
17790 state.config.graph.language = "ja".to_owned();
17791 let m = pr_message(&state, 'A');
17792 assert_eq!(
17793 m.title,
17794 format!("chore: land candidate A of run {}", state.id)
17795 );
17796 assert!(
17797 m.body.contains(&format!(
17798 "<summary>Original task</summary>\n\n{task}\n\n</details>"
17799 )),
17800 "{}",
17801 m.body
17802 );
17803 }
17804
17805 #[test]
17806 fn pr_message_scrubs_home_paths_and_addresses() {
17807 let state = state_with_summary(
17808 "fix it in /Users/someone/src/x",
17809 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17810 );
17811 let m = pr_message(&state, 'A');
17812 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17813 assert!(!m.body.contains(leak), "{}", m.body);
17814 }
17815 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17816 }
17817
17818 #[test]
17819 fn pr_message_survives_a_task_that_closes_details() {
17820 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17821 let m = pr_message(&state, 'A');
17822 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17823 }
17824
17825 #[test]
17826 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17827 let cmd = manual_merge_command(
17828 MergeStyle::Squash,
17829 Path::new("/repo"),
17830 "b",
17831 "fix: \"quoted\" $(x) `y`\n\nbody",
17832 );
17833 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17834 }
17835
17836 #[test]
17837 fn manual_merge_command_matches_the_configured_style() {
17838 let repo = Path::new("/repo");
17839 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17840
17841 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17842 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17843
17844 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17845 assert_eq!(
17846 squash,
17847 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17848 \"Merge magi run 0832 (candidate A)\""
17849 );
17850
17851 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17852 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17853 }
17854
17855 #[test]
17856 fn a_nudge_gets_a_quarter_of_the_budget() {
17857 assert_eq!(retry_budget(secs(1200), true), secs(300));
17859 assert_eq!(retry_budget(secs(3600), true), secs(900));
17860 }
17861
17862 #[test]
17863 fn a_resent_prompt_keeps_the_whole_budget() {
17864 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17867 assert_eq!(retry_budget(secs(60), false), secs(60));
17868 }
17869
17870 #[test]
17871 fn the_floor_never_exceeds_the_original_budget() {
17872 assert_eq!(retry_budget(secs(60), true), secs(60));
17876 assert_eq!(retry_budget(secs(480), true), secs(120));
17877 assert_eq!(retry_budget(secs(0), true), secs(0));
17878 }
17879
17880 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17881 agent::CommandEvidence {
17882 id: "item1".to_owned(),
17883 description: "cargo test".to_owned(),
17884 exit_code,
17885 result_summary: String::new(),
17886 source: "codex".to_owned(),
17887 }
17888 }
17889
17890 #[test]
17891 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17892 assert!(!has_unconfirmed_command(&[]));
17896 }
17897
17898 #[test]
17899 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17900 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17904 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17905 assert!(!has_unconfirmed_command(&[
17906 evidence(Some(0)),
17907 evidence(Some(101))
17908 ]));
17909 }
17910
17911 #[test]
17912 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17913 assert!(has_unconfirmed_command(&[
17914 evidence(Some(0)),
17915 evidence(None)
17916 ]));
17917 }
17918
17919 #[test]
17920 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17921 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17922 assert_eq!(
17923 verified_noop_claim(true, &[], text).as_deref(),
17924 Some("already fixed by b32cfc4, on main.")
17925 );
17926 }
17927
17928 #[test]
17929 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17930 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17933 assert!(verified_noop_claim(false, &[], text).is_none());
17934 }
17935
17936 #[test]
17937 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17938 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17939 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17940 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17942 }
17943
17944 #[test]
17945 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17946 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17947 }
17948
17949 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17952 runner.state.candidates = shape
17953 .iter()
17954 .enumerate()
17955 .map(|(i, &(empty, verified))| Candidate {
17956 index: i,
17957 label: (b'A' + i as u8) as char,
17958 agent: "sonnet".to_owned(),
17959 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17960 worktree: PathBuf::from(format!("/wt/{i}")),
17961 summary: String::new(),
17962 stat: String::new(),
17963 files: 0,
17964 commits: 0,
17965 empty,
17966 failed: None,
17967 verified_noop: verified.map(str::to_owned),
17968 duration_ms: 0,
17969 folded: false,
17970 })
17971 .collect();
17972 }
17973
17974 #[test]
17975 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17976 ask_test_home();
17977 let mut runner = runner_at(RunStatus::Implementing);
17978 set_candidates(
17979 &mut runner,
17980 &[
17981 (true, Some("already on main at b32cfc4")),
17982 (true, Some("same fix, see the existing test")),
17983 ],
17984 );
17985
17986 runner
17987 .after_implement()
17988 .expect("a verified no-op is not an error");
17989
17990 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17991 }
17992
17993 #[test]
17994 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17995 ask_test_home();
17996 let mut runner = runner_at(RunStatus::Implementing);
17997 set_candidates(
18001 &mut runner,
18002 &[(true, Some("already on main at b32cfc4")), (true, None)],
18003 );
18004
18005 let err = runner
18006 .after_implement()
18007 .expect_err("an unverified empty candidate must still fail the run");
18008
18009 assert!(
18010 err.to_string().contains("no candidate produced a change"),
18011 "{err}"
18012 );
18013 assert_eq!(runner.state.status, RunStatus::Failed);
18014 }
18015
18016 #[test]
18017 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18018 ask_test_home();
18019 let mut runner = runner_at(RunStatus::Implementing);
18020 set_candidates(&mut runner, &[(true, None), (true, None)]);
18021
18022 let err = runner
18023 .after_implement()
18024 .expect_err("no candidate declared anything; this is an ordinary failure");
18025
18026 assert!(
18027 err.to_string().contains("no candidate produced a change"),
18028 "{err}"
18029 );
18030 assert_eq!(runner.state.status, RunStatus::Failed);
18031 }
18032}
18033
18034#[derive(Debug)]
18037struct GraftRestoreFailed(String);
18038
18039impl std::fmt::Display for GraftRestoreFailed {
18040 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18041 write!(
18042 f,
18043 "could not restore the winner to its pre-graft commit: {}",
18044 self.0
18045 )
18046 }
18047}
18048
18049impl std::error::Error for GraftRestoreFailed {}
18050
18051#[cfg(test)]
18055mod resource_wait_tests {
18056 use super::*;
18057 use std::collections::BTreeMap;
18058
18059 fn sh(id: &str, script: &str) -> AgentSpec {
18060 AgentSpec {
18061 id: id.to_owned(),
18062 kind: crate::config::AgentKind::Command,
18063 model: None,
18064 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18065 extra_args: Vec::new(),
18066 env: BTreeMap::new(),
18067 prompt_delivery: None,
18068 }
18069 }
18070
18071 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18072 SeatJob {
18073 seat: SeatState::new("impl-A", &spec.id, 1),
18074 spec,
18075 cwd: dir.to_path_buf(),
18076 prompt: "do it".to_owned(),
18077 timeout,
18078 allow_write: true,
18079 sessions: false,
18080 artifacts: dir.join("art"),
18081 stem: "impl-A".to_owned(),
18082 handover: None,
18083 }
18084 }
18085
18086 fn state(resource_wait_secs: u64) -> RunState {
18087 let mut config = Config::default();
18088 config.graph.timeout_resource_wait = resource_wait_secs;
18089 RunState::new(
18090 PathBuf::from("/repo"),
18091 "main".to_owned(),
18092 "abc1234".to_owned(),
18093 "task".to_owned(),
18094 config,
18095 )
18096 }
18097
18098 fn hold(cache: &Path) -> crate::cache::Guard {
18099 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18100 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18101 crate::cache::AcquireOutcome::Acquired(g) => g,
18102 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18103 }
18104 }
18105
18106 async fn run(
18107 jobs: Vec<SeatJob>,
18108 sem: Arc<Semaphore>,
18109 cache: Option<&Path>,
18110 st: &mut RunState,
18111 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18112 let prompts = Prompts::default();
18113 let ctx = WaveCtx {
18114 run: "r",
18115 node: "implement",
18116 prompts: &prompts,
18117 cache,
18118 round: None,
18119 carry_seats: false,
18120 pauses: None,
18121 };
18122 wave(jobs, sem, &ctx, st, 0).await
18123 }
18124
18125 #[tokio::test]
18126 async fn verification_cache_wait_preserves_the_command_budget() {
18127 crate::run::pin_test_home();
18128 let tmp = tempfile::tempdir().unwrap();
18129 let cache = tmp.path().join("target");
18130 let guard = hold(&cache);
18131 let release = tokio::spawn(async move {
18132 tokio::time::sleep(Duration::from_millis(100)).await;
18133 guard.release();
18134 });
18135 let mut st = state(10);
18136 let budget = Duration::from_millis(50);
18137 let wait = repository_wait(&st, None);
18138 let (outcomes, _) = with_cache_lease(
18139 &mut st,
18140 Some(&cache),
18141 "gate",
18142 "gate",
18143 tmp.path(),
18144 "abc1234",
18145 budget,
18146 "test gate",
18147 &wait,
18148 |_, available| async move {
18149 assert_eq!(
18150 available, budget,
18151 "contention must not shorten verification"
18152 );
18153 (Vec::new(), false, Vec::new())
18154 },
18155 )
18156 .await
18157 .unwrap();
18158 release.await.unwrap();
18159 assert!(outcomes.is_empty(), "{outcomes:?}");
18160 assert!(
18161 st.events
18162 .iter()
18163 .any(|event| event.message.contains("waiting for"))
18164 );
18165 assert!(st.resource_wait.is_none());
18166 hold(&cache).release();
18168 }
18169
18170 #[tokio::test]
18171 async fn verification_cache_wait_expires_without_starting_commands() {
18172 crate::run::pin_test_home();
18173 let tmp = tempfile::tempdir().unwrap();
18174 let cache = tmp.path().join("target");
18175 let _held = hold(&cache);
18176 let mut st = state(0);
18177 let wait = repository_wait(&st, None);
18178 let (outcomes, _) = with_cache_lease(
18179 &mut st,
18180 Some(&cache),
18181 "gate",
18182 "gate",
18183 tmp.path(),
18184 "abc1234",
18185 Duration::from_secs(60),
18186 "test gate",
18187 &wait,
18188 |_, _| async { panic!("commands must not start without the cache lease") },
18189 )
18190 .await
18191 .unwrap();
18192 assert_eq!(outcomes.len(), 1);
18193 assert!(outcomes[0].resource_blocked);
18194 assert!(st.resource_wait.is_none());
18195 }
18196
18197 #[tokio::test]
18198 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18199 crate::run::pin_test_home();
18200 let tmp = tempfile::tempdir().unwrap();
18201 let cache = tmp.path().join("target-a");
18202 let guard = hold(&cache);
18203 tokio::spawn(async move {
18205 tokio::time::sleep(Duration::from_millis(1500)).await;
18206 guard.release();
18207 });
18208 let mut st = state(60);
18209 let results = run(
18210 vec![job(
18211 sh("alpha", "echo ok"),
18212 tmp.path(),
18213 Duration::from_secs(1),
18214 )],
18215 Arc::new(Semaphore::new(1)),
18216 Some(&cache),
18217 &mut st,
18218 )
18219 .await;
18220 assert!(
18221 matches!(results[0].2, AgentOutcome::Ok(_)),
18222 "the wait must not be charged to the agent"
18223 );
18224 assert!(st.resource_block.is_none());
18225 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18226 assert!(st.resource_wait.is_none() && st.active.is_empty());
18227 }
18228
18229 #[tokio::test]
18230 #[cfg(unix)]
18231 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18232 use std::os::fd::AsRawFd as _;
18233 crate::run::pin_test_home();
18234 let tmp = tempfile::tempdir().unwrap();
18235 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18236 let common = tmp.path().join(".git");
18237 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18238 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18239 assert_eq!(
18240 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18241 0
18242 );
18243 let mut st = state(1);
18244 st.config.coordination = crate::config::test_coordination();
18245 let mut reader = job(
18246 sh("reader", "echo must-not-run"),
18247 tmp.path(),
18248 Duration::from_secs(30),
18249 );
18250 reader.allow_write = false;
18251 let results = run(
18252 vec![reader.clone()],
18253 Arc::new(Semaphore::new(1)),
18254 None,
18255 &mut st,
18256 )
18257 .await;
18258 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18259 assert_eq!(results[0].1.turns, 0);
18260 assert!(FailClass::of(&results[0].2).is_none());
18261 assert_eq!(
18262 st.resource_block.as_ref().unwrap().kind,
18263 "repository coordination"
18264 );
18265 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18266 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18267 drop(external);
18268 let editor = crate::repo_lock::SeatLease::for_checkout(
18269 tmp.path(),
18270 true,
18271 crate::config::test_coordination().as_ref(),
18272 )
18273 .await
18274 .unwrap();
18275 let pause = Pause::new();
18276 let interrupt = Pause::new();
18277 let prompts = Prompts::default();
18278 let ctx = WaveCtx {
18279 run: "r",
18280 node: "review",
18281 prompts: &prompts,
18282 cache: None,
18283 round: None,
18284 carry_seats: false,
18285 pauses: Some((&pause, &interrupt)),
18286 };
18287 let (results, ()) = tokio::join!(
18288 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18289 async {
18290 tokio::time::sleep(Duration::from_millis(50)).await;
18291 pause.park();
18292 }
18293 );
18294 assert!(
18295 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18296 );
18297 assert_eq!(results[0].1.turns, 0);
18298 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18299 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18300 drop(editor);
18301 }
18302
18303 #[tokio::test]
18304 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18305 crate::run::pin_test_home();
18306 let tmp = tempfile::tempdir().unwrap();
18307 let cache = tmp.path().join("target-b");
18308 let _held = hold(&cache);
18309 let mut st = state(1);
18310 let results = run(
18311 vec![job(
18312 sh("alpha", "echo ok"),
18313 tmp.path(),
18314 Duration::from_secs(1),
18315 )],
18316 Arc::new(Semaphore::new(1)),
18317 Some(&cache),
18318 &mut st,
18319 )
18320 .await;
18321 let (_, seat, out) = &results[0];
18322 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18323 assert!(wave_blocked(&results));
18324 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18325 assert_eq!(seat.turns, 0, "nothing was asked");
18326 assert!(
18327 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18328 "no agent was spawned"
18329 );
18330 assert!(FailClass::of(out).is_none(), "not a failure class");
18331 let block = st.resource_block.as_ref().expect("recorded for the resume");
18332 assert_eq!(block.kind, "shared build cache");
18333 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18334 assert!(st.active.is_empty());
18335 }
18336
18337 #[tokio::test]
18338 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18339 crate::run::pin_test_home();
18340 let tmp = tempfile::tempdir().unwrap();
18341 let mut st = state(1);
18342 let results = run(
18343 vec![job(
18344 sh("alpha", "echo ok"),
18345 tmp.path(),
18346 Duration::from_secs(1),
18347 )],
18348 Arc::new(Semaphore::new(0)),
18349 None,
18350 &mut st,
18351 )
18352 .await;
18353 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18354 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18355 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18356 }
18357
18358 #[tokio::test]
18359 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18360 crate::run::pin_test_home();
18361 let tmp = tempfile::tempdir().unwrap();
18362 let mut st = state(60);
18363 let results = run(
18364 vec![job(
18365 sh("alpha", "sleep 30"),
18366 tmp.path(),
18367 Duration::from_secs(1),
18368 )],
18369 Arc::new(Semaphore::new(1)),
18370 None,
18371 &mut st,
18372 )
18373 .await;
18374 let out = &results[0].2;
18375 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18376 let class = FailClass::of(out).expect("a failure");
18377 assert_eq!(class, FailClass::Timeout);
18378 assert!(should_hand_over(None, &class), "existing handover policy");
18379 assert!(st.resource_block.is_none());
18380 }
18381}