1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47use crate::refs;
48use crate::run::{
49 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
50 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
51 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
52 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
53 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
54 Tally, VoteRecord, tail, write_artifact,
55};
56use crate::verdict::{
57 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
58 ReviewVote, Severity,
59};
60use crate::worktree_setup;
61
62const OUTPUT_TAIL: usize = 8_000;
64
65const EVENT_OUTPUT_TAIL: usize = 2_000;
68
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
72
73const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
90
91pub(crate) const STAGNANT_LIMIT: usize = 2;
105
106const BASE_SYNC_ROUNDS: usize = 4;
119
120const MAX_FIX_CONTINUATIONS: usize = 2;
136
137#[derive(Clone)]
143struct SeatJob {
144 spec: AgentSpec,
145 seat: SeatState,
146 cwd: PathBuf,
147 prompt: String,
148 timeout: Duration,
149 allow_write: bool,
150 sessions: bool,
151 artifacts: PathBuf,
152 stem: String,
153 handover: Option<String>,
158}
159
160enum AgentOutcome {
172 Ok(AgentOutput),
174 Quota(AgentOutput),
176 Dropped(AgentOutput),
179 Failed(String),
181 ResourceBlocked(String),
187}
188
189#[derive(Debug, Clone, Default)]
216pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
217
218impl Pause {
219 #[must_use]
221 pub fn new() -> Self {
222 Self::default()
223 }
224
225 pub fn park(&self) {
227 self.0.store(true, Ordering::SeqCst);
228 }
229
230 pub fn park_because(&self, reason: impl Into<String>) {
236 self.park_for(ParkCause::Interrupt {
237 reason: reason.into(),
238 });
239 }
240
241 pub fn park_for(&self, cause: ParkCause) {
244 let mut reason_guard = self
245 .1
246 .lock()
247 .unwrap_or_else(std::sync::PoisonError::into_inner);
248 if reason_guard.is_none() {
249 *reason_guard = Some(cause);
250 }
251 drop(reason_guard);
252 self.park();
253 }
254
255 #[must_use]
257 pub fn parked(&self) -> bool {
258 self.0.load(Ordering::SeqCst)
259 }
260
261 #[must_use]
263 pub fn reason(&self) -> Option<String> {
264 self.cause().map(|c| c.label())
265 }
266
267 #[must_use]
269 pub fn cause(&self) -> Option<ParkCause> {
270 self.1
271 .lock()
272 .unwrap_or_else(std::sync::PoisonError::into_inner)
273 .clone()
274 }
275}
276
277pub struct Runner {
279 pub state: RunState,
281 roles: ResolvedRoles,
282 sem: Arc<Semaphore>,
283 pause: Pause,
287 interrupt: Pause,
293}
294
295async fn review_base(
300 repo: &Path,
301 remote: &str,
302 base_branch: &str,
303 base_commit: &str,
304 branch: &str,
305) -> String {
306 review_base_checked(repo, remote, base_branch, base_commit, branch)
307 .await
308 .0
309}
310
311async fn review_base_checked(
316 repo: &Path,
317 remote: &str,
318 base_branch: &str,
319 base_commit: &str,
320 branch: &str,
321) -> (String, bool) {
322 let tracking = format!("{remote}/{base_branch}");
323 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
324 if git::rev_exists(repo, &tracking).await
325 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
326 && !mb.is_empty()
327 {
328 return (mb, fresh);
329 }
330 let mb = git::merge_base(repo, base_commit, branch)
331 .await
332 .ok()
333 .filter(|mb| !mb.is_empty())
334 .unwrap_or_else(|| base_commit.to_owned());
335 (mb, false)
336}
337
338pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
342 if !is_review_run(state) {
343 return;
344 }
345 let base = review_base(
346 &state.repo,
347 &state.config.merge.remote,
348 &state.base_branch,
349 &state.base_commit,
350 branch,
351 )
352 .await;
353 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
354 && !subjects.is_empty()
355 && state.reviewed_commits.as_ref() != Some(&subjects)
356 {
357 state.reviewed_commits = Some(subjects);
358 state.save().ok();
359 }
360}
361
362async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
366 let (base, trusted) = review_base_checked(
367 &state.repo,
368 &state.config.merge.remote,
369 &state.base_branch,
370 &state.base_commit,
371 branch,
372 )
373 .await;
374 if !trusted {
375 return None;
376 }
377 git::subjects(&state.repo, &state.base_commit, &base)
378 .await
379 .ok()
380}
381
382pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
386 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
387 if is_leak(computed) {
388 return false;
389 }
390 let cur = current.trim();
391 cur.is_empty()
392 || cur.starts_with(REVIEW_PROMPT_OPENING)
393 || cur.starts_with("chore: land candidate")
394 || cur.starts_with("magi: candidate")
395 || is_leak(cur)
396}
397
398async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
427 let tracking = format!("{remote}/{branch}");
428 let fetched = git::fetch(repo, remote, branch).await;
429 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
430 let local_exists = git::branch_exists(repo, branch).await?;
431 if !fresh {
432 if !local_exists {
433 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
434 }
435 tracing::warn!(
436 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
437 );
438 return Ok(());
439 }
440 let remote_sha = git::rev_parse(repo, &tracking).await?;
441 if !local_exists {
442 git::git(repo, &["branch", branch, &tracking]).await?;
443 return Ok(());
444 }
445 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
446 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
447 return Ok(());
448 }
449 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
450 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
456 .await?
457 {
458 crate::reconcile::Reconciliation::Pushed => {
459 tracing::warn!(
460 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
461 short(&local_sha),
462 short(&remote_sha)
463 );
464 return Ok(());
465 }
466 crate::reconcile::Reconciliation::Placeholder => {}
467 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
468 }
469 }
470 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
471 if !out.ok() {
472 bail!(
473 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
474 short(&local_sha),
475 short(&remote_sha),
476 out.stderr
477 );
478 }
479 tracing::warn!(
480 "local `{branch}` was stale: fast-forwarded {} -> {}",
481 short(&local_sha),
482 short(&remote_sha)
483 );
484 Ok(())
485}
486
487async fn resolve_base(
488 repo: &Path,
489 base_branch: &str,
490 remote: &str,
491 coordination: Option<&crate::config::Coordination>,
492) -> Result<crate::run::FetchedBase> {
493 let _lease = crate::repo_lock::SeatLease::operation_with(
494 repo,
495 coordination,
496 &crate::repo_lock::WaitPolicy::current(),
497 )
498 .await?;
499 let tracking = format!("{remote}/{base_branch}");
500 let fetched = git::fetch(repo, remote, base_branch).await;
501 if let Ok(out) = &fetched
502 && out.ok()
503 && git::rev_exists(repo, &tracking).await
504 {
505 return Ok(crate::run::FetchedBase {
506 remote: remote.to_owned(),
507 branch: base_branch.to_owned(),
508 commit: git::rev_parse(repo, &tracking).await?,
509 fetched_at: jiff::Timestamp::now(),
510 });
511 }
512 let why = match &fetched {
513 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
514 Ok(_) => format!("{remote} has no {base_branch}"),
515 Err(e) => e.to_string(),
516 };
517 bail!(
520 "cannot read {tracking} ({why}); refusing to branch off the local \
521 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
522 base / remote in magi.toml. This is launcher preparation: check the \
523 launcher's Git administration access, network and repository coordination \
524 before retrying; an unattended seat cannot approve those permissions"
525 )
526}
527
528struct FixClaim {
545 path: PathBuf,
546}
547
548impl FixClaim {
549 fn acquire(dir: &Path) -> Result<Self> {
550 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
551 let path = dir.join("fix.lock");
552 match Self::create(&path) {
553 Ok(claim) => Ok(claim),
554 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
555 if Self::reclaim_if_dead(&path) {
556 Self::create(&path).with_context(|| format!("lock {}", path.display()))
557 } else {
558 bail!(
559 "another repair or `magi fix` is already running for this run ({} exists)",
560 path.display()
561 )
562 }
563 }
564 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
565 }
566 }
567
568 fn create(path: &Path) -> std::io::Result<Self> {
569 let mut f = std::fs::OpenOptions::new()
570 .write(true)
571 .create_new(true)
572 .open(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
627 repo: &Path,
628 instruction: String,
629 also_scan: &str,
630 config: Config,
631 origin: Origin,
632 ) -> Result<Self> {
633 let repo = git::toplevel(repo).await?;
634 let missing = agent::missing_programs(&config.agents);
635 if !missing.is_empty() {
636 bail!(
637 "these agent programs are not on PATH: {}. Fix the roster in \
638 magi.toml or install them.",
639 missing.join(", ")
640 );
641 }
642 let base_branch =
643 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
644 .await?;
645 let fetched_base = resolve_base(
646 &repo,
647 &base_branch,
648 &config.merge.remote,
649 config.coordination.as_ref(),
650 )
651 .await?;
652 let base_commit = fetched_base.commit.clone();
653 let roles = config.resolve_roles()?;
654 let max_parallel = config.graph.max_parallel.max(1);
655 let seeds = refs::resolve(
658 &repo,
659 &base_commit,
660 &config.merge.remote,
661 &format!("{also_scan}\n{instruction}"),
662 )
663 .await;
664 refs::plan(&repo, &seeds).await?;
665 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
666 state.fetched_base = Some(fetched_base);
670 state.origin = Some(origin);
671 for seed in &seeds {
672 state.event(
673 "seed",
674 refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
675 );
676 }
677 state.seeds = seeds;
678 state.event("start", format!("run {} created", state.id));
679 state.save()?;
680 Ok(Self {
681 state,
682 roles,
683 sem: Arc::new(Semaphore::new(max_parallel)),
684 pause: Pause::new(),
685 interrupt: Pause::new(),
686 })
687 }
688
689 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
703 Self::review_taking_over(repo, branch, config, None, origin).await
704 }
705
706 pub async fn review_taking_over(
713 repo: &Path,
714 branch: &str,
715 config: Config,
716 takeover: Option<crate::handover::Takeover>,
717 origin: Origin,
718 ) -> Result<Self> {
719 let repo = git::toplevel(repo).await?;
720 let missing = agent::missing_programs(&config.agents);
721 if !missing.is_empty() {
722 bail!(
723 "these agent programs are not on PATH: {}. Fix the roster in \
724 magi.toml or install them.",
725 missing.join(", ")
726 );
727 }
728 let base_branch =
729 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
730 .await?;
731 if base_branch == branch {
732 bail!("`{branch}` is the base branch; there is nothing to review against");
733 }
734 let fetched_base = resolve_base(
735 &repo,
736 &base_branch,
737 &config.merge.remote,
738 config.coordination.as_ref(),
739 )
740 .await?;
741 let base_commit = fetched_base.commit.clone();
742
743 let roles = config.resolve_roles()?;
744 let max_parallel = config.graph.max_parallel.max(1);
745 let mut state = RunState::new(
746 repo.clone(),
747 base_branch,
748 base_commit.clone(),
749 String::new(),
750 config,
751 );
752 state.fetched_base = Some(fetched_base);
753 state.origin = Some(origin);
754
755 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
762 earlier: Vec::new(),
763 home: crate::run::home(),
764 choice: None,
765 task_instruction: None,
766 });
767 let wait = repository_wait(&state, None);
768 let coordination_config = state.config.coordination.clone();
769 let coordination =
770 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
771 .await?;
772 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
773 if let Some(released) = &released {
774 state.event(
775 "release",
776 format!(
777 "took `{branch}` over from run {}: its worktree was released: {}",
778 crate::run::short_of(&released.old_id),
779 released.audit
780 ),
781 );
782 }
783 if let Some(choice) = takeover.choice.as_ref()
787 && let Err(e) =
788 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
789 .await
790 {
791 if let Some(released) = &released {
792 released.restore(&repo, branch).await;
793 }
794 return Err(e.context("applying the owner's answer about the diverged branch"));
795 }
796 for id in takeover.earlier.iter().rev() {
802 if let Ok(old) = RunState::load_under(id, &takeover.home)
803 && let Some(head) = old.pr_head.clone()
804 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
805 {
806 state.pr_head = Some(head);
807 break;
808 }
809 }
810 drop(coordination);
811 let opened = Self::open_review(
812 &repo,
813 branch,
814 state,
815 roles,
816 max_parallel,
817 base_commit,
818 &takeover,
819 )
820 .await;
821 if opened.is_err()
822 && let Some(released) = &released
823 {
824 let _coordination = crate::repo_lock::SeatLease::operation_with(
825 &repo,
826 coordination_config.as_ref(),
827 &wait,
828 )
829 .await?;
830 released.restore(&repo, branch).await;
831 }
832 opened
833 }
834
835 pub async fn recover_pr(
837 repo: &Path,
838 task: &str,
839 selection: crate::pr_recovery::Selection,
840 config: Config,
841 ) -> Result<Self> {
842 selection.check_task(task, &selection.instruction)?;
843 let base_branch =
844 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
845 .await?;
846 let fetched_base = resolve_base(
847 repo,
848 &base_branch,
849 &config.merge.remote,
850 config.coordination.as_ref(),
851 )
852 .await?;
853 let base_commit = fetched_base.commit.clone();
854 selection
855 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
856 .await?;
857 let roles = config.resolve_roles()?;
858 let max_parallel = config.graph.max_parallel.max(1);
859 let mut state = RunState::new(
860 repo.to_path_buf(),
861 base_branch,
862 base_commit.clone(),
863 selection.instruction.clone(),
864 config,
865 );
866 state.fetched_base = Some(fetched_base);
867 state.origin = Some(crate::run::Origin::queue(task));
868 state.pr_head = Some(selection.branch.clone());
869 state.pr_recovery = Some(selection.clone());
870 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
871 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
872 }
873
874 async fn open_selected(
875 repo: &Path,
876 state: RunState,
877 roles: ResolvedRoles,
878 max_parallel: usize,
879 base_commit: String,
880 selection: crate::pr_recovery::Selection,
881 ) -> Result<Self> {
882 let mut state = state;
883 let branch = state.branch_for('A');
884 let wait = repository_wait(&state, None);
885 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
886 git::git(repo, &["branch", &branch, &selection.head]).await?;
887 drop(coordination);
888 state.event(
889 "recovery",
890 format!(
891 "selected public implementation {} from {}",
892 selection.head, selection.url
893 ),
894 );
895 let takeover = crate::handover::Takeover {
896 earlier: Vec::new(),
897 home: crate::run::home(),
898 choice: None,
899 task_instruction: Some(selection.instruction.clone()),
900 };
901 state.save()?;
902 let id = state.id.clone();
903 let opened = Self::open_review(
904 repo,
905 &branch,
906 state,
907 roles,
908 max_parallel,
909 base_commit,
910 &takeover,
911 )
912 .await;
913 match opened {
914 Ok(runner) => Ok(runner),
915 Err(e) => {
916 if let Ok(mut partial) = RunState::load(&id) {
917 partial.status = RunStatus::Blocked;
918 partial.driver_exited = true;
919 partial.event(
920 "recovery",
921 format!("startup failed; retained source and candidate {branch}: {e:#}"),
922 );
923 partial.save()?;
924 }
925 Err(crate::pr_recovery::StartupFailed {
926 run: id,
927 cause: format!("{e:#}"),
928 }
929 .into())
930 }
931 }
932 }
933
934 async fn inherit(
939 repo: &Path,
940 branch: &str,
941 base_commit: &str,
942 takeover: &crate::handover::Takeover,
943 ) -> Option<crate::recovery::RecoveryRecord> {
944 use crate::recovery::{Relation, assess};
945 let newest = takeover.earlier.last()?;
946 let mut chosen = None;
947 for id in takeover.earlier.iter().rev() {
948 if let Ok(old) = RunState::load_under(id, &takeover.home)
949 && !old.reviews.is_empty()
950 && (old.winner().is_some_and(|w| w.branch == branch)
951 || old.pr_head.as_deref() == Some(branch))
952 {
953 chosen = Some(old);
954 break;
955 }
956 }
957 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
958 .await
959 .unwrap_or_default();
960 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
961 Some(last) if last.head == tip => Relation::Same,
962 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
963 _ => Relation::Diverged,
964 };
965 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
966 Some(assess(from, chosen.as_ref(), base_commit, relation))
967 }
968
969 async fn open_review(
972 repo: &Path,
973 branch: &str,
974 mut state: RunState,
975 roles: ResolvedRoles,
976 max_parallel: usize,
977 base_commit: String,
978 takeover: &crate::handover::Takeover,
979 ) -> Result<Self> {
980 let wait = repository_wait(&state, None);
981 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
982 if state.pr_recovery.is_none() {
983 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
984 }
985 let start = review_base(
988 repo,
989 &state.config.merge.remote,
990 &state.base_branch,
991 &base_commit,
992 branch,
993 )
994 .await;
995 let log = git::log_oneline(repo, &start, branch)
996 .await
997 .unwrap_or_default();
998 let log_text = if log.trim().is_empty() {
999 "(no commit messages)"
1000 } else {
1001 log.trim()
1002 };
1003 let mut instruction = format!(
1004 "Review the work already on branch `{branch}`. There is no task \
1005 statement: what the change claims to do is whatever its commits \
1006 say.\n\n{log_text}"
1007 );
1008 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1012 if let Some(task) = takeover
1013 .task_instruction
1014 .as_deref()
1015 .filter(|t| !t.trim().is_empty())
1016 && rec.reviewed_head.is_some()
1017 {
1018 instruction = format!(
1019 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1020 task.trim()
1021 );
1022 rec.criteria_inherited = true;
1023 }
1024 state.event("recovery", rec.summary());
1025 state.recovery = Some(rec);
1026 }
1027 if let Some(selected) = &state.pr_recovery {
1028 instruction = selected.instruction.clone();
1029 }
1030 state.task_instruction = takeover.task_instruction.clone();
1031 state.instruction = instruction;
1032 state.reviewed_commits = Some(
1033 git::subjects(repo, &start, branch)
1034 .await
1035 .unwrap_or_default(),
1036 );
1037
1038 let worktree = state.worktree_root().join("under-review");
1041 if let Some(parent) = worktree.parent() {
1042 tokio::fs::create_dir_all(parent).await.ok();
1043 }
1044 let path = worktree.to_string_lossy().to_string();
1045 git::git(repo, &["worktree", "add", &path, branch])
1046 .await
1047 .with_context(|| {
1048 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1049 })?;
1050
1051 let setup = match ensure_setup_config(&mut state, repo).await {
1052 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1053 Err(e) => Err(e),
1054 };
1055 if let Err(e) = setup {
1056 git::worktree_remove_clean(repo, &worktree).await.ok();
1057 return Err(e);
1058 }
1059 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1060 .await
1061 .unwrap_or(0);
1062 if commits == 0 {
1063 git::worktree_remove_clean(repo, &worktree).await.ok();
1064 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1065 }
1066 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1067 .await
1068 .map(|f| f.len())
1069 .unwrap_or(0);
1070 if files == 0
1071 && let (Ok(head_tree), Ok(base_tree)) = (
1072 git::tree_of(&worktree, "HEAD").await,
1073 git::tree_of(&worktree, &base_commit).await,
1074 )
1075 && head_tree == base_tree
1076 {
1077 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1078 git::worktree_remove_clean(repo, &worktree).await.ok();
1079 bail!(
1080 "`{branch}` at {} has a tree identical to base {}; this usually means \
1081 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1082 against `{}/{branch}`) rather than an empty change",
1083 short(&head),
1084 short(&base_commit),
1085 state.config.merge.remote
1086 );
1087 }
1088 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1089 .await
1090 .unwrap_or_default();
1091
1092 state.candidates.push(Candidate {
1093 index: 0,
1094 label: 'A',
1095 agent: EXISTING_BRANCH.to_owned(),
1098 branch: branch.to_owned(),
1099 worktree,
1100 summary: String::new(),
1101 stat,
1102 files,
1103 commits,
1104 empty: false,
1105 failed: None,
1106 verified_noop: None,
1107 duration_ms: 0,
1108 folded: false,
1109 });
1110 state.tally = Some(Tally {
1111 first_choice: BTreeMap::from([('A', 0)]),
1112 borda: BTreeMap::new(),
1113 winner: 'A',
1114 rankings: 0,
1115 unanimous_initial: false,
1116 deliberated: false,
1117 changed_votes: 0,
1118 unanimous_final: false,
1119 tie_break: None,
1120 judges: 0,
1124 present: 0,
1125 quorum: 0,
1126 met_quorum: true,
1127 uncontested: Some("review-only run: nothing competed".to_owned()),
1128 });
1129 state.status = RunStatus::Reviewing;
1130 state.prep_complete = Some(true);
1131 state.event(
1132 "start",
1133 format!(
1134 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1135 state.id
1136 ),
1137 );
1138 state.save()?;
1139 Ok(Self {
1140 state,
1141 roles,
1142 sem: Arc::new(Semaphore::new(max_parallel)),
1143 pause: Pause::new(),
1144 interrupt: Pause::new(),
1145 })
1146 }
1147
1148 pub fn resume(id: &str) -> Result<Self> {
1150 let state = RunState::load(id)?;
1151 crate::pr_head::ensure_owner(&state)?;
1152 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1153 bail!(
1154 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1155 );
1156 }
1157 if let Some(to) = &state.released_to {
1158 bail!(
1159 "run {} cannot be resumed: its worktree was released to run {}",
1160 state.short(),
1161 crate::run::short_of(to)
1162 );
1163 }
1164 let roles = state.config.resolve_roles()?;
1165 if let Some(id) = &state.config.graph.selected_implementer
1166 && state.candidates.iter().any(|c| c.agent != *id)
1167 {
1168 bail!(
1169 "replacement implementer pin does not match the saved candidates; task must stay held"
1170 );
1171 }
1172 let max_parallel = state.config.graph.max_parallel.max(1);
1173 Ok(Self {
1174 state,
1175 roles,
1176 sem: Arc::new(Semaphore::new(max_parallel)),
1177 pause: Pause::new(),
1178 interrupt: Pause::new(),
1179 })
1180 }
1181
1182 pub async fn execute(&mut self) -> Result<()> {
1189 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1192 || self.state.land_approval.is_some()
1193 || self.state.merge_decision.is_some()
1194 || !self.state.merge_repairs.is_empty()
1195 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1196 {
1197 let claim = FixClaim::acquire(&self.state.dir())?;
1198 let disk = RunState::load(&self.state.id)?;
1199 let checkpoint = |s: &RunState| {
1200 (
1201 s.merge_repairs.len(),
1202 s.merge_repairs
1203 .last()
1204 .map(|r| (r.question.clone(), r.phase)),
1205 s.land_approval
1206 .as_ref()
1207 .map(|a| (a.question.clone(), a.head.clone())),
1208 )
1209 };
1210 if checkpoint(&disk) != checkpoint(&self.state) {
1211 bail!(
1212 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1213 );
1214 }
1215 Some(claim)
1216 } else {
1217 None
1218 };
1219 let mut execution = self.claim_driver()?;
1220 let runner = &mut *execution.runner;
1221 let wait = runner.repository_wait();
1222 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1225 let result = match result {
1226 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1227 record_repository_block(&mut runner.state, &e);
1228 runner.park_here()?;
1229 Ok(())
1230 }
1231 other => other,
1232 };
1233 execution.driver.finish(&mut runner.state)?;
1234 let ended = if result.is_err() {
1235 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1236 } else {
1237 crate::notices::run_ended(&runner.state)
1238 };
1239 if let Some(notice) = ended {
1240 crate::notices::raise(notice);
1241 }
1242 result
1243 }
1244
1245 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1246 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1247 }
1248
1249 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1250 let wait = self.repository_wait();
1251 coordinate_operation(&mut self.state, cwd, &wait).await
1252 }
1253
1254 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1256 let owner = crate::pr_head::owner_lock(&crate::run::home(), &self.state.id)?;
1257 let disk = RunState::load(&self.state.id)?;
1258 crate::pr_head::ensure_owner(&disk)?;
1259 if disk.liveness(false) == crate::run::Liveness::Live {
1260 bail!("run already has a live driver; wait for its execution to end");
1261 }
1262 if let Some(to) = &disk.released_to {
1263 bail!(
1264 "run {} cannot continue: its worktree was released to run {}",
1265 self.state.short(),
1266 crate::run::short_of(to)
1267 );
1268 }
1269 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1270 let mut execution = DriverExecution {
1271 runner: self,
1272 driver,
1273 };
1274 let _owner = owner;
1277 let state = &mut execution.runner.state;
1278 state.parked = false;
1283 state.resource_wait = None;
1284 state.resource_block = None;
1285 state.clear_active();
1292 let pid = std::process::id();
1307 state.driver_pid = Some(pid);
1308 state.driver_started_at = crate::proc::process_started_at(pid);
1309 state.driver_exited = false;
1310 state.begin_pass();
1311 execution.driver.bind(state);
1312 state.save()?;
1313 execution.driver.record(state)?;
1314 Ok(execution)
1315 }
1316
1317 async fn execute_graph(&mut self) -> Result<()> {
1318 self.persist_candidates().await?;
1321 if self
1322 .state
1323 .merge_decision
1324 .as_ref()
1325 .is_some_and(|decision| !decision.closed)
1326 && self.state.status != RunStatus::Landing
1327 {
1328 if crate::merge_decision::resumable_hold(&self.state) {
1329 self.state.merge = Some(MergeOutcome {
1330 mode: MergeMode::Pr,
1331 ok: false,
1332 detail: self
1333 .state
1334 .merge_decision
1335 .as_ref()
1336 .unwrap()
1337 .target
1338 .pr
1339 .clone(),
1340 empty: false,
1341 });
1342 self.run_land().await?;
1343 self.settle_questions();
1344 return Ok(());
1345 }
1346 bail!(
1347 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1348 );
1349 }
1350 if self.state.config.graph.selected_implementer.is_some()
1351 && self.state.status == RunStatus::Blocked
1352 && self.state.tally.is_none()
1353 && self.state.candidate_preservation.is_empty()
1354 {
1355 bail!(
1356 "selected implementation is held; explicitly release the owning task to retry it"
1357 );
1358 }
1359
1360 if self.state.merge_repairs.last().is_some_and(|r| {
1361 matches!(
1362 r.phase,
1363 crate::run::MergeRepairPhase::Requested
1364 | crate::run::MergeRepairPhase::Fixing
1365 | crate::run::MergeRepairPhase::Reviewing
1366 )
1367 }) {
1368 if self.repair_merge_review().await? {
1369 self.run_land().await?;
1370 }
1371 self.settle_questions();
1372 return Ok(());
1373 }
1374 if self.state.pr_publication_pending {
1377 self.merge().await?;
1378 if self.resource_halt()? {
1379 return Ok(());
1380 }
1381 self.state.save()?;
1382 self.settle_questions();
1383 return Ok(());
1384 }
1385 if self.state.status == RunStatus::Stalled {
1398 if self.recover_stall().await? {
1399 self.finish_after_tally().await?;
1400 } else {
1401 self.state.save()?;
1403 }
1404 return Ok(());
1405 }
1406 if self.state.status == RunStatus::Landing {
1416 self.run_land().await?;
1417 self.settle_questions();
1422 return Ok(());
1423 }
1424 if self.state.github_text.is_some()
1429 && self.state.merge.is_none()
1430 && self.state.status == RunStatus::Gating
1431 {
1432 self.merge().await?;
1433 if self.resource_halt()? {
1434 return Ok(());
1435 }
1436 self.state.save()?;
1437 return Ok(());
1438 }
1439 if let Err(e) = self.prep().await {
1440 if e.is::<crate::repo_lock::ResourceBlocked>() {
1441 return Err(e);
1442 }
1443 self.state.status = RunStatus::Blocked;
1444 self.state
1445 .prep_failure
1446 .get_or_insert_with(|| format!("{e:#}"));
1447 self.state
1448 .event("prep", format!("preparation stopped: {e:#}"));
1449 self.state.save()?;
1450 return Err(e);
1451 }
1452 if self.park_here()? {
1453 return Ok(());
1454 }
1455 self.advise().await?;
1456 if self.park_here()? {
1457 return Ok(());
1458 }
1459 self.implement().await?;
1460 if self.park_here()? {
1461 return Ok(());
1462 }
1463 if self.state.status == RunStatus::VerifiedNoop {
1467 return Ok(());
1468 }
1469 self.judge().await?;
1470 if self.park_here()? {
1471 return Ok(());
1472 }
1473 self.deliberate().await?;
1474 if self.park_here()? {
1475 return Ok(());
1476 }
1477 self.vote().await?;
1478 if self.park_here()? {
1479 return Ok(());
1480 }
1481 self.tally()?;
1482 if self.state.status == RunStatus::Stalled {
1487 self.state.save()?;
1491 return Ok(());
1492 }
1493 self.finish_after_tally().await?;
1494 Ok(())
1495 }
1496
1497 fn stop_for_resource(&mut self) -> Result<()> {
1502 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1504 {
1505 self.state.status = RunStatus::Blocked;
1506 }
1507 self.state.save()
1508 }
1509
1510 fn resource_halt(&mut self) -> Result<bool> {
1512 if self.state.resource_block.is_none() {
1513 return Ok(false);
1514 }
1515 self.stop_for_resource()?;
1516 Ok(true)
1517 }
1518
1519 fn park_here(&mut self) -> Result<bool> {
1526 let parking = self.pause.parked() || self.interrupt.parked();
1527 if !parking && self.state.resource_block.is_some() {
1528 self.stop_for_resource()?;
1529 return Ok(true);
1530 }
1531 if !self.pause.parked() && !self.interrupt.parked() {
1536 return Ok(false);
1537 }
1538 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1539 let why = match cause.as_ref().map(ParkCause::label) {
1540 Some(reason) => format!(
1541 "parked after `{}` ({reason}) — resume to carry on from here",
1542 self.state.status.as_str()
1543 ),
1544 None => format!(
1545 "parked after `{}` — resume to carry on from here",
1546 self.state.status.as_str()
1547 ),
1548 };
1549 self.state.event("park", why);
1550 self.state
1551 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1552 self.state.parked = true;
1553 self.state.save()?;
1554 Ok(true)
1555 }
1556
1557 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1559 if self.state.prep_failure.is_none() {
1560 return Ok(());
1561 }
1562 let failed_at = self
1563 .state
1564 .events
1565 .iter()
1566 .rev()
1567 .find(|event| event.node == "prep")
1568 .map_or(self.state.updated_at, |event| event.at);
1569 anyhow::ensure!(
1570 approval_at > failed_at,
1571 "preparation retry requires a new explicit release after the failure"
1572 );
1573 self.state.prep_failure = None;
1574 self.state.prep_complete = Some(false);
1575 self.state
1576 .event("prep-retry", "operator approved retry of preparation");
1577 Ok(())
1578 }
1579
1580 pub fn on_pause(&mut self, pause: Pause) {
1582 self.pause = pause;
1583 }
1584
1585 pub fn watch_interrupt(&mut self, pause: Pause) {
1591 self.interrupt = pause;
1592 }
1593
1594 fn settle_questions(&mut self) {
1614 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1615 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1616 }
1617 }
1618
1619 async fn finish_after_tally(&mut self) -> Result<()> {
1622 self.graft().await?;
1625 if self.resource_halt()? {
1626 return Ok(());
1627 }
1628 self.fold_losers().await?;
1629 self.sync_to_base().await?;
1634 if self.state.status == RunStatus::AlreadyInBase {
1635 return Ok(());
1636 }
1637 self.review_loop().await?;
1638 if self.resource_halt()? {
1639 return Ok(());
1640 }
1641 self.sync_to_base().await?;
1642 if self.state.status == RunStatus::AlreadyInBase {
1643 return Ok(());
1644 }
1645 self.gate().await?;
1646 self.merge().await?;
1647 if self.resource_halt()? {
1648 return Ok(());
1649 }
1650 self.state.save()?;
1651 Ok(())
1652 }
1653
1654 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1661 let result = match ensure_setup_config(&mut self.state, repo).await {
1662 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1663 Err(e) => Err(e),
1664 };
1665 if let Err(e) = result {
1666 git::worktree_remove_clean(repo, wt).await.ok();
1667 if in_prep {
1668 self.rollback_prep(repo).await;
1669 }
1670 self.state.status = RunStatus::Blocked;
1671 self.state.event(
1672 "setup",
1673 format!("worktree setup failed in {}: {e:#}", wt.display()),
1674 );
1675 self.state.save()?;
1676 return Err(e);
1677 }
1678 Ok(())
1679 }
1680
1681 async fn rollback_prep(&mut self, repo: &Path) {
1683 let root = self.state.worktree_root();
1684 for c in std::mem::take(&mut self.state.candidates) {
1685 if matches!(
1686 git::worktree_remove_clean(repo, &c.worktree).await,
1687 Ok(true)
1688 ) {
1689 git::branch_delete(repo, &c.branch).await.ok();
1690 } else {
1691 self.state.candidates.push(c);
1692 }
1693 }
1694 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1695 for j in 1..=seats {
1696 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1697 .await
1698 .ok();
1699 }
1700 for k in 1..=self.state.config.graph.advisors {
1701 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1702 .await
1703 .ok();
1704 }
1705 }
1706
1707 async fn prep(&mut self) -> Result<()> {
1708 if let Some(reason) = &self.state.prep_failure {
1709 bail!(
1710 "candidate preparation previously failed: {reason}; correct the source before retrying"
1711 );
1712 }
1713 if self
1714 .state
1715 .prep_complete
1716 .unwrap_or(!self.state.candidates.is_empty())
1717 {
1718 return Ok(());
1719 }
1720 let repo = self.state.repo.clone();
1721 let _coordination = self.coordinate(&repo).await?;
1724 self.rollback_prep(&repo).await;
1725 if !self.state.candidates.is_empty() {
1726 bail!(
1727 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1728 );
1729 }
1730 self.state.prep_complete = Some(false);
1731 self.state.status = RunStatus::Prep;
1732 self.state.save()?;
1733 let repo = self.state.repo.clone();
1734 let base = self.state.base_commit.clone();
1735 let plan = refs::plan(&repo, &self.state.seeds).await?;
1736 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1737 let root = self.state.worktree_root();
1738 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1739
1740 let hooks_dir = self.state.dir().join("hooks");
1743 if self.state.config.blind.commit_msg_hook {
1744 std::fs::create_dir_all(&hooks_dir)
1745 .with_context(|| format!("create {}", hooks_dir.display()))?;
1746 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1747 let path = hooks_dir.join("commit-msg");
1748 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1749 make_executable(&path)?;
1750 git::acquire_worktree_config(&repo).await?;
1758 self.state.enabled_worktree_config = true;
1759 }
1760
1761 ensure_setup_config(&mut self.state, &repo).await?;
1764
1765 for (index, (spec, label)) in self
1766 .roles
1767 .implementers
1768 .clone()
1769 .into_iter()
1770 .zip(labels)
1771 .enumerate()
1772 {
1773 let branch = self.state.branch_for(label);
1774 let worktree = root.join(format!("cand-{label}"));
1775 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1776 if self.state.config.blind.commit_msg_hook {
1777 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1778 }
1779 git::local_exclude(&worktree, "/.magi/").await?;
1780 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1781 git::worktree_remove_clean(&repo, &worktree).await.ok();
1784 git::branch_delete(&repo, &branch).await.ok();
1785 self.rollback_prep(&repo).await;
1786 self.state.status = RunStatus::Blocked;
1787 self.state
1788 .event("prep", format!("worktree setup failed: {e:#}"));
1789 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1790 self.state.save()?;
1791 return Err(e);
1792 }
1793 self.state.candidates.push(Candidate {
1794 index,
1795 label,
1796 agent: spec.id.clone(),
1797 branch,
1798 worktree,
1799 summary: String::new(),
1800 stat: String::new(),
1801 files: 0,
1802 commits: 0,
1803 empty: false,
1804 failed: None,
1805 verified_noop: None,
1806 duration_ms: 0,
1807 folded: false,
1808 });
1809 self.state.save()?;
1810 for pick in &plan.picks {
1811 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1812 {
1813 self.state.status = RunStatus::Blocked;
1814 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1815 self.state
1816 .event("prep", format!("cannot apply referenced commit: {e}"));
1817 self.state.save()?;
1818 return Err(e);
1819 }
1820 }
1821 }
1822
1823 for j in 1..=self.roles.judges.len() {
1824 let wt = root.join(format!("judge-{j}"));
1825 if !wt.exists() {
1826 git::worktree_add_detached(&repo, &wt, &base).await?;
1827 self.setup_seat_worktree(&repo, &wt, true).await?;
1828 }
1829 }
1830
1831 if self.state.config.graph.advise {
1839 for k in 1..=self.state.config.graph.advisors {
1840 let wt = root.join(format!("advisor-{k}"));
1841 if !wt.exists() {
1842 git::worktree_add_detached(&repo, &wt, &base).await?;
1843 self.setup_seat_worktree(&repo, &wt, true).await?;
1844 }
1845 }
1846 }
1847
1848 let authors: Vec<&str> = self
1853 .roles
1854 .implementers
1855 .iter()
1856 .map(|a| a.id.as_str())
1857 .collect();
1858 let overlap: Vec<String> = self
1859 .roles
1860 .judges
1861 .iter()
1862 .enumerate()
1863 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1864 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1865 .collect();
1866 if !overlap.is_empty() {
1867 let note = format!(
1868 "{} also authored a candidate; blind, but the panel is less \
1869 independent than {} distinct agents would be",
1870 overlap.join(", "),
1871 self.roles.judges.len()
1872 );
1873 self.state.event("prep", note);
1874 }
1875
1876 self.state.event(
1877 "prep",
1878 format!(
1879 "{} candidates, {} judges, base {} ({})",
1880 self.state.candidates.len(),
1881 self.roles.judges.len(),
1882 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1883 self.state.base_branch
1884 ),
1885 );
1886 self.state.prep_complete = Some(true);
1887 self.state.status = RunStatus::Implementing;
1888 self.state.save()?;
1889 Ok(())
1890 }
1891
1892 async fn advise(&mut self) -> Result<()> {
1925 let implement_untouched = self
1926 .state
1927 .candidates
1928 .iter()
1929 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1930 if !self.state.config.graph.advise || self.state.advise_attempted {
1931 return Ok(());
1932 }
1933 if !implement_untouched {
1934 self.state.event(
1935 "advise",
1936 "skipping the design-deliberation stage: at least one \
1937 candidate already shows implementation progress, so this \
1938 run is past the point the stage exists to run before"
1939 .to_owned(),
1940 );
1941 self.state.advise_attempted = true;
1942 self.state.save()?;
1943 return Ok(());
1944 }
1945 let run_id = self.state.id.clone();
1946 let prompts = self.state.config.prompts.clone();
1947 let instruction = self.state.instruction.clone();
1948 let language = self.state.config.graph.language.clone();
1949 let root = self.state.worktree_root();
1950 let n = self.state.config.graph.advisors;
1951 let where_recorded = self.state.dir().join("run.json");
1952
1953 let seats = match self.state.config.advisors() {
1954 Ok(seats) if !seats.is_empty() => seats,
1955 Ok(_) => {
1956 self.state.event(
1957 "advise",
1958 format!(
1959 "[graph] advisors is 0; skipping the design-deliberation \
1960 stage and continuing without a synthesis brief (see {})",
1961 where_recorded.display()
1962 ),
1963 );
1964 self.state.advise_attempted = true;
1965 self.state.save()?;
1966 return Ok(());
1967 }
1968 Err(e) => {
1969 self.state.event(
1970 "advise",
1971 format!(
1972 "could not resolve advisor seats ({e:#}); continuing \
1973 without a design-deliberation brief (see {})",
1974 where_recorded.display()
1975 ),
1976 );
1977 self.state.advise_attempted = true;
1978 self.state.save()?;
1979 return Ok(());
1980 }
1981 };
1982
1983 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1984 let artifacts = agent::artifacts_dir(&self.state.dir());
1985 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1986
1987 let mut jobs = Vec::new();
1988 for (i, spec) in seats.iter().cloned().enumerate() {
1989 let seat_key = format!("advisor-{}", i + 1);
1990 let seat = self.seat(&seat_key, &spec.id);
1991 jobs.push(SeatJob {
1992 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1993 spec,
1994 seat,
1995 cwd: worktrees[i % worktrees.len()].clone(),
1996 timeout,
1997 allow_write: false,
1998 sessions: false,
1999 artifacts: artifacts.clone(),
2000 stem: seat_key,
2001 handover: None,
2002 });
2003 }
2004
2005 self.state.event(
2006 "advise",
2007 format!(
2008 "{} advisor seat(s) sketching a design in parallel",
2009 jobs.len()
2010 ),
2011 );
2012 let mut quota_losses = Vec::new();
2013 let cache = self.state.config.cache_dir();
2014 let ctx = WaveCtx {
2015 carry_seats: false,
2016 pauses: Some((&self.pause, &self.interrupt)),
2017 run: &run_id,
2018 node: "advise",
2019 prompts: &prompts,
2020 cache: cache.as_deref(),
2021 round: None,
2022 };
2023 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2024 let results = ask_json_wave::<Proposal>(
2025 jobs,
2026 Arc::clone(&self.sem),
2027 self.state.config.graph.retries,
2028 &advisor_roster,
2029 &ctx,
2030 &mut quota_losses,
2031 &mut self.state,
2032 &|p: &Proposal| p.validate(),
2033 )
2034 .await;
2035 if self.state.resource_block.is_some() {
2036 return self.stop_for_resource();
2037 }
2038 self.state.quota.extend(quota_losses);
2039
2040 let mut records = Vec::with_capacity(results.len());
2041 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2042 let agent_id = seat.agent.clone();
2043 self.state.seats.insert(seat.key.clone(), seat);
2044 match res {
2045 Ok((proposal, out)) => {
2046 self.state
2047 .event("advise", format!("advisor-{} proposed a design", i + 1));
2048 records.push(advise::AdvisorRecord::proposed(
2049 i + 1,
2050 agent_id,
2051 proposal,
2052 out.duration_ms,
2053 ));
2054 }
2055 Err(e) => {
2056 self.state.event(
2057 "advise",
2058 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2059 );
2060 records.push(advise::AdvisorRecord::failed(
2061 i + 1,
2062 agent_id,
2063 e.to_string(),
2064 ));
2065 }
2066 }
2067 }
2068
2069 let mut advice = advise::Advice {
2070 records,
2071 synthesis: None,
2072 };
2073 if advice.proposals().is_empty() {
2074 self.state.event(
2075 "advise",
2076 "no advisor produced a usable proposal; continuing without a \
2077 synthesis brief"
2078 .to_owned(),
2079 );
2080 } else {
2081 match self
2082 .synthesize_brief(
2083 &advice,
2084 &instruction,
2085 &language,
2086 &worktrees[0],
2087 &artifacts,
2088 &run_id,
2089 &prompts,
2090 cache.as_deref(),
2091 )
2092 .await
2093 {
2094 Ok(Some(text)) => {
2095 self.state.event(
2096 "advise",
2097 "synthesized a design brief for the implementer".to_owned(),
2098 );
2099 advice.synthesis = Some(text);
2100 }
2101 Ok(None) => {
2102 self.state.event(
2103 "advise",
2104 "the synthesis seat produced nothing usable; continuing \
2105 without a design brief"
2106 .to_owned(),
2107 );
2108 }
2109 Err(e) => {
2110 if e.is::<crate::repo_lock::ResourceBlocked>() {
2111 return Err(e);
2112 }
2113 self.state.event(
2114 "advise",
2115 format!("could not synthesize a design brief: {e:#}"),
2116 );
2117 }
2118 }
2119 }
2120 advise::apply_reflection(&mut advice);
2121
2122 self.state.advice = Some(advice);
2123 self.state.advise_attempted = true;
2124 self.state.save()?;
2125 Ok(())
2126 }
2127
2128 #[allow(clippy::too_many_arguments)]
2140 async fn synthesize_brief(
2141 &mut self,
2142 advice: &advise::Advice,
2143 instruction: &str,
2144 language: &str,
2145 cwd: &Path,
2146 artifacts: &Path,
2147 run_id: &str,
2148 prompts: &Prompts,
2149 cache: Option<&Path>,
2150 ) -> Result<Option<String>> {
2151 let chain = agent::pick_chain(
2152 &self.state.config.agents,
2153 self.state.config.roles.synthesizer.as_ref(),
2154 &agent::installed,
2155 "synthesizer",
2156 )?;
2157 let proposals = advice.proposals();
2158 let mut prompt = prompt::with_overlay(
2159 prompt::synthesize_brief(instruction, &proposals, language),
2160 prompts.overlay("advise"),
2161 );
2162 if cache.is_some() {
2163 prompt.push('\n');
2168 prompt.push_str(&prompt::build_cache_note("advise", false));
2169 }
2170 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2171 let mut last = None;
2176 for (n, spec) in chain.iter().enumerate() {
2177 if n > 0 {
2178 self.state
2179 .event("advise", format!("synthesis falling back to {}", spec.id));
2180 }
2181 let mut seat = self.seat("advise-synthesis", &spec.id);
2182 let wait = self.repository_wait();
2183 let outcome = wait
2184 .scope(agent::invoke(
2185 spec,
2186 &mut seat,
2187 &Invocation {
2188 coordination: self.state.config.coordination.as_ref(),
2189 fetched_base: None,
2190 cwd,
2191 prompt: &prompt,
2192 timeout,
2193 allow_write: false,
2194 unsandboxed: false,
2195 sessions: false,
2196 artifacts,
2197 stem: &if n == 0 {
2198 "advise-synthesis".to_owned()
2199 } else {
2200 format!("advise-synthesis-{}", spec.id)
2201 },
2202 run: run_id,
2203 node: "advise",
2204 cache_dir: None,
2205 attachments: &[],
2206 writable: &[],
2207 },
2208 ))
2209 .await;
2210 if outcome.is_ok() {
2211 self.state.seats.insert(seat.key.clone(), seat);
2212 }
2213 let advance = agent::chain_advances(&outcome);
2214 last = Some(outcome);
2215 if !advance {
2216 break;
2217 }
2218 }
2219 let out = last.expect("a chain holds at least one agent")?;
2222 if !out.usable() {
2223 return Ok(None);
2224 }
2225 let text =
2226 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2227 Ok((!text.trim().is_empty()).then_some(text))
2228 }
2229
2230 async fn implement(&mut self) -> Result<()> {
2233 self.persist_candidates().await?;
2234 let run_id = self.state.id.clone();
2239 let prompts = self.state.config.prompts.clone();
2240 let todo: Vec<usize> = self
2241 .state
2242 .candidates
2243 .iter()
2244 .enumerate()
2245 .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
2246 .map(|(i, _)| i)
2247 .collect();
2248 if todo.is_empty() {
2249 return self.after_implement();
2250 }
2251 self.state.status = RunStatus::Implementing;
2252
2253 let language = self.state.config.graph.language.clone();
2254 let github_language = self.state.config.graph.github_language.clone();
2255 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2256 let sessions = self.state.config.graph.sessions;
2257 let artifacts = agent::artifacts_dir(&self.state.dir());
2258 let brief = self
2262 .state
2263 .advice
2264 .as_ref()
2265 .and_then(|a| a.synthesis.as_deref())
2266 .map(str::to_owned);
2267 let attachments = self.state.attachments.clone();
2268
2269 let mut jobs = Vec::new();
2270 for &i in &todo {
2271 let (index, label, worktree) = {
2272 let c = &self.state.candidates[i];
2273 (c.index, c.label, c.worktree.clone())
2274 };
2275 let spec = self.roles.implementers[index].clone();
2276 let seat_key = format!("impl-{label}");
2277 let seat = self.seat(&seat_key, &spec.id);
2278 let instruction = seeded_instruction(&self.state);
2279 jobs.push(SeatJob {
2280 spec,
2281 seat,
2282 prompt: prompt::implement_in(
2283 &instruction,
2284 &worktree.to_string_lossy(),
2285 &language,
2286 &github_language,
2287 brief.as_deref(),
2288 &attachments,
2289 ),
2290 cwd: worktree,
2291 timeout,
2292 allow_write: true,
2293 sessions,
2294 artifacts: artifacts.clone(),
2295 stem: format!("impl-{label}"),
2296 handover: None,
2297 });
2298 }
2299
2300 self.state.event(
2301 "implement",
2302 format!("{} candidates in parallel", jobs.len()),
2303 );
2304 let mut sent = jobs.clone();
2310 let cache = self.state.config.cache_dir();
2311 let ctx = WaveCtx {
2312 carry_seats: false,
2313 pauses: Some((&self.pause, &self.interrupt)),
2314 run: &run_id,
2315 node: "implement",
2316 prompts: &prompts,
2317 cache: cache.as_deref(),
2318 round: None,
2319 };
2320 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2321 if !wave_blocked(&results) {
2322 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2323 .await;
2324 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2325 .await?;
2326 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2327 .await;
2328 }
2329 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2330
2331 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2332 && results
2333 .iter()
2334 .any(|(_, _, out)| FailClass::of(out).is_some());
2335 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2336 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2337 continue;
2338 }
2339 let seat_key = seat.key.clone();
2340 let agent = seat.agent.clone();
2349 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2350 self.state.seats.insert(seat.key.clone(), seat);
2351 let label = self.state.candidates[i].label;
2352
2353 let (summary, duration, failed, verified_claim) = match out {
2354 AgentOutcome::Ok(o) => {
2355 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2356 let failed = (!o.usable()).then(|| {
2357 if o.timed_out {
2358 "agent timed out".to_owned()
2359 } else {
2360 format!(
2361 "agent exited with {}",
2362 crate::notices::exit_status(o.exit_code)
2363 )
2364 }
2365 });
2366 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2367 (text, o.duration_ms, failed, verified_claim)
2368 }
2369 AgentOutcome::Dropped(o) => {
2375 let why = o
2376 .dropped
2377 .as_ref()
2378 .map(|d| d.why.as_str())
2379 .unwrap_or("the CLI ended the stream without delivering its answer");
2380 (
2381 String::new(),
2382 o.duration_ms,
2383 Some(format!("the CLI dropped the stream ({why})")),
2384 None,
2385 )
2386 }
2387 AgentOutcome::Quota(o) => {
2388 self.state.quota.push(QuotaLoss {
2389 seat: seat_key,
2390 node: "implement".to_owned(),
2391 at: Timestamp::now(),
2392 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2393 });
2394 (
2395 String::new(),
2396 o.duration_ms,
2397 Some("rate limited (quota); produced no change".to_owned()),
2398 None,
2399 )
2400 }
2401 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2402 (String::new(), 0, Some(e), None)
2403 }
2404 };
2405
2406 let c = &mut self.state.candidates[i];
2409 if !exhausted_the_fallback_chain {
2410 c.agent = agent;
2411 }
2412 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2413 c.duration_ms = duration;
2414 c.failed = failed;
2415 c.verified_noop = verified_claim;
2416 self.state
2417 .candidate_preservation
2418 .insert(label, "awaiting persistence".to_owned());
2419 }
2420 self.state.save()?;
2421 self.persist_candidates().await?;
2422
2423 if resource_blocked {
2424 return self.stop_for_resource();
2425 }
2426 if selected_failed {
2427 self.state.status = RunStatus::Blocked;
2428 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2429 self.state.save()?;
2430 bail!(
2431 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2432 );
2433 }
2434 self.after_implement()
2435 }
2436
2437 async fn persist_candidates(&mut self) -> Result<()> {
2439 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2440 for label in labels {
2441 let i = self
2442 .state
2443 .candidates
2444 .iter()
2445 .position(|c| c.label == label)
2446 .context("pending candidate is missing")?;
2447 let c = self.state.candidates[i].clone();
2448 let result = async {
2449 let wait = crate::repo_lock::WaitPolicy::new(
2450 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2451 || false,
2452 );
2453 let _coordination =
2454 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2455 if let Err(e) = git::snapshot_uncommitted(
2458 &c.worktree,
2459 &self
2460 .state
2461 .dir()
2462 .join("artifacts")
2463 .join(format!("candidate-{label}-recovery")),
2464 )
2465 .await
2466 {
2467 self.state.event(
2468 "preserve",
2469 format!("candidate {label} snapshot incomplete: {e:#}"),
2470 );
2471 }
2472 let rescued = git::rescue_commit(
2473 &c.worktree,
2474 &format!("magi: candidate {label} (uncommitted work)"),
2475 )
2476 .await?;
2477 self.state.note_withheld("implement", &rescued.withheld);
2478 let base = &self.state.base_commit;
2479 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2480 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2481 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2482 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2483 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2484 if c.empty && commits > 0 && !patch.trim().is_empty()
2485 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2486 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2487 || self.state.tally.is_some())
2488 {
2489 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2493 }
2494 let c = &mut self.state.candidates[i];
2495 c.commits = commits;
2496 c.stat = stat;
2497 c.files = files;
2498 c.empty = commits == 0 || patch.trim().is_empty();
2499 if !c.empty {
2500 c.failed = None;
2501 c.verified_noop = None;
2502 }
2503 self.state.event(
2504 "preserve",
2505 format!("candidate {label}: {files} files, {commits} commits"),
2506 );
2507 Ok::<_, anyhow::Error>(())
2508 }
2509 .await;
2510 match result {
2511 Ok(()) => {
2512 self.state.candidate_preservation.remove(&label);
2513 }
2514 Err(e) => {
2515 let reason = format!("candidate {label} preservation failed: {e:#}");
2516 self.state.event("preserve", &reason);
2517 self.state.candidate_preservation.insert(label, reason);
2518 }
2519 }
2520 if self.state.candidate_preservation.is_empty()
2521 && self.state.status == RunStatus::Blocked
2522 && self.state.candidates.iter().all(|c| c.failed.is_none())
2523 {
2524 self.state.status = RunStatus::Implementing;
2528 self.state.event(
2529 "preserve",
2530 "candidate persistence recovered; implementation can continue",
2531 );
2532 }
2533 self.state.save()?;
2534 }
2535 if !self.state.candidate_preservation.is_empty() {
2536 self.state.status = RunStatus::Blocked;
2537 self.state.save()?;
2538 bail!(
2539 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2540 );
2541 }
2542 Ok(())
2543 }
2544
2545 async fn resume_undelivered(
2573 &mut self,
2574 results: &mut [(usize, SeatState, AgentOutcome)],
2575 sent: &[SeatJob],
2576 prompts: &Prompts,
2577 run_id: &str,
2578 ) {
2579 for (wi, seat, out) in results.iter_mut() {
2580 let Some(dropped) = (match &*out {
2581 AgentOutcome::Dropped(o) => o.dropped.clone(),
2582 _ => None,
2583 }) else {
2584 continue;
2585 };
2586 let Some(job) = sent.get(*wi) else { continue };
2587 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2589 self.state.event(
2590 "implement",
2591 format!(
2592 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2593 work is in the tree",
2594 seat.key, dropped.output_tokens, dropped.why
2595 ),
2596 );
2597 continue;
2598 }
2599 if !has_context(&job.spec, seat, job.sessions) {
2607 self.state.event(
2608 "implement",
2609 format!(
2610 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2611 is no session left to resume",
2612 seat.key, dropped.output_tokens, dropped.why
2613 ),
2614 );
2615 continue;
2616 }
2617 self.state.event(
2618 "implement",
2619 format!(
2620 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2621 conversation",
2622 seat.key, dropped.output_tokens, dropped.why
2623 ),
2624 );
2625 let mut retry = job.clone();
2626 retry.seat = seat.clone();
2627 retry.prompt = prompt::resume_after_drop(&dropped.why);
2628 retry.timeout = retry_budget(job.timeout, true);
2629 retry.stem = format!("{}-resume", job.stem);
2630 let cache = self.state.config.cache_dir();
2631 let ctx = WaveCtx {
2632 carry_seats: false,
2633 pauses: Some((&self.pause, &self.interrupt)),
2634 run: run_id,
2635 node: "implement",
2636 prompts,
2637 cache: cache.as_deref(),
2638 round: None,
2639 };
2640 let (resumed_seat, resumed) =
2641 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2642 *seat = resumed_seat;
2643 *out = resumed;
2644 }
2645 }
2646
2647 async fn resume_seat_handovers(
2712 &mut self,
2713 results: &mut [(usize, SeatState, AgentOutcome)],
2714 sent: &mut [SeatJob],
2715 prompts: &Prompts,
2716 run_id: &str,
2717 ) -> Result<()> {
2718 let instruction = seeded_instruction(&self.state);
2719 let language = self.state.config.graph.language.clone();
2720 let github_language = self.state.config.graph.github_language.clone();
2721 let brief = self
2722 .state
2723 .advice
2724 .as_ref()
2725 .and_then(|a| a.synthesis.as_deref())
2726 .map(str::to_owned);
2727 let attachments = self.state.attachments.clone();
2728 for (wi, seat, out) in results.iter_mut() {
2729 let others: BTreeSet<String> = sent
2732 .iter()
2733 .enumerate()
2734 .filter(|(j, _)| j != wi)
2735 .map(|(_, j)| j.spec.id.clone())
2736 .collect();
2737 let Some(job) = sent.get_mut(*wi) else {
2738 continue;
2739 };
2740 let start = self
2745 .roles
2746 .implementer_roster
2747 .iter()
2748 .position(|s| s.id == job.spec.id)
2749 .unwrap_or(0);
2750 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2751 let mut fallback_attempt = 0usize;
2752 let mut prev: Option<FailClass> = None;
2753 while let Some(cur) = FailClass::of(&*out) {
2754 if !should_hand_over(prev.as_ref(), &cur) {
2755 break;
2756 }
2757 let Some(next) =
2758 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2759 .cloned()
2760 else {
2761 break;
2762 };
2763 tried.insert(next.id.clone());
2764 fallback_attempt += 1;
2765
2766 let coordination = self.coordinate(&job.cwd).await?;
2767 if let Ok(r) = git::rescue_commit(
2768 &job.cwd,
2769 &format!(
2770 "magi: candidate {} (uncommitted work before {} fallback)",
2771 seat.key,
2772 if cur == FailClass::Quota {
2773 "quota"
2774 } else {
2775 "handover"
2776 }
2777 ),
2778 )
2779 .await
2780 {
2781 self.state.note_withheld("implement", &r.withheld);
2782 }
2783 drop(coordination);
2784
2785 record_handover(
2786 &mut self.state,
2787 "implement",
2788 &seat.key,
2789 &seat.agent,
2790 &next.id,
2791 &cur,
2792 &fail_reason(&*out),
2793 );
2794 prev = Some(cur.clone());
2795
2796 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2797 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2798 job.spec = next.clone();
2806 let mut retry = job.clone();
2807 retry.seat = new_seat;
2808 retry.prompt = prompt::implement_in(
2809 &instruction,
2810 &job.cwd.to_string_lossy(),
2811 &language,
2812 &github_language,
2813 brief.as_deref(),
2814 &attachments,
2815 );
2816 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2817 let cache = self.state.config.cache_dir();
2818 let ctx = WaveCtx {
2819 carry_seats: false,
2820 pauses: Some((&self.pause, &self.interrupt)),
2821 run: run_id,
2822 node: "implement",
2823 prompts,
2824 cache: cache.as_deref(),
2825 round: None,
2826 };
2827 let (fallback_seat, fallback_out) = run_one(
2828 retry,
2829 Arc::clone(&self.sem),
2830 &ctx,
2831 &mut self.state,
2832 fallback_attempt,
2833 )
2834 .await;
2835 *seat = fallback_seat;
2836 *out = fallback_out;
2837 }
2838 }
2839 Ok(())
2840 }
2841
2842 async fn resume_unconfirmed_commands(
2866 &mut self,
2867 results: &mut [(usize, SeatState, AgentOutcome)],
2868 sent: &[SeatJob],
2869 prompts: &Prompts,
2870 run_id: &str,
2871 ) {
2872 for (wi, seat, out) in results.iter_mut() {
2873 let AgentOutcome::Ok(o) = &*out else {
2874 continue;
2875 };
2876 if !has_unconfirmed_command(&o.commands) {
2877 continue;
2878 }
2879 let Some(job) = sent.get(*wi) else { continue };
2880 if !has_context(&job.spec, seat, job.sessions) {
2881 self.state.event(
2882 "implement",
2883 format!(
2884 "{}: the reply named a command whose own CLI never confirmed the exit \
2885 status of, but there is no session left to resume",
2886 seat.key
2887 ),
2888 );
2889 continue;
2890 }
2891 self.state.event(
2892 "implement",
2893 format!(
2894 "{}: the reply named a command whose own CLI never confirmed the exit \
2895 status of; resuming the conversation",
2896 seat.key
2897 ),
2898 );
2899 let mut retry = job.clone();
2900 retry.seat = seat.clone();
2901 retry.prompt = prompt::resume_incomplete(
2902 "a command in your last reply had no confirmed exit status",
2903 );
2904 retry.timeout = retry_budget(job.timeout, true);
2905 retry.stem = format!("{}-confirm", job.stem);
2906 let cache = self.state.config.cache_dir();
2907 let ctx = WaveCtx {
2908 carry_seats: false,
2909 pauses: Some((&self.pause, &self.interrupt)),
2910 run: run_id,
2911 node: "implement",
2912 prompts,
2913 cache: cache.as_deref(),
2914 round: None,
2915 };
2916 let (resumed_seat, resumed) =
2917 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2918 *seat = resumed_seat;
2919 *out = resumed;
2920 }
2921 }
2922
2923 async fn continue_fix_report(
2944 &mut self,
2945 mut seat: SeatState,
2946 parse_err: String,
2947 job: &SeatJob,
2948 prompts: &Prompts,
2949 run_id: &str,
2950 round: usize,
2951 ) -> (
2952 SeatState,
2953 Option<FixReport>,
2954 Option<String>,
2955 ContinuationRecord,
2956 ) {
2957 let mut last_err = parse_err;
2958 let mut cumulative_wait_ms = 0u64;
2959 let mut attempts = 0usize;
2960 loop {
2961 if !has_context(&job.spec, &seat, job.sessions) {
2962 self.state.event(
2963 "fix",
2964 format!(
2965 "round {round}: fixer's reply had no adoption report ({last_err}); no \
2966 session left to resume into"
2967 ),
2968 );
2969 let outcome = if attempts == 0 {
2970 ContinuationOutcome::NoSession
2971 } else {
2972 ContinuationOutcome::Exhausted
2973 };
2974 return (
2975 seat,
2976 None,
2977 Some(format!("unparsable fix report: {last_err}")),
2978 ContinuationRecord {
2979 attempts,
2980 cumulative_wait_ms,
2981 outcome,
2982 },
2983 );
2984 }
2985 if attempts >= MAX_FIX_CONTINUATIONS {
2986 self.state.event(
2987 "fix",
2988 format!(
2989 "round {round}: fixer's reply still had no adoption report after \
2990 {attempts} continuation(s) ({last_err}); giving up"
2991 ),
2992 );
2993 return (
2994 seat,
2995 None,
2996 Some(format!(
2997 "unparsable fix report after {attempts} continuation(s): {last_err}"
2998 )),
2999 ContinuationRecord {
3000 attempts,
3001 cumulative_wait_ms,
3002 outcome: ContinuationOutcome::Exhausted,
3003 },
3004 );
3005 }
3006 attempts += 1;
3007 self.state.event(
3008 "fix",
3009 format!(
3010 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3011 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3012 ),
3013 );
3014 let mut retry = job.clone();
3015 retry.seat = seat.clone();
3016 retry.prompt = prompt::resume_incomplete(&last_err);
3017 retry.timeout = retry_budget(job.timeout, true);
3018 retry.stem = format!("{}-continue{attempts}", job.stem);
3019 let cache = self.state.config.cache_dir();
3020 let ctx = WaveCtx {
3021 carry_seats: false,
3022 pauses: Some((&self.pause, &self.interrupt)),
3023 run: run_id,
3024 node: "fix",
3025 prompts,
3026 cache: cache.as_deref(),
3027 round: Some(round),
3028 };
3029 let (resumed_seat, resumed_out) = run_one(
3030 retry,
3031 Arc::clone(&self.sem),
3032 &ctx,
3033 &mut self.state,
3034 attempts,
3035 )
3036 .await;
3037 seat = resumed_seat;
3038 match resumed_out {
3039 AgentOutcome::Ok(o) => {
3040 cumulative_wait_ms += o.duration_ms;
3041 match verdict::extract_json::<FixReport>(&o.text) {
3042 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3043 self.state.event(
3044 "fix",
3045 format!(
3046 "round {round}: fixer's adoption report recovered after \
3047 {attempts} continuation(s)"
3048 ),
3049 );
3050 return (
3051 seat,
3052 Some(report),
3053 None,
3054 ContinuationRecord {
3055 attempts,
3056 cumulative_wait_ms,
3057 outcome: ContinuationOutcome::Resumed,
3058 },
3059 );
3060 }
3061 Ok(_) => {
3069 last_err = "the reply parsed, but it reported a command whose own CLI \
3070 never confirmed an exit status"
3071 .to_owned();
3072 }
3073 Err(e) => last_err = e.to_string(),
3074 }
3075 }
3076 AgentOutcome::Quota(o) => {
3077 cumulative_wait_ms += o.duration_ms;
3078 self.state.quota.push(QuotaLoss {
3079 seat: seat.key.clone(),
3080 node: "fix".to_owned(),
3081 at: Timestamp::now(),
3082 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3083 });
3084 self.state.event(
3085 "fix",
3086 format!(
3087 "round {round}: continuation rate limited (quota); not retrying now"
3088 ),
3089 );
3090 return (
3091 seat,
3092 None,
3093 Some("rate limited (quota) while recovering the fix report".to_owned()),
3094 ContinuationRecord {
3095 attempts,
3096 cumulative_wait_ms,
3097 outcome: ContinuationOutcome::QuotaLost,
3098 },
3099 );
3100 }
3101 AgentOutcome::Dropped(o) => {
3102 cumulative_wait_ms += o.duration_ms;
3103 let why = o
3104 .dropped
3105 .as_ref()
3106 .map(|d| d.why.as_str())
3107 .unwrap_or("the CLI ended the stream without delivering its answer");
3108 last_err = format!("the CLI dropped the stream ({why})");
3109 }
3110 AgentOutcome::Failed(e) => last_err = e,
3111 AgentOutcome::ResourceBlocked(e) => {
3114 return (
3115 seat,
3116 None,
3117 Some(format!("the continuation was not started: {e}")),
3118 ContinuationRecord {
3119 attempts: attempts.saturating_sub(1),
3120 cumulative_wait_ms,
3121 outcome: ContinuationOutcome::NoSession,
3122 },
3123 );
3124 }
3125 }
3126 }
3127 }
3128
3129 fn after_implement(&mut self) -> Result<()> {
3130 if !self.state.candidate_preservation.is_empty() {
3131 bail!("candidate preservation incomplete; judging is unsafe");
3132 }
3133 if self.state.leaks.is_empty() {
3135 let cfg = self.state.config.blind.clone();
3136 let mut leaks = Vec::new();
3137 for c in &self.state.candidates {
3138 let Some(patch) =
3139 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3140 else {
3141 continue;
3142 };
3143 leaks.extend(blind::scan(
3144 &format!("candidate {} patch", c.label),
3145 &patch,
3146 &cfg.vendor_tokens,
3147 ));
3148 }
3149 if !leaks.is_empty() {
3150 let summary = leaks
3151 .iter()
3152 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3153 .collect::<Vec<_>>()
3154 .join(", ");
3155 match cfg.on_leak {
3156 LeakPolicy::Fail => {
3157 self.state.status = RunStatus::Failed;
3158 self.state
3159 .event("blind", format!("vendor text in a patch: {summary}"));
3160 self.state.leaks = leaks;
3161 self.state.save()?;
3162 self.settle_questions();
3163 bail!(
3164 "blind.on_leak = \"fail\" and vendor text reached a \
3165 judged patch: {summary}"
3166 );
3167 }
3168 LeakPolicy::Redact => self.state.event(
3169 "blind",
3170 format!("redacting vendor text for judging: {summary}"),
3171 ),
3172 LeakPolicy::Warn => self.state.event(
3173 "blind",
3174 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3175 ),
3176 }
3177 self.state.leaks = leaks;
3178 }
3179 }
3180
3181 if self.state.viable().is_empty() {
3182 if self.state.all_candidates_verified_noop() {
3183 self.state.status = RunStatus::VerifiedNoop;
3194 self.state.save()?;
3195 self.settle_questions();
3196 return Ok(());
3197 }
3198 self.state.status = RunStatus::Failed;
3199 self.state.save()?;
3200 self.settle_questions();
3201 bail!("no candidate produced a change; nothing to judge");
3202 }
3203 self.state.status = RunStatus::Judging;
3204 self.state.save()?;
3205 Ok(())
3206 }
3207
3208 async fn judge(&mut self) -> Result<()> {
3211 let run_id = self.state.id.clone();
3216 let prompts = self.state.config.prompts.clone();
3217 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3218 return Ok(());
3219 }
3220 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3221 if viable.len() == 1 {
3222 self.state.judge_skipped = true;
3229 self.state.event(
3230 "judge",
3231 format!(
3232 "only candidate {} produced a change; judging skipped",
3233 viable[0].label
3234 ),
3235 );
3236 self.state.save()?;
3237 return Ok(());
3238 }
3239 self.state.status = RunStatus::Judging;
3240
3241 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3242 let language = self.state.config.graph.language.clone();
3243 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3244 let sessions = self.state.config.graph.sessions;
3245 let artifacts = agent::artifacts_dir(&self.state.dir());
3246 let root = self.state.worktree_root();
3247 let base_short = short(&self.state.base_commit);
3248
3249 let mut jobs = Vec::new();
3250 let mut orders = Vec::new();
3251 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3252 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3253 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3254 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3255 let seat_key = format!("judge-{}", j + 1);
3256 let seat = self.seat(&seat_key, &spec.id);
3257 jobs.push(SeatJob {
3258 prompt: prompt::judge(
3259 &self.state.instruction,
3260 &views,
3261 self.roles.judges.len(),
3262 &base_short,
3263 &language,
3264 ),
3265 spec,
3266 seat,
3267 cwd: root.join(format!("judge-{}", j + 1)),
3268 timeout,
3269 allow_write: false,
3270 sessions,
3271 artifacts: artifacts.clone(),
3272 stem: format!("judge-{}", j + 1),
3273 handover: None,
3274 });
3275 }
3276
3277 self.state.event(
3278 "judge",
3279 format!(
3280 "{} judges ranking {} candidates blind",
3281 jobs.len(),
3282 viable.len()
3283 ),
3284 );
3285 let labels_for_check = labels.clone();
3286 let mut quota_losses = Vec::new();
3287 let cache = self.state.config.cache_dir();
3288 let ctx = WaveCtx {
3289 carry_seats: false,
3290 pauses: Some((&self.pause, &self.interrupt)),
3291 run: &run_id,
3292 node: "judge",
3293 prompts: &prompts,
3294 cache: cache.as_deref(),
3295 round: None,
3296 };
3297 let results = ask_json_wave::<Ranking>(
3298 jobs,
3299 Arc::clone(&self.sem),
3300 self.state.config.graph.retries,
3301 &self.roles.judge_roster,
3302 &ctx,
3303 &mut quota_losses,
3304 &mut self.state,
3305 &move |r: &Ranking| r.validate(&labels_for_check),
3306 )
3307 .await;
3308 if self.state.resource_block.is_some() {
3309 return self.stop_for_resource();
3310 }
3311 self.state.quota.extend(quota_losses);
3312
3313 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3314 let agent_id = seat.agent.clone();
3315 self.state.seats.insert(seat.key.clone(), seat);
3316 let mut record = Judgement {
3317 judge: j + 1,
3318 seat: format!("judge-{}", j + 1),
3319 agent: agent_id,
3320 ranking: Vec::new(),
3321 reasons: BTreeMap::new(),
3322 confidence: None,
3323 strengths: BTreeMap::new(),
3324 order: orders[j].clone(),
3325 failed: None,
3326 duration_ms: 0,
3327 };
3328 match res {
3329 Ok((ranking, out)) => {
3330 record.ranking = ranking.normalized();
3331 record.reasons = ranking.reasons;
3332 record.strengths = ranking.strengths;
3333 record.confidence = ranking.confidence;
3334 record.duration_ms = out.duration_ms;
3335 self.state.event(
3336 "judge",
3337 format!(
3338 "judge {} ranked {}",
3339 j + 1,
3340 record.ranking.iter().collect::<String>()
3341 ),
3342 );
3343 }
3344 Err(e) => {
3345 record.failed = Some(e.to_string());
3346 self.state
3347 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3348 }
3349 }
3350 self.state.judgements.push(record);
3351 self.state.save()?;
3352 }
3353 Ok(())
3354 }
3355
3356 async fn deliberate(&mut self) -> Result<()> {
3359 let run_id = self.state.id.clone();
3364 let prompts = self.state.config.prompts.clone();
3365 if !self.state.deliberation.is_empty() {
3366 return Ok(());
3367 }
3368 let tops: Vec<char> = self
3369 .state
3370 .judgements
3371 .iter()
3372 .filter_map(|j| j.ranking.first().copied())
3373 .collect();
3374 let rounds = self.state.config.graph.deliberate_rounds;
3375 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3376 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3377 self.state.event(
3378 "deliberate",
3379 format!("judges agreed on {} outright; no deliberation", tops[0]),
3380 );
3381 }
3382 self.state.status = RunStatus::Voting;
3383 self.state.save()?;
3384 return Ok(());
3385 }
3386
3387 self.state.status = RunStatus::Deliberating;
3388 self.state.event(
3389 "deliberate",
3390 format!(
3391 "split: first choices were {} — opening {rounds} round(s)",
3392 tops.iter().collect::<String>()
3393 ),
3394 );
3395
3396 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3397 let language = self.state.config.graph.language.clone();
3398 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3399 let sessions = self.state.config.graph.sessions;
3400 let artifacts = agent::artifacts_dir(&self.state.dir());
3401 let root = self.state.worktree_root();
3402 let base_short = short(&self.state.base_commit);
3403
3404 for round in 1..=rounds {
3408 let mut turns: Vec<DeliberationTurn> = Vec::new();
3409 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3410 if self.state.judgements[j].failed.is_some() {
3411 continue;
3412 }
3413 let seat_key = format!("judge-{}", j + 1);
3414 let spec = self.occupant(&seat_key, spec);
3415 let mut seat = self.seat(&seat_key, &spec.id);
3416 let transcript = self.transcript(&turns, j);
3417 let build = |context: Option<&str>| {
3418 prompt::deliberate(
3419 &self.state.instruction,
3420 context,
3421 &transcript,
3422 round,
3423 rounds,
3424 &language,
3425 )
3426 };
3427 let block = self.candidate_block(&viable, &base_short);
3428 let full = build(Some(&block));
3429 let text = if has_context(&spec, &seat, sessions) {
3430 build(None)
3431 } else {
3432 full.clone()
3433 };
3434 let job = SeatJob {
3435 spec,
3436 seat: seat.clone(),
3437 prompt: text,
3438 cwd: root.join(format!("judge-{}", j + 1)),
3439 timeout,
3440 allow_write: false,
3441 sessions,
3442 artifacts: artifacts.clone(),
3443 stem: format!("delib-{round}-judge-{}", j + 1),
3444 handover: Some(full),
3445 };
3446 let cache = self.state.config.cache_dir();
3447 let ctx = WaveCtx {
3448 carry_seats: false,
3449 pauses: Some((&self.pause, &self.interrupt)),
3450 run: &run_id,
3451 node: "deliberate",
3452 prompts: &prompts,
3453 cache: cache.as_deref(),
3454 round: None,
3455 };
3456 let mut losses = Vec::new();
3460 let mut results = ask_wave_with::<String>(
3461 vec![job],
3462 Arc::clone(&self.sem),
3463 0,
3464 &self.roles.judge_roster,
3465 &ctx,
3466 &mut losses,
3467 &mut self.state,
3468 &|text: &str| {
3469 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3470 },
3471 )
3472 .await;
3473 self.state.quota.extend(losses);
3474 let (updated, res, _) = results.pop().expect("one job in, one result out");
3475 seat = updated;
3476 let agent_id = seat.agent.clone();
3477 self.state.seats.insert(seat.key.clone(), seat);
3478 let body = match res {
3479 Ok((body, _)) => body,
3480 Err(e) => {
3483 self.state
3484 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3485 continue;
3486 }
3487 };
3488 let tentative = verdict::extract_json::<Position>(&body)
3489 .ok()
3490 .and_then(|p| p.tentative)
3491 .and_then(|s| s.trim().chars().next())
3492 .map(|c| c.to_ascii_uppercase());
3493 self.state.event(
3494 "deliberate",
3495 format!(
3496 "round {round}: judge {} now favours {}",
3497 j + 1,
3498 tentative.map_or("—".to_owned(), |c| c.to_string())
3499 ),
3500 );
3501 turns.push(DeliberationTurn {
3502 judge: j + 1,
3503 agent: agent_id,
3504 body: blind::sanitize_prose(&body, &self.state.config.blind),
3505 tentative,
3506 });
3507 }
3508 self.state
3509 .deliberation
3510 .push(DeliberationRound { round, turns });
3511 self.state.save()?;
3512 }
3513
3514 self.state.status = RunStatus::Voting;
3515 self.state.save()?;
3516 Ok(())
3517 }
3518
3519 async fn vote(&mut self) -> Result<()> {
3522 let run_id = self.state.id.clone();
3527 let prompts = self.state.config.prompts.clone();
3528 if !self.state.votes.is_empty() {
3529 return Ok(());
3530 }
3531 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3532 if viable.len() == 1 {
3533 return Ok(());
3534 }
3535 self.state.status = RunStatus::Voting;
3536
3537 let language = self.state.config.graph.language.clone();
3538 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3539 let sessions = self.state.config.graph.sessions;
3540 let artifacts = agent::artifacts_dir(&self.state.dir());
3541 let root = self.state.worktree_root();
3542 let base_short = short(&self.state.base_commit);
3543 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3544
3545 let mut jobs = Vec::new();
3546 let mut seats_at = Vec::new();
3547 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3548 if self
3549 .state
3550 .judgements
3551 .get(j)
3552 .is_some_and(|r| r.failed.is_some())
3553 {
3554 continue;
3555 }
3556 let seat_key = format!("judge-{}", j + 1);
3557 let spec = self.occupant(&seat_key, spec);
3558 let seat = self.seat(&seat_key, &spec.id);
3559 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3560 let text = if has_context(&spec, &seat, sessions) {
3561 prompt::final_vote(&viable, &language)
3562 } else {
3563 full.clone()
3564 };
3565 jobs.push(SeatJob {
3566 spec,
3567 seat,
3568 prompt: text,
3569 cwd: root.join(format!("judge-{}", j + 1)),
3570 timeout,
3571 allow_write: false,
3572 sessions,
3573 artifacts: artifacts.clone(),
3574 stem: format!("vote-judge-{}", j + 1),
3575 handover: Some(full),
3576 });
3577 seats_at.push(j);
3578 }
3579
3580 self.state.event(
3581 "vote",
3582 format!(
3583 "collecting {} final votes one by one, privately",
3584 jobs.len()
3585 ),
3586 );
3587 let allowed = viable.clone();
3588 let mut quota_losses = Vec::new();
3589 let cache = self.state.config.cache_dir();
3590 let ctx = WaveCtx {
3591 carry_seats: false,
3592 pauses: Some((&self.pause, &self.interrupt)),
3593 run: &run_id,
3594 node: "vote",
3595 prompts: &prompts,
3596 cache: cache.as_deref(),
3597 round: None,
3598 };
3599 let results = ask_json_wave::<FinalVote>(
3600 jobs,
3601 Arc::clone(&self.sem),
3602 self.state.config.graph.retries,
3603 &self.roles.judge_roster,
3604 &ctx,
3605 &mut quota_losses,
3606 &mut self.state,
3607 &move |v: &FinalVote| match v.label() {
3608 Some(c) if allowed.contains(&c) => Ok(()),
3609 other => bail!("vote {other:?} is not one of {allowed:?}"),
3610 },
3611 )
3612 .await;
3613 if self.state.resource_block.is_some() {
3614 return self.stop_for_resource();
3615 }
3616 self.state.quota.extend(quota_losses);
3617
3618 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3619 let agent_id = seat.agent.clone();
3620 self.state.seats.insert(seat.key.clone(), seat);
3621 let initial = self
3622 .state
3623 .judgements
3624 .get(j)
3625 .and_then(|r| r.ranking.first().copied());
3626 let mut record = VoteRecord {
3627 judge: j + 1,
3628 agent: agent_id,
3629 vote: None,
3630 reason: String::new(),
3631 changed: false,
3632 };
3633 match res {
3634 Ok((v, _)) => {
3635 record.vote = v.label();
3636 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3637 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3638 self.state.event(
3639 "vote",
3640 format!(
3641 "judge {} voted {}{}",
3642 j + 1,
3643 record.vote.unwrap_or('?'),
3644 if record.changed { " (changed)" } else { "" }
3645 ),
3646 );
3647 }
3648 Err(e) => {
3649 self.state
3650 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3651 }
3652 }
3653 self.state.votes.push(record);
3654 self.state.save()?;
3655 }
3656 Ok(())
3657 }
3658
3659 fn tally(&mut self) -> Result<()> {
3662 if self.state.tally.is_some() {
3663 return Ok(());
3664 }
3665 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3666 let tops: Vec<char> = self
3667 .state
3668 .judgements
3669 .iter()
3670 .filter_map(|j| j.ranking.first().copied())
3671 .collect();
3672 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3673
3674 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3677 let mut cast: Vec<char> = Vec::new();
3678 for (i, j) in self.state.judgements.iter().enumerate() {
3679 let vote = self
3680 .state
3681 .votes
3682 .iter()
3683 .find(|v| v.judge == i + 1)
3684 .and_then(|v| v.vote)
3685 .or_else(|| j.ranking.first().copied());
3686 if let Some(v) = vote {
3687 *first_choice.entry(v).or_insert(0) += 1;
3688 cast.push(v);
3689 }
3690 }
3691
3692 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3693 for j in &self.state.judgements {
3694 let n = j.ranking.len();
3695 for (pos, label) in j.ranking.iter().enumerate() {
3696 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3697 }
3698 }
3699
3700 let best = first_choice.values().copied().max().unwrap_or(0);
3701 let mut leaders: Vec<char> = first_choice
3702 .iter()
3703 .filter(|(_, v)| **v == best)
3704 .map(|(k, _)| *k)
3705 .collect();
3706 let mut tie_break = None;
3707 if leaders.len() > 1 {
3708 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3709 let borda_leaders: Vec<char> = leaders
3710 .iter()
3711 .copied()
3712 .filter(|l| borda[l] == top_borda)
3713 .collect();
3714 tie_break = Some(if borda_leaders.len() == 1 {
3715 format!(
3716 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3717 leaders.len()
3718 )
3719 } else {
3720 format!(
3721 "{} way tie on both first-choice votes and Borda points, broken by label order",
3722 leaders.len()
3723 )
3724 });
3725 leaders = borda_leaders;
3726 leaders.sort_unstable();
3727 }
3728 let winner = *leaders
3729 .first()
3730 .or(viable.first())
3731 .context("no candidate to declare a winner from")?;
3732
3733 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3734 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3735 let deliberated = !self.state.deliberation.is_empty();
3736
3737 let quota_seats: std::collections::BTreeSet<&str> =
3741 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3742 let mut present = 0usize;
3743 for (i, j) in self.state.judgements.iter().enumerate() {
3744 if quota_seats.contains(j.seat.as_str()) {
3745 continue;
3746 }
3747 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3748 let voted = self
3749 .state
3750 .votes
3751 .iter()
3752 .any(|v| v.judge == i + 1 && v.vote.is_some());
3753 if ranked || voted {
3754 present += 1;
3755 }
3756 }
3757 let needs_quorum = viable.len() > 1;
3763 let judges_total = if needs_quorum {
3764 self.roles.judges.len()
3765 } else {
3766 0
3767 };
3768 let quorum = if needs_quorum {
3769 judges_total / 2 + 1
3770 } else {
3771 0
3772 };
3773 let met_quorum = !needs_quorum || present >= quorum;
3774 let uncontested = (!needs_quorum).then(|| {
3775 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3776 });
3777
3778 self.state.event(
3779 "tally",
3780 match &uncontested {
3781 Some(reason) => format!("winner {winner} — {reason}"),
3782 None => format!(
3783 "winner {winner} — votes {} | initial {} | {} changed | \
3784 {present}/{judges_total} judges{}",
3785 first_choice
3786 .iter()
3787 .map(|(k, v)| format!("{k}:{v}"))
3788 .collect::<Vec<_>>()
3789 .join(" "),
3790 if unanimous_initial {
3791 "unanimous"
3792 } else {
3793 "split"
3794 },
3795 changed_votes,
3796 if met_quorum {
3797 String::new()
3798 } else {
3799 format!(" — below quorum ({quorum} required)")
3800 },
3801 ),
3802 },
3803 );
3804 if !met_quorum {
3805 self.state.event(
3806 "stall",
3807 format!(
3808 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3809 the run stops here, resumable"
3810 ),
3811 );
3812 }
3813 self.state.tally = Some(Tally {
3814 first_choice,
3815 borda,
3816 winner,
3817 rankings: tops.len(),
3818 unanimous_initial,
3819 deliberated,
3820 changed_votes,
3821 unanimous_final,
3822 tie_break,
3823 judges: judges_total,
3824 present,
3825 quorum,
3826 met_quorum,
3827 uncontested,
3828 });
3829 self.state.status = if met_quorum {
3830 RunStatus::Reviewing
3831 } else {
3832 RunStatus::Stalled
3833 };
3834 self.state.save()?;
3835 Ok(())
3836 }
3837
3838 #[allow(clippy::too_many_lines)]
3859 async fn recover_stall(&mut self) -> Result<bool> {
3860 let run_id = self.state.id.clone();
3865 let prompts = self.state.config.prompts.clone();
3866 let quota_seats: BTreeSet<&str> =
3871 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3872 let absent: Vec<String> = self
3873 .state
3874 .judgements
3875 .iter()
3876 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3877 .map(|j| j.seat.clone())
3878 .collect();
3879 if absent.is_empty() {
3880 return Ok(false);
3881 }
3882 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3883 if viable.len() <= 1 {
3884 return Ok(false);
3885 }
3886 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3887 let language = self.state.config.graph.language.clone();
3888 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3889 let sessions = self.state.config.graph.sessions;
3890 let artifacts = agent::artifacts_dir(&self.state.dir());
3891 let root = self.state.worktree_root();
3892 let base_short = short(&self.state.base_commit);
3893 let candidates: Vec<Candidate> = viable.clone();
3894
3895 let mut positions: Vec<usize> = absent
3897 .iter()
3898 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3899 .collect();
3900 if positions.is_empty() {
3901 return Ok(false);
3902 }
3903 positions.sort_unstable();
3904 positions.dedup();
3905
3906 let mut judge_jobs = Vec::new();
3908 for &j in &positions {
3909 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3910 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3911 let seat_key = format!("judge-{}", j + 1);
3912 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3913 let seat = self.seat(&seat_key, &spec.id);
3914 judge_jobs.push(SeatJob {
3915 spec,
3916 seat,
3917 prompt: prompt::judge(
3918 &self.state.instruction,
3919 &views,
3920 self.roles.judges.len(),
3921 &base_short,
3922 &language,
3923 ),
3924 cwd: root.join(seat_key),
3925 timeout,
3926 allow_write: false,
3927 sessions,
3928 artifacts: artifacts.clone(),
3929 stem: format!("judge-{}-recover", j + 1),
3930 handover: None,
3931 });
3932 }
3933
3934 let labels_for_check = labels.clone();
3935 let mut judge_losses = Vec::new();
3936 let retries = self.state.config.graph.retries;
3937 let cache = self.state.config.cache_dir();
3938 let ctx = WaveCtx {
3939 carry_seats: false,
3940 pauses: Some((&self.pause, &self.interrupt)),
3941 run: &run_id,
3942 node: "judge",
3943 prompts: &prompts,
3944 cache: cache.as_deref(),
3945 round: None,
3946 };
3947 let results = ask_json_wave::<Ranking>(
3948 judge_jobs,
3949 Arc::clone(&self.sem),
3950 retries,
3951 &self.roles.judge_roster,
3952 &ctx,
3953 &mut judge_losses,
3954 &mut self.state,
3955 &move |r: &Ranking| r.validate(&labels_for_check),
3956 )
3957 .await;
3958 if self.state.resource_block.is_some() {
3959 self.stop_for_resource()?;
3960 return Ok(false);
3961 }
3962
3963 let mut recovered: BTreeSet<usize> = BTreeSet::new();
3965 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3966 let agent_id = seat.agent.clone();
3967 self.state.seats.insert(seat.key.clone(), seat);
3968 let record = &mut self.state.judgements[j];
3969 match res {
3970 Ok((ranking, out)) => {
3971 record.agent = agent_id;
3972 record.ranking = ranking.normalized();
3973 record.reasons = ranking.reasons;
3974 record.strengths = ranking.strengths;
3975 record.confidence = ranking.confidence;
3976 record.failed = None;
3977 record.duration_ms = out.duration_ms;
3978 recovered.insert(j);
3979 self.state.event(
3980 "recover",
3981 format!("judge {} ranked again after the limit", j + 1),
3982 );
3983 }
3984 Err(e) => {
3985 self.state
3986 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3987 }
3988 }
3989 }
3990
3991 let mut vote_jobs = Vec::new();
3993 let mut vote_pos: Vec<usize> = Vec::new();
3994 for &j in &recovered {
3995 let seat_key = format!("judge-{}", j + 1);
3996 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3997 let seat = self.seat(&seat_key, &spec.id);
3998 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3999 let text = if has_context(&spec, &seat, sessions) {
4000 prompt::final_vote(&labels, &language)
4001 } else {
4002 full.clone()
4003 };
4004 vote_jobs.push(SeatJob {
4005 spec,
4006 seat,
4007 prompt: text,
4008 cwd: root.join(seat_key),
4009 timeout,
4010 allow_write: false,
4011 sessions,
4012 artifacts: artifacts.clone(),
4013 stem: format!("vote-judge-{}-recover", j + 1),
4014 handover: Some(full),
4015 });
4016 vote_pos.push(j);
4017 }
4018 let allowed = labels.clone();
4019 let mut vote_losses = Vec::new();
4020 let vote_retries = self.state.config.graph.retries;
4021 let vote_cache = self.state.config.cache_dir();
4022 let ctx = WaveCtx {
4023 carry_seats: false,
4024 pauses: Some((&self.pause, &self.interrupt)),
4025 run: &run_id,
4026 node: "vote",
4027 prompts: &prompts,
4028 cache: vote_cache.as_deref(),
4029 round: None,
4030 };
4031 let votes = ask_json_wave::<FinalVote>(
4032 vote_jobs,
4033 Arc::clone(&self.sem),
4034 vote_retries,
4035 &self.roles.judge_roster,
4036 &ctx,
4037 &mut vote_losses,
4038 &mut self.state,
4039 &move |v: &FinalVote| match v.label() {
4040 Some(c) if allowed.contains(&c) => Ok(()),
4041 other => bail!("vote {other:?} is not one of {allowed:?}"),
4042 },
4043 )
4044 .await;
4045 if self.state.resource_block.is_some() {
4046 self.stop_for_resource()?;
4047 return Ok(false);
4048 }
4049 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4050 let agent_id = seat.agent.clone();
4051 self.state.seats.insert(seat.key.clone(), seat);
4052 match res {
4053 Ok((v, _)) => {
4054 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4055 rec.vote = v.label();
4056 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4057 } else {
4058 self.state.votes.push(VoteRecord {
4059 judge: j + 1,
4060 agent: agent_id,
4061 vote: v.label(),
4062 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4063 changed: false,
4064 });
4065 }
4066 self.state.event(
4067 "recover",
4068 format!("judge {} voted again after the limit", j + 1),
4069 );
4070 }
4071 Err(e) => {
4072 self.state
4073 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4074 }
4075 }
4076 }
4077
4078 let recovered_keys: BTreeSet<String> = recovered
4082 .iter()
4083 .map(|&j| format!("judge-{}", j + 1))
4084 .collect();
4085 self.state
4086 .quota
4087 .retain(|q| !recovered_keys.contains(&q.seat));
4088 for loss in judge_losses.into_iter().chain(vote_losses) {
4092 if recovered_keys.contains(&loss.seat) {
4093 continue;
4094 }
4095 self.state.quota.retain(|q| q.seat != loss.seat);
4096 self.state.quota.push(loss);
4097 }
4098
4099 self.state.tally = None;
4101 self.tally()?;
4102 Ok(self
4103 .state
4104 .tally
4105 .as_ref()
4106 .map(|t| t.met_quorum)
4107 .unwrap_or(false))
4108 }
4109
4110 async fn graft(&mut self) -> Result<()> {
4127 let Err(e) = self.graft_inner().await else {
4128 return Ok(());
4129 };
4130 if e.is::<crate::repo_lock::ResourceBlocked>() {
4131 return Err(e);
4132 }
4133 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4134 self.state.status = RunStatus::Blocked;
4135 self.state.event("graft", format!("blocked: {e:#}"));
4136 self.state.save()?;
4137 return Err(e);
4138 }
4139 if let Some(rec) = self.state.graft.clone()
4144 && !rec.outcome.settled()
4145 {
4146 if let Some(winner) = self.state.winner().cloned() {
4147 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4148 if re.is::<crate::repo_lock::ResourceBlocked>() {
4149 return Err(re);
4150 }
4151 self.state.status = RunStatus::Blocked;
4152 self.state.event("graft", format!("blocked: {re:#}"));
4153 self.state.save()?;
4154 return Err(re);
4155 }
4156 }
4157 if let Some(g) = self.state.graft.as_mut() {
4158 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4159 }
4160 }
4161 self.state.event("graft", format!("skipped: {e:#}"));
4162 if let Err(e) = self.state.save() {
4163 tracing::warn!("save after graft: {e:#}");
4164 }
4165 Ok(())
4166 }
4167
4168 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4171 let _coordination = self.coordinate(wt).await?;
4172 crate::recovery::restore_repair(wt, pre)
4173 .await
4174 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4175 }
4176
4177 fn graft_skipped(&mut self, why: &str) {
4179 let message = format!("skipped: {why}");
4180 if !self
4181 .state
4182 .events
4183 .iter()
4184 .any(|e| e.node == "graft" && e.message == message)
4185 {
4186 self.state.event("graft", message);
4187 }
4188 }
4189
4190 async fn graft_inner(&mut self) -> Result<()> {
4191 if let Some(rec) = self.state.graft.clone()
4194 && !rec.outcome.settled()
4195 {
4196 if self.state.reviews.is_empty()
4200 && let Some(winner) = self.state.winner().cloned()
4201 {
4202 self.graft_restore(&winner.worktree, &rec.pre).await?;
4203 }
4204 if rec.outcome == GraftOutcome::WaitingForRestore {
4205 self.state.graft = None;
4206 self.state.save()?;
4207 } else {
4208 if let Some(g) = self.state.graft.as_mut() {
4209 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4210 }
4211 self.state.event(
4212 "graft",
4213 "rolled back: the previous round was interrupted; the winner is untouched",
4214 );
4215 self.state.save()?;
4216 return Ok(());
4217 }
4218 }
4219 let plan = match graft::plan(&self.state) {
4220 Ok(plan) => plan,
4221 Err(skip) => {
4222 if skip != graft::Skip::AlreadyDone {
4223 self.graft_skipped(skip.reason());
4224 self.state.save()?;
4225 }
4226 return Ok(());
4227 }
4228 };
4229 let Some(winner) = self.state.winner().cloned() else {
4230 return Ok(());
4231 };
4232 let wt = winner.worktree.clone();
4233 if !git::is_clean(&wt).await? {
4234 self.graft_skipped("the winner's worktree has uncommitted changes");
4235 self.state.save()?;
4236 return Ok(());
4237 }
4238 let pre = git::rev_parse(&wt, "HEAD").await?;
4239 let Some(winner_lines) = git::changed_lines(&wt, &self.state.base_commit, &pre).await
4240 else {
4241 self.graft_skipped("the winner's diff could not be measured");
4242 self.state.save()?;
4243 return Ok(());
4244 };
4245 let limit = graft::limit(winner_lines);
4246 self.state.graft = Some(GraftRecord {
4249 pre: pre.clone(),
4250 post: None,
4251 lines: None,
4252 limit: Some(limit),
4253 agent: None,
4254 outcome: GraftOutcome::Running,
4255 requested: graft::requested(&plan),
4256 adoption: None,
4257 adoption_error: None,
4258 });
4259 self.state.event(
4260 "graft",
4261 format!(
4262 "porting strengths of candidate(s) {} onto {} (limit {limit} changed lines)",
4263 plan.strengths
4264 .iter()
4265 .map(|(l, _, _)| l.to_string())
4266 .collect::<Vec<_>>()
4267 .join(", "),
4268 plan.winner
4269 ),
4270 );
4271 self.state.save()?;
4272
4273 let fix_prompt = prompt::graft(
4274 &self.state.instruction,
4275 plan.winner,
4276 &plan.strengths,
4277 limit,
4278 &self.state.config.graph.language,
4279 );
4280 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4281 let sessions = self.state.config.graph.sessions;
4282 let artifacts = agent::artifacts_dir(&self.state.dir());
4283 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4284 let (_, seat, out) = self
4285 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4286 SeatJob {
4287 prompt: fix_prompt.clone(),
4288 spec,
4289 seat,
4290 cwd: wt.clone(),
4291 timeout,
4292 allow_write: true,
4293 sessions,
4294 artifacts: artifacts.clone(),
4295 stem: "graft".to_owned(),
4296 handover: None,
4297 }
4298 })
4299 .await;
4300 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4301 self.state.graft.as_mut().expect("graft record").outcome =
4304 GraftOutcome::WaitingForRestore;
4305 return self.stop_for_resource();
4306 }
4307 let mut why: Option<String> = None;
4308 match &out {
4309 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4310 why = Some("the fixer produced nothing usable".to_owned());
4311 }
4312 AgentOutcome::Ok(_) => {}
4313 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4314 AgentOutcome::Quota(o) => {
4315 self.state.quota.push(QuotaLoss {
4316 seat: seat.key.clone(),
4317 node: "graft".to_owned(),
4318 at: Timestamp::now(),
4319 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4320 });
4321 why = Some("rate limited (quota)".to_owned());
4322 }
4323 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4324 why = Some(format!(
4325 "the fixer failed: {}",
4326 e.lines()
4327 .next()
4328 .unwrap_or("")
4329 .chars()
4330 .take(160)
4331 .collect::<String>()
4332 ));
4333 }
4334 }
4335 let agent_id = seat.agent.clone();
4336 self.state.seats.insert(seat.key.clone(), seat);
4337
4338 let mut post = None;
4339 let mut lines = None;
4340 let _coordination = self.coordinate(&wt).await?;
4341 if why.is_none() {
4342 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4343 if let Ok(r) = &rescued {
4344 self.state.note_withheld("graft", &r.withheld);
4345 }
4346 let head = git::rev_parse(&wt, "HEAD").await?;
4347 let dirty = match &rescued {
4351 Ok(r) => git::status_porcelain(&wt)
4352 .await
4353 .map(|st| {
4354 st.lines().any(|l| {
4355 !r.withheld
4356 .iter()
4357 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4358 })
4359 })
4360 .unwrap_or(true),
4361 Err(_) => true,
4362 };
4363 if dirty {
4364 why = Some("the fixer left changes that could not be committed".to_owned());
4365 } else if head == pre {
4366 why = Some("the fixer changed nothing".to_owned());
4367 } else {
4368 lines = git::changed_lines(&wt, &pre, &head).await;
4369 if graft::within_budget(lines, limit) {
4370 post = Some(head);
4371 } else {
4372 why = Some(format!(
4373 "{} changed lines against a limit of {limit}",
4374 lines.map_or_else(
4375 || "an unmeasurable number of".to_owned(),
4376 |l| l.to_string()
4377 )
4378 ));
4379 }
4380 }
4381 }
4382 let (adoption, adoption_error) = match &out {
4383 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4384 _ => (None, None),
4385 };
4386 let requested = graft::requested(&plan);
4387 let outcome = match why {
4388 None => {
4389 self.state.event(
4390 "graft",
4391 format!(
4392 "applied: {} changed lines (limit {limit})",
4393 lines.unwrap_or(0)
4394 ),
4395 );
4396 GraftOutcome::Applied
4397 }
4398 Some(why) => {
4399 crate::recovery::restore_repair(&wt, &pre)
4402 .await
4403 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4404 self.state.event(
4405 "graft",
4406 format!("rolled back: {why}; the winner is untouched"),
4407 );
4408 GraftOutcome::RolledBack(why)
4409 }
4410 };
4411 self.state.graft = Some(GraftRecord {
4412 pre,
4413 post,
4414 lines,
4415 limit: Some(limit),
4416 agent: Some(agent_id),
4417 outcome,
4418 requested,
4419 adoption,
4420 adoption_error,
4421 });
4422 self.state.save()?;
4423 Ok(())
4424 }
4425
4426 async fn fold_losers(&mut self) -> Result<()> {
4429 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4430 return Ok(());
4431 };
4432 if !self.state.candidate_preservation.is_empty() {
4433 bail!("candidate preservation incomplete; refusing cleanup");
4434 }
4435 let repo = self.state.repo.clone();
4436 let _coordination = self.coordinate(&repo).await?;
4437 let mut folded = Vec::new();
4438 for i in 0..self.state.candidates.len() {
4439 let c = &self.state.candidates[i];
4440 if c.label == winner || c.folded {
4441 continue;
4442 }
4443 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4444 remove_preserved_worktree(&repo, &wt).await?;
4445 if git::branch_exists(&repo, &branch).await?
4446 && !git::branch_delete(&repo, &branch).await?
4447 {
4448 bail!("candidate branch removal refused");
4449 }
4450 self.state.candidates[i].folded = true;
4451 self.state.save()?;
4452 folded.push(label.to_string());
4453 }
4454 let root = self.state.worktree_root();
4456 for j in 1..=self.roles.judges.len() {
4457 let wt = root.join(format!("judge-{j}"));
4458 if wt.exists() {
4459 remove_preserved_worktree(&repo, &wt).await?;
4460 }
4461 }
4462 if self.state.config.graph.advise {
4465 for k in 1..=self.state.config.graph.advisors {
4466 let wt = root.join(format!("advisor-{k}"));
4467 if wt.exists() {
4468 remove_preserved_worktree(&repo, &wt).await?;
4469 }
4470 }
4471 }
4472 if !folded.is_empty() {
4473 self.state
4474 .event("fold", format!("folded candidates {}", folded.join(", ")));
4475 self.state.save()?;
4476 }
4477 Ok(())
4478 }
4479
4480 async fn sync_to_base(&mut self) -> Result<()> {
4520 if self.state.status == RunStatus::AlreadyInBase {
4521 return Ok(());
4522 }
4523 let conflicted = self
4524 .state
4525 .base_sync
4526 .as_ref()
4527 .is_some_and(|s| s.conflict.is_some());
4528 let Some(winner) = self.state.winner().cloned() else {
4529 return Ok(());
4530 };
4531
4532 let repo = self.state.repo.clone();
4533 let remote = self.state.config.merge.remote.clone();
4534 let base_branch = self.state.base_branch.clone();
4535 let tracking = format!("{remote}/{base_branch}");
4536
4537 git::fetch(&repo, &remote, &base_branch).await.ok();
4538 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4543 return Ok(());
4544 };
4545
4546 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4547 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4548 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4549
4550 if (behind > 0 || conflicted || head == tip)
4559 && self
4560 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4561 .await?
4562 {
4563 return Ok(());
4564 }
4565 if conflicted {
4566 return Ok(());
4567 }
4568
4569 if behind == 0 {
4570 if let Some(from) = self
4579 .state
4580 .rebase_fixes
4581 .iter()
4582 .rev()
4583 .find_map(|r| r.from.clone())
4584 && from != head
4585 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4586 .await
4587 .is_ok_and(|o| o.ok())
4588 {
4589 git::sync_to_head(&winner.worktree).await?;
4590 }
4591 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4597 if let Some(why) = &conflict {
4598 self.state.status = RunStatus::Blocked;
4599 self.state.event("land", why.clone());
4600 }
4601 self.state.base_sync = Some(BaseSync {
4602 tip,
4603 behind: 0,
4604 attempts,
4605 conflict,
4606 already_in: None,
4607 });
4608 self.state.save()?;
4609 return Ok(());
4610 }
4611
4612 if attempts >= BASE_SYNC_ROUNDS {
4613 let why = format!(
4614 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4615 rebase(s); rebasing again would only race it",
4616 winner.branch
4617 );
4618 self.state.status = RunStatus::Blocked;
4619 self.state.base_sync = Some(BaseSync {
4620 tip,
4621 behind,
4622 attempts,
4623 conflict: Some(why.clone()),
4624 already_in: None,
4625 });
4626 self.state.event("land", why);
4627 self.state.save()?;
4628 return Ok(());
4629 }
4630
4631 self.state.event(
4632 "land",
4633 format!(
4634 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4635 winner.branch
4636 ),
4637 );
4638 self.state.save()?;
4639
4640 let branch_tracking = format!("{remote}/{}", winner.branch);
4646 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4647 let remote_tip = if self.state.task_pr_branch().is_none()
4648 && matches!(&fetched_branch, Ok(o) if o.ok())
4649 {
4650 git::rev_parse(&repo, &branch_tracking).await.ok()
4651 } else {
4652 None
4653 };
4654 if let Some(theirs) = &remote_tip
4658 && !git::is_ancestor(&repo, theirs, &head).await
4659 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4660 .await
4661 .is_ok_and(|missing| missing.is_empty())
4662 {
4663 let why = format!(
4664 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4665 them",
4666 short(theirs),
4667 winner.branch
4668 );
4669 self.state.status = RunStatus::Blocked;
4670 self.state.base_sync = Some(BaseSync {
4671 tip,
4672 behind,
4673 attempts,
4674 conflict: Some(why.clone()),
4675 already_in: None,
4676 });
4677 self.state.event("land", why);
4678 self.state.save()?;
4679 return Ok(());
4680 }
4681
4682 let scratch = self.state.dir().join("base-sync");
4683 let rebased = match crate::rebase::rebase_with_fixer(
4684 &mut self.state,
4685 &scratch,
4686 &winner.branch,
4687 &tracking,
4688 )
4689 .await
4690 {
4691 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4692 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4693 Err(e) => Err(e),
4694 };
4695 let attempts = attempts + 1;
4696 match rebased {
4697 Ok(None) => {
4698 git::sync_to_head(&winner.worktree).await?;
4702 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4703 let mut conflict = None;
4704 if let Some(pinned) = &remote_tip {
4705 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4706 match pushed {
4707 Ok(o) if o.ok() => self.state.event(
4708 "land",
4709 format!("pushed rebased {} to {remote}", winner.branch),
4710 ),
4711 Ok(o) => {
4712 conflict = Some(format!(
4713 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4714 winner.branch,
4715 short(pinned),
4716 o.stderr.chars().take(600).collect::<String>()
4717 ));
4718 }
4719 Err(e) => {
4720 conflict = Some(format!(
4721 "rebased {} locally but could not push it: {e:#}",
4722 winner.branch
4723 ));
4724 }
4725 }
4726 }
4727 if let Some(why) = &conflict {
4728 self.state.status = RunStatus::Blocked;
4729 self.state.event("land", why.clone());
4730 }
4731 self.state.base_sync = Some(BaseSync {
4732 tip: tip.clone(),
4733 behind: 0,
4734 attempts,
4735 conflict,
4736 already_in: None,
4737 });
4738 self.state
4739 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4740 }
4741 Ok(Some(conflict)) => {
4742 let why = format!(
4743 "{} conflicts with {tracking} and did not rebase: {}",
4744 winner.branch,
4745 conflict.chars().take(600).collect::<String>()
4746 );
4747 self.state.status = RunStatus::Blocked;
4748 self.state.base_sync = Some(BaseSync {
4749 tip,
4750 behind,
4751 attempts,
4752 conflict: Some(why.clone()),
4753 already_in: None,
4754 });
4755 self.state.event("land", why);
4756 }
4757 Err(e) => {
4758 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4759 self.state.status = RunStatus::Blocked;
4760 self.state.base_sync = Some(BaseSync {
4761 tip,
4762 behind,
4763 attempts,
4764 conflict: Some(why.clone()),
4765 already_in: None,
4766 });
4767 self.state.event("land", why);
4768 }
4769 }
4770 self.state.save()?;
4771 Ok(())
4772 }
4773
4774 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4780 if self.state.task_pr_branch().is_some() {
4783 return None;
4784 }
4785 let repo = self.state.repo.clone();
4786 let remote = self.state.config.merge.remote.clone();
4787 let fetched = git::fetch(&repo, &remote, branch).await;
4788 if !matches!(&fetched, Ok(o) if o.ok()) {
4789 return None;
4790 }
4791 let branch_tracking = format!("{remote}/{branch}");
4792 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4793 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4794 return None;
4795 }
4796 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4797 .await
4798 .is_ok_and(|missing| missing.is_empty())
4799 {
4800 return Some(format!(
4801 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4802 them",
4803 short(&theirs)
4804 ));
4805 }
4806 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4807 Ok(o) if o.ok() => {
4808 self.state
4809 .event("land", format!("pushed rebased {branch} to {remote}"));
4810 None
4811 }
4812 Ok(o) => Some(format!(
4813 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4814 someone may have pushed): {}",
4815 short(&theirs),
4816 o.stderr.chars().take(600).collect::<String>()
4817 )),
4818 Err(e) => Some(format!(
4819 "{branch} is rebased locally but could not be pushed: {e:#}"
4820 )),
4821 }
4822 }
4823
4824 async fn settle_already_in(
4841 &mut self,
4842 branch: &str,
4843 tip: &str,
4844 head: &str,
4845 attempts: usize,
4846 behind: usize,
4847 ) -> Result<bool> {
4848 let repo = self.state.repo.clone();
4849 let remote = self.state.config.merge.remote.clone();
4850 let start = self.state.base_commit.clone();
4851 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4852 Ok(Some(e)) => e,
4853 Ok(None) => return Ok(false),
4854 Err(e) => {
4855 tracing::warn!("already-in-base check for {branch}: {e:#}");
4856 return Ok(false);
4857 }
4858 };
4859 let mut verified = vec![head.to_owned()];
4860 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4861 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4862 if matches!(&fetched, Ok(o) if o.ok())
4863 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4864 && theirs != head
4865 {
4866 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4867 Ok(Some(_)) => verified.push(theirs),
4868 _ => return Ok(false),
4869 }
4870 }
4871 let base_branch = self.state.base_branch.clone();
4872 let message = format!(
4873 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4874 land",
4875 evidence.names(),
4876 evidence.proof.as_str()
4877 );
4878 let closed =
4879 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4880 .await;
4881 match closed {
4882 Ok(Ok(url)) => self
4883 .state
4884 .event("land", format!("closed {url}: superseded on {base_branch}")),
4885 Ok(Err(why)) => self
4886 .state
4887 .event("land", format!("did not close a pull request: {why}")),
4888 Err(e) => {
4889 let why = format!(
4890 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4891 not be closed ({e:#}); resume to retry"
4892 );
4893 self.state.status = RunStatus::Blocked;
4894 self.state.base_sync = Some(BaseSync {
4895 tip: tip.to_owned(),
4896 behind,
4897 attempts,
4898 conflict: Some(why.clone()),
4899 already_in: None,
4900 });
4901 self.state.event("land", why);
4902 self.state.save()?;
4903 return Ok(true);
4904 }
4905 }
4906 self.state.status = RunStatus::AlreadyInBase;
4907 self.state.base_sync = Some(BaseSync {
4908 tip: tip.to_owned(),
4909 behind,
4910 attempts,
4911 conflict: None,
4912 already_in: Some(evidence),
4913 });
4914 self.state.event("land", message);
4915 self.state.save()?;
4916 self.settle_questions();
4917 Ok(true)
4918 }
4919
4920 fn landing_base(&self) -> String {
4930 self.state
4931 .base_sync
4932 .as_ref()
4933 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4934 }
4935
4936 pub async fn fix_selected(
4969 &mut self,
4970 ids: &[String],
4971 reason: &str,
4972 allow_stale: bool,
4973 ) -> Result<()> {
4974 let reason = reason.trim();
4975 if reason.is_empty() {
4976 bail!("a fix request needs a reason — that is the operator's own record of why");
4977 }
4978 if ids.is_empty() {
4979 bail!("no finding id given");
4980 }
4981 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
4982 bail!(
4983 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
4984 has already concluded — can be given a targeted fix. A run still \
4985 in progress should simply be resumed; a `merged` run's branch has \
4986 already landed, so its answer is a fresh `magi review <branch>`, \
4987 not reopening this run's own record",
4988 self.state.id,
4989 self.state.status.as_str()
4990 );
4991 }
4992 let Some(winner) = self.state.winner().cloned() else {
4993 bail!("run {} has no winning candidate to fix", self.state.id);
4994 };
4995 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
4996 bail!(
4997 "branch `{}` no longer exists; this run cannot be extended",
4998 winner.branch
4999 );
5000 }
5001 let home = crate::run::home();
5002 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5003 bail!(
5004 "run {} is currently being worked on by another magi process",
5005 self.state.id
5006 );
5007 }
5008 let _claim = FixClaim::acquire(&self.state.dir())?;
5014
5015 let mut seen = BTreeSet::new();
5019 let mut findings = Vec::new();
5020 let mut missing = Vec::new();
5021 for id in ids {
5022 if !seen.insert(id.clone()) {
5023 continue;
5024 }
5025 match self.state.finding(id) {
5026 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5027 id: f.id.clone(),
5028 severity: f.severity,
5029 reviewer_vote: rec.vote,
5030 round: round.round,
5031 round_head: round.head.clone(),
5032 reviewer: rec.reviewer,
5033 agent: rec.agent.clone(),
5034 file: f.file.clone(),
5035 line: f.line,
5036 title: f.title.clone(),
5037 detail: f.detail.clone(),
5038 outcome: OperatorFixOutcome::Pending,
5039 }),
5040 None => missing.push(id.clone()),
5041 }
5042 }
5043 if !missing.is_empty() {
5044 bail!(
5045 "unknown finding id(s): {}; nothing was changed",
5046 missing.join(", ")
5047 );
5048 }
5049
5050 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5051 let stale_details: Vec<(String, String)> = findings
5052 .iter()
5053 .filter(|f| f.round_head != head_at_request)
5054 .map(|f| (f.id.clone(), f.round_head.clone()))
5055 .collect();
5056 let stale = !stale_details.is_empty();
5057 if stale && !allow_stale {
5058 bail!(
5059 "the branch has moved since some finding(s) were raised — {} — now \
5060 at {}; pass --allow-stale to fix anyway, or re-run review first",
5061 stale_details
5062 .iter()
5063 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5064 .collect::<Vec<_>>()
5065 .join(", "),
5066 short(&head_at_request)
5067 );
5068 }
5069
5070 let request = OperatorFixRequest {
5071 requested_at: Timestamp::now(),
5072 reason: reason.to_owned(),
5073 findings,
5074 head_at_request: head_at_request.clone(),
5075 allow_stale,
5076 stale,
5077 fix: None,
5078 result_head: None,
5079 follow_up_review_run: None,
5080 };
5081 self.state.event(
5082 "fix",
5083 format!(
5084 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5085 request.findings.len(),
5086 request
5087 .findings
5088 .iter()
5089 .map(|f| f.id.as_str())
5090 .collect::<Vec<_>>()
5091 .join(", "),
5092 ),
5093 );
5094 self.state.operator_fixes.push(request);
5101 self.state.save()?;
5102 let request_index = self.state.operator_fixes.len() - 1;
5103
5104 let repo = self.state.repo.clone();
5113 let coordination = self.coordinate(&repo).await?;
5114 if winner.worktree.exists() {
5115 let dirty = git::git(
5118 &winner.worktree,
5119 &["status", "--porcelain", "--untracked-files=all"],
5120 )
5121 .await?;
5122 let only_withheld = dirty.lines().all(|l| {
5123 l.strip_prefix("?? ")
5124 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5125 });
5126 if !only_withheld {
5127 bail!(
5128 "`{}` has uncommitted changes; refusing to touch it — commit or \
5129 discard them first",
5130 winner.worktree.display()
5131 );
5132 }
5133 git::worktree_remove(&self.state.repo, &winner.worktree)
5134 .await
5135 .ok();
5136 }
5137 let fix_worktree = self.state.worktree_root().join("operator-fix");
5138 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5139 git::git(
5140 &self.state.repo,
5141 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5142 )
5143 .await
5144 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5145 if !git::is_clean(&fix_worktree).await? {
5146 git::worktree_remove(&self.state.repo, &fix_worktree)
5147 .await
5148 .ok();
5149 bail!(
5150 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5151 winner.branch
5152 );
5153 }
5154 let repo_for_setup = self.state.repo.clone();
5155 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5156 Ok(()) => {
5157 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5158 }
5159 Err(e) => Err(e),
5160 };
5161 if let Err(e) = setup {
5162 git::worktree_remove(&self.state.repo, &fix_worktree)
5163 .await
5164 .ok();
5165 return Err(e);
5166 }
5167
5168 let run_id = self.state.id.clone();
5169 let prompts = self.state.config.prompts.clone();
5170 let language = self.state.config.graph.language.clone();
5171 let sessions = self.state.config.graph.sessions;
5172 let artifacts = agent::artifacts_dir(&self.state.dir());
5173 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5174 .findings
5175 .iter()
5176 .map(|f| Finding {
5177 id: f.id.clone(),
5178 severity: f.severity,
5179 file: f.file.clone(),
5180 line: f.line,
5181 title: f.title.clone(),
5182 detail: f.detail.clone(),
5183 })
5184 .collect();
5185 let fix_prompt = prompt::operator_fix_in(
5186 &self.state.instruction,
5187 &finding_list,
5188 reason,
5189 &stale_details,
5190 &head_at_request,
5191 &language,
5192 &self.state.config.graph.github_language,
5193 );
5194 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5195 drop(coordination);
5196 let (job, seat, out) = self
5197 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5198 prompt: fix_prompt.clone(),
5199 spec,
5200 seat,
5201 cwd: fix_worktree.clone(),
5202 timeout,
5203 allow_write: true,
5204 sessions,
5205 artifacts: artifacts.clone(),
5206 stem: "operator-fix".to_owned(),
5207 handover: None,
5208 })
5209 .await;
5210 let agent_id = seat.agent.clone();
5211
5212 let mut fix = FixRecord {
5213 agent: agent_id,
5214 addressed: Vec::new(),
5215 rejected: Vec::new(),
5216 notes: String::new(),
5217 committed: false,
5218 failed: None,
5219 duration_ms: 0,
5220 continuation: None,
5221 };
5222 let mut final_seat = seat.clone();
5223 match out {
5224 AgentOutcome::Ok(o) => {
5225 fix.duration_ms = o.duration_ms;
5226 let parsed = verdict::extract_json::<FixReport>(&o.text);
5227 let incomplete_reason = match &parsed {
5228 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5229 "the reply parsed, but it reported a command whose own CLI \
5230 never confirmed an exit status"
5231 .to_owned(),
5232 ),
5233 Ok(_) => None,
5234 Err(e) => Some(e.to_string()),
5235 };
5236 match incomplete_reason {
5237 None => {
5238 let report = parsed.expect("checked Ok above");
5239 fix.addressed = report.addressed;
5240 fix.rejected = report.rejected;
5241 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5242 }
5243 Some(reason) => {
5244 let (resumed_seat, resolved, failure, cont) = self
5245 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5246 .await;
5247 fix.duration_ms += cont.cumulative_wait_ms;
5248 fix.continuation = Some(cont);
5249 final_seat = resumed_seat;
5250 match resolved {
5251 Some(report) => {
5252 fix.addressed = report.addressed;
5253 fix.rejected = report.rejected;
5254 fix.notes =
5255 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5256 }
5257 None => fix.failed = failure,
5258 }
5259 }
5260 }
5261 }
5262 AgentOutcome::Dropped(o) => {
5263 fix.duration_ms = o.duration_ms;
5264 let why = o
5265 .dropped
5266 .as_ref()
5267 .map(|d| d.why.as_str())
5268 .unwrap_or("the CLI ended the stream without delivering its answer");
5269 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5270 }
5271 AgentOutcome::Quota(o) => {
5272 self.state.quota.push(QuotaLoss {
5273 seat: final_seat.key.clone(),
5274 node: "fix".to_owned(),
5275 at: Timestamp::now(),
5276 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5277 });
5278 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5279 }
5280 AgentOutcome::Failed(e) => fix.failed = Some(e),
5281 AgentOutcome::ResourceBlocked(e) => {
5282 fix.failed = Some(format!("the fixer was not started: {e}"));
5283 }
5284 }
5285 if fix.continuation.is_none() {
5286 fix.continuation = Some(ContinuationRecord::not_needed());
5287 }
5288 self.state.seats.insert(final_seat.key.clone(), final_seat);
5289
5290 let rescue_message = format!(
5291 "magi: operator-selected fix ({}) (uncommitted work)",
5292 self.state.operator_fixes[request_index]
5293 .findings
5294 .iter()
5295 .map(|f| f.id.as_str())
5296 .collect::<Vec<_>>()
5297 .join(", ")
5298 );
5299 let repo = self.state.repo.clone();
5300 let coordination = self.coordinate(&repo).await?;
5301 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5302 self.state.note_withheld("fix", &r.withheld);
5303 }
5304 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5305 fix.committed = after != head_at_request;
5306 git::worktree_remove(&self.state.repo, &fix_worktree)
5307 .await
5308 .ok();
5309 drop(coordination);
5310
5311 self.state.event(
5312 "fix",
5313 match &fix.failed {
5314 Some(reason) => format!(
5315 "operator fix: adoption report was lost ({reason}); {}",
5316 if fix.committed {
5317 "committed"
5318 } else {
5319 "NO new commit"
5320 }
5321 ),
5322 None => format!(
5323 "operator fix: {} addressed, {} rejected, {}",
5324 fix.addressed.len(),
5325 fix.rejected.len(),
5326 if fix.committed {
5327 "committed"
5328 } else {
5329 "NO new commit"
5330 }
5331 ),
5332 },
5333 );
5334
5335 for f in &mut self.state.operator_fixes[request_index].findings {
5342 f.outcome = if fix.failed.is_some() {
5343 OperatorFixOutcome::Unreported
5344 } else if fix.addressed.contains(&f.id) {
5345 OperatorFixOutcome::Addressed
5346 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5347 OperatorFixOutcome::Rejected { why: r.why.clone() }
5348 } else {
5349 OperatorFixOutcome::Unreported
5350 };
5351 }
5352
5353 let committed = fix.committed;
5354 if committed {
5355 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5356 }
5357 self.state.operator_fixes[request_index].fix = Some(fix);
5358 self.state.save()?;
5361
5362 if committed {
5363 self.state.event(
5364 "fix",
5365 format!(
5366 "operator fix committed {}; opening a follow-up review-only run",
5367 short(&after)
5368 ),
5369 );
5370 let origin =
5373 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5374 match Self::review(
5375 &self.state.repo,
5376 &winner.branch,
5377 self.state.config.clone(),
5378 origin,
5379 )
5380 .await
5381 {
5382 Ok(mut follow_up) => {
5383 follow_up.state.event(
5384 "start",
5385 format!(
5386 "requested by an operator fix on run {} for finding(s) {}",
5387 self.state.id,
5388 self.state.operator_fixes[request_index]
5389 .findings
5390 .iter()
5391 .map(|f| f.id.as_str())
5392 .collect::<Vec<_>>()
5393 .join(", "),
5394 ),
5395 );
5396 follow_up.state.save()?;
5397 let follow_up_id = follow_up.state.id.clone();
5398 let adopted = match crate::direct::adopt(
5401 &follow_up.state,
5402 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5403 ) {
5404 Ok(a) => a,
5405 Err(e) => {
5406 self.state.event(
5409 "fix",
5410 format!(
5411 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5412 ),
5413 );
5414 self.state.save()?;
5415 return Ok(());
5416 }
5417 };
5418 let executed = follow_up.execute().await;
5419 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5420 if let Some(a) = adopted {
5421 a.finish(&follow_up.state, executed);
5422 }
5423 if let Some(e) = failure {
5424 self.state.event(
5425 "fix",
5426 format!(
5427 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5428 ),
5429 );
5430 }
5431 self.state.operator_fixes[request_index].follow_up_review_run =
5432 Some(follow_up_id);
5433 }
5434 Err(e) => {
5435 self.state.event(
5436 "fix",
5437 format!("committed the fix but could not open a follow-up review: {e:#}"),
5438 );
5439 }
5440 }
5441 self.state.save()?;
5442 }
5443
5444 Ok(())
5445 }
5446
5447 async fn ask_fixer(
5467 &mut self,
5468 winner: &Candidate,
5469 node: &'static str,
5470 round: Option<usize>,
5471 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5472 ) -> (SeatJob, SeatState, AgentOutcome) {
5473 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5474 self.ask_fixer_among(attempts, node, round, None, build)
5475 .await
5476 }
5477
5478 async fn ask_fixer_among(
5483 &mut self,
5484 attempts: Vec<(AgentSpec, String)>,
5485 node: &'static str,
5486 round: Option<usize>,
5487 restore: Option<(&Path, &str)>,
5488 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5489 ) -> (SeatJob, SeatState, AgentOutcome) {
5490 let run_id = self.state.id.clone();
5491 let prompts = self.state.config.prompts.clone();
5492 let cache = self.state.config.cache_dir();
5493 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5494 let mut last = None;
5495 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5496 let seat = self.seat(&key, &spec.id);
5497 let mut job = build(spec.clone(), seat);
5498 if i > 0 {
5499 job.stem = format!("{}-{}", job.stem, spec.id);
5500 }
5501 let ctx = WaveCtx {
5502 carry_seats: false,
5503 pauses: Some((&self.pause, &self.interrupt)),
5504 run: &run_id,
5505 node,
5506 prompts: &prompts,
5507 cache: cache.as_deref(),
5508 round,
5509 };
5510 let (seat, out) =
5511 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5512 let advances = match &out {
5515 AgentOutcome::Ok(o) => agent::output_advances(o),
5516 AgentOutcome::ResourceBlocked(_) => false,
5517 _ => true,
5518 };
5519 if let Some(next) = ids.get(i + 1)
5520 && advances
5521 {
5522 self.state.seats.insert(seat.key.clone(), seat.clone());
5523 let class =
5524 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5525 let reason = match &out {
5526 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5527 other => fail_reason(other),
5528 };
5529 if let Some((wt, rev)) = restore
5533 && let Err(e) = self.graft_restore(wt, rev).await
5534 {
5535 let why =
5536 format!("could not restore the worktree before the next fixer: {e:#}");
5537 tracing::warn!("{why}");
5538 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5539 AgentOutcome::ResourceBlocked(why)
5540 } else {
5541 AgentOutcome::Failed(why)
5542 };
5543 last = Some((job, seat, out));
5544 break;
5545 }
5546 record_handover(
5547 &mut self.state,
5548 node,
5549 &seat.key,
5550 &spec.id,
5551 next,
5552 &class,
5553 &reason,
5554 );
5555 continue;
5556 }
5557 last = Some((job, seat, out));
5558 break;
5559 }
5560 last.expect("the fixer chain always has an entry")
5561 }
5562
5563 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5574 use crate::recovery::{Mode, Repair};
5575 if !self.state.reviews.is_empty() {
5576 return Ok(());
5577 }
5578 let Some(mut rec) = self.state.recovery.clone() else {
5579 return Ok(());
5580 };
5581 if rec.mode != Mode::VerifyRepair {
5582 return Ok(());
5583 }
5584 let wt = winner.worktree.clone();
5585 let mut head = git::rev_parse(&wt, "HEAD").await?;
5586 match rec.repair.clone() {
5587 Repair::Done { .. } => return Ok(()),
5588 Repair::WaitingForRestore { before } => {
5589 self.graft_restore(&wt, &before).await?;
5590 head = before;
5591 rec.repair = Repair::NotStarted;
5592 self.state.recovery = Some(rec.clone());
5593 self.state.save()?;
5594 }
5595 Repair::Started { before } => {
5596 if let Err(e) = self.graft_restore(&wt, &before).await {
5600 if e.is::<crate::repo_lock::ResourceBlocked>() {
5601 return Err(e);
5602 }
5603 return self
5604 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5605 .await;
5606 }
5607 rec.repair = Repair::Done {
5608 before: before.clone(),
5609 after: before,
5610 committed: false,
5611 failed: Some("interrupted; rolled back".to_owned()),
5612 };
5613 self.state
5614 .event("recovery", "the repair was interrupted and rolled back");
5615 self.state.recovery = Some(rec);
5616 self.state.save()?;
5617 return Ok(());
5618 }
5619 Repair::NotStarted => {}
5620 }
5621 if crate::recovery::recheck(&mut rec, &head) {
5622 self.state.event("recovery", rec.summary());
5623 self.state.recovery = Some(rec);
5624 self.state.save()?;
5625 return Ok(());
5626 }
5627 rec.repair = Repair::Started {
5628 before: head.clone(),
5629 };
5630 self.state.recovery = Some(rec.clone());
5631 self.state.save()?;
5632
5633 let language = self.state.config.graph.language.clone();
5634 let sessions = self.state.config.graph.sessions;
5635 let artifacts = agent::artifacts_dir(&self.state.dir());
5636 let fix_prompt = prompt::recovery_fix(
5637 &self.state.instruction,
5638 &rec,
5639 self.state.config.graph.review_rounds,
5640 &language,
5641 );
5642 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5643 self.state.event(
5644 "recovery",
5645 format!(
5646 "repair: the fixer starts from the failed verification on {}",
5647 short(&head)
5648 ),
5649 );
5650 let (_job, seat, out) = self
5651 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5652 prompt: fix_prompt.clone(),
5653 spec,
5654 seat,
5655 cwd: wt.clone(),
5656 timeout,
5657 allow_write: true,
5658 sessions,
5659 artifacts: artifacts.clone(),
5660 stem: "fix-recovery".to_owned(),
5661 handover: None,
5662 })
5663 .await;
5664 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5665 rec.repair = Repair::WaitingForRestore { before: head };
5666 self.state.recovery = Some(rec);
5667 return self.stop_for_resource();
5668 }
5669 let failed = match &out {
5670 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5671 AgentOutcome::Quota(o) => {
5672 self.state.quota.push(QuotaLoss {
5673 seat: seat.key.clone(),
5674 node: "fix".to_owned(),
5675 at: Timestamp::now(),
5676 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5677 });
5678 Some("rate limited (quota); the fixer could not run".to_owned())
5679 }
5680 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5681 AgentOutcome::Failed(e) => Some(e.clone()),
5682 };
5683 self.state.seats.insert(seat.key.clone(), seat);
5684 let coordination = self.coordinate(&wt).await?;
5685 let failed = match failed {
5686 Some(why) => {
5688 if let Err(e) = git::restore_to(&wt, &head).await {
5689 return self
5690 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5691 .await;
5692 }
5693 Some(why)
5694 }
5695 None => {
5696 if let Ok(r) = git::rescue_commit(
5697 &wt,
5698 "magi: recovery repair of the failed verification (uncommitted work)",
5699 )
5700 .await
5701 {
5702 self.state.note_withheld("fix", &r.withheld);
5703 }
5704 None
5705 }
5706 };
5707 let after = git::rev_parse(&wt, "HEAD").await?;
5708 drop(coordination);
5709 self.state.event(
5710 "recovery",
5711 match &failed {
5712 Some(why) => format!("repair: nothing kept ({why})"),
5713 None if after != head => {
5714 format!("repair: committed {}", short(&after))
5715 }
5716 None => "repair: the fixer made no commit".to_owned(),
5717 },
5718 );
5719 rec.repair = Repair::Done {
5720 before: head.clone(),
5721 committed: after != head,
5722 after,
5723 failed,
5724 };
5725 self.state.recovery = Some(rec);
5726 self.state.save()?;
5727 Ok(())
5728 }
5729
5730 async fn stop_recovery(
5733 &mut self,
5734 mut rec: crate::recovery::RecoveryRecord,
5735 head: &str,
5736 why: String,
5737 ) -> Result<()> {
5738 rec.repair = crate::recovery::Repair::Started {
5740 before: head.to_owned(),
5741 };
5742 self.state.recovery = Some(rec);
5743 self.state
5744 .event("recovery", format!("repair could not be undone: {why}"));
5745 self.state.status = RunStatus::Blocked;
5746 self.state.save()?;
5747 bail!("recovery repair could not be rolled back: {why}")
5748 }
5749
5750 async fn review_loop(&mut self) -> Result<()> {
5751 if self
5756 .state
5757 .base_sync
5758 .as_ref()
5759 .is_some_and(|s| s.conflict.is_some())
5760 {
5761 return Ok(());
5762 }
5763 let run_id = self.state.id.clone();
5768 let prompts = self.state.config.prompts.clone();
5769 let Some(winner) = self.state.winner().cloned() else {
5770 return Ok(());
5771 };
5772 let max_rounds = self.state.config.graph.review_rounds;
5773 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5783 if status == RunStatus::Gating {
5786 self.record_contested_handoff();
5787 }
5788 self.state.status = status;
5789 self.state.save()?;
5790 return Ok(());
5791 }
5792 self.state.status = RunStatus::Reviewing;
5793 if self
5803 .state
5804 .reviews
5805 .last()
5806 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5807 {
5808 let shell = self.state.config.shell();
5809 return self
5810 .stop_reviewing(
5811 "the last round's own verification never resolved",
5812 &shell,
5813 &winner.worktree,
5814 )
5815 .await;
5816 }
5817
5818 self.recovery_repair(&winner).await?;
5819 if self.resource_halt()? {
5820 return Ok(());
5821 }
5822
5823 let repo = self.state.repo.clone();
5824 let root = self.state.worktree_root();
5825 let language = self.state.config.graph.language.clone();
5826 let sessions = self.state.config.graph.sessions;
5827 let artifacts = self.merge_repair_artifacts();
5828 let base = self.landing_base();
5829 let base_short = short(&base);
5830 let reviewers = self.roles.reviewers.clone();
5831 let shell = self.state.config.shell();
5832
5833 for round in (self.state.reviews.len() + 1)..=max_rounds {
5834 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5835 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5836 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5837 let prev_verification = self
5846 .state
5847 .reviews
5848 .last()
5849 .and_then(|r| r.verification_summary(&head));
5850 let prior_section = match (&self.state.recovery, round) {
5853 (Some(rec), 1) if self.state.reviews.is_empty() => {
5854 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5855 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5856 Some(crate::recovery::head_of(
5857 &git::diff(&winner.worktree, reviewed, "HEAD")
5858 .await
5859 .unwrap_or_default(),
5860 crate::recovery::SINCE_DIFF_MAX,
5861 ))
5862 }
5863 _ => None,
5864 };
5865 Some(prompt::recovery_review(rec, since.as_deref()))
5866 }
5867 _ => None,
5868 };
5869
5870 let mut jobs = Vec::new();
5874 let coordination = self.coordinate(&repo).await?;
5875 for (r, spec) in reviewers.iter().cloned().enumerate() {
5876 let wt = root.join(format!("review-{}", r + 1));
5877 if wt.exists() {
5880 git::reset_detached(&wt, &head).await?;
5881 } else {
5882 git::worktree_add_detached(&repo, &wt, &head).await?;
5883 }
5884 self.setup_seat_worktree(&repo, &wt, false).await?;
5885 let seat_key = format!("review-{}", r + 1);
5886 let spec = pick_start_spec(
5889 &self.roles.reviewer_roster,
5890 spec,
5891 self.state.seat_history.get(&seat_key),
5892 );
5893 let seat = self.seat(&seat_key, &spec.id);
5894 jobs.push(SeatJob {
5895 prompt: prompt::review(&prompt::ReviewCtx {
5896 instruction: &self.state.instruction,
5897 branch: &winner.branch,
5898 base_short: &base_short,
5899 stat: &stat,
5900 patch: &patch,
5901 verification: prev_verification.as_ref(),
5902 prior: prior_section.as_deref(),
5903 reviewers: reviewers.len(),
5904 round,
5905 rounds: max_rounds,
5906 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5909 lens: Lens::for_seat(r),
5910 language: &language,
5911 github_language: &self.state.config.graph.github_language,
5912 }),
5913 spec,
5914 seat,
5915 cwd: wt,
5916 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5917 allow_write: false,
5918 sessions,
5919 artifacts: artifacts.clone(),
5920 stem: format!("review-{round}-{}", r + 1),
5921 handover: None,
5922 });
5923 }
5924
5925 drop(coordination);
5926 self.state.event(
5927 "review",
5928 format!(
5929 "round {round}: {} reviewers on {}",
5930 jobs.len(),
5931 short(&head)
5932 ),
5933 );
5934 let mut quota_losses = Vec::new();
5935 let review_retries = self.state.config.graph.retries;
5936 let review_cache = self.state.config.cache_dir();
5937 let ctx = WaveCtx {
5938 carry_seats: true,
5939 pauses: Some((&self.pause, &self.interrupt)),
5940 run: &run_id,
5941 node: "review",
5942 prompts: &prompts,
5943 cache: review_cache.as_deref(),
5944 round: Some(round),
5945 };
5946 let results = ask_json_wave::<Review>(
5947 jobs,
5948 Arc::clone(&self.sem),
5949 review_retries,
5950 &self.roles.reviewer_roster,
5951 &ctx,
5952 &mut quota_losses,
5953 &mut self.state,
5954 &|_: &Review| Ok(()),
5955 )
5956 .await;
5957 if self.state.resource_block.is_some() {
5958 return self.stop_for_resource();
5959 }
5960 let round_quota_missing = quota_losses.len();
5964 self.state.quota.extend(quota_losses);
5965
5966 let mut records = Vec::new();
5967 let mut all_findings = Vec::new();
5968 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
5969 let agent_id = seat.agent.clone();
5970 self.state.seats.insert(seat.key.clone(), seat);
5971 let mut record = ReviewRecord {
5972 reviewer: r + 1,
5973 agent: agent_id,
5974 summary: String::new(),
5975 findings: Vec::new(),
5976 vote: None,
5977 failed: None,
5978 duration_ms: 0,
5979 attempts,
5985 };
5986 match res {
5987 Ok((review, out)) => {
5988 record.summary =
5996 blind::sanitize_prose(&review.summary, &self.state.config.blind);
5997 record.vote = Some(review.vote);
5998 record.duration_ms = out.duration_ms;
5999 for (n, mut f) in review.findings.into_iter().enumerate() {
6000 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6003 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6004 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6005 f.file = f
6011 .file
6012 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6013 all_findings.push(f.clone());
6014 record.findings.push(f);
6015 }
6016 self.state.event(
6017 "review",
6018 format!(
6019 "round {round}: reviewer {} voted {} with {} finding(s)",
6020 r + 1,
6021 review.vote.label(),
6022 record.findings.len()
6023 ),
6024 );
6025 }
6026 Err(e) => {
6027 record.failed = Some(e.to_string());
6028 self.state.event(
6029 "review",
6030 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6031 );
6032 }
6033 }
6034 records.push(record);
6035 }
6036
6037 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6044 let vote_split =
6045 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6046 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6047 if vote_split {
6048 self.state.event(
6049 "review",
6050 format!(
6051 "round {round}: votes split ({}) — one round of reconsideration",
6052 initial_votes
6053 .iter()
6054 .map(|v| v.label())
6055 .collect::<Vec<_>>()
6056 .join(", ")
6057 ),
6058 );
6059 let panel: Vec<ReviewSeatReport<'_>> = records
6062 .iter()
6063 .filter_map(|r| {
6064 r.vote.map(|vote| ReviewSeatReport {
6065 reviewer: r.reviewer,
6066 vote,
6067 summary: &r.summary,
6068 findings: &r.findings,
6069 })
6070 })
6071 .collect();
6072
6073 let mut jobs = Vec::new();
6074 let mut seats_at = Vec::new();
6075 for (r, spec) in reviewers.iter().cloned().enumerate() {
6076 if records[r].vote.is_none() {
6080 continue;
6081 }
6082 let wt = root.join(format!("review-{}", r + 1));
6083 let seat_key = format!("review-{}", r + 1);
6084 let spec = self.occupant(&seat_key, spec);
6085 let seat = self.seat(&seat_key, &spec.id);
6086 let build = |with_patch: bool| {
6094 prompt::review_reconsider(&ReviewReconsiderCtx {
6095 instruction: &self.state.instruction,
6096 reviewer: r + 1,
6097 lens: Lens::for_seat(r),
6098 panel: &panel,
6099 patch: with_patch.then_some(ReviewPatch {
6100 branch: &winner.branch,
6101 base_short: &base_short,
6102 stat: &stat,
6103 patch: &patch,
6104 }),
6105 round,
6106 rounds: max_rounds,
6107 language: &language,
6108 })
6109 };
6110 let full = build(true);
6111 let prompt = if has_context(&spec, &seat, sessions) {
6112 build(false)
6113 } else {
6114 full.clone()
6115 };
6116 jobs.push(SeatJob {
6117 prompt,
6118 spec,
6119 seat,
6120 cwd: wt,
6121 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6122 allow_write: false,
6123 sessions,
6124 artifacts: artifacts.clone(),
6125 stem: format!("review-{round}-reconsider-{}", r + 1),
6126 handover: Some(full),
6127 });
6128 seats_at.push(r);
6129 }
6130
6131 let mut recon_quota_losses = Vec::new();
6132 let recon_cache = self.state.config.cache_dir();
6133 let recon_ctx = WaveCtx {
6134 carry_seats: true,
6135 pauses: Some((&self.pause, &self.interrupt)),
6136 run: &run_id,
6137 node: "review",
6138 prompts: &prompts,
6139 cache: recon_cache.as_deref(),
6140 round: Some(round),
6141 };
6142 let recon_results = ask_json_wave::<ReviewRevote>(
6143 jobs,
6144 Arc::clone(&self.sem),
6145 review_retries,
6146 &self.roles.reviewer_roster,
6147 &recon_ctx,
6148 &mut recon_quota_losses,
6149 &mut self.state,
6150 &|_: &ReviewRevote| Ok(()),
6151 )
6152 .await;
6153 if self.state.resource_block.is_some() {
6154 return self.stop_for_resource();
6155 }
6156 self.state.quota.extend(recon_quota_losses);
6157
6158 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6159 let agent_id = seat.agent.clone();
6160 self.state.seats.insert(seat.key.clone(), seat);
6161 let mut rec = ReviewRevoteRecord {
6162 reviewer: r + 1,
6163 agent: agent_id,
6164 vote: None,
6165 reason: String::new(),
6166 failed: None,
6167 };
6168 match res {
6169 Ok((rv, _)) => {
6170 rec.vote = Some(rv.vote);
6171 rec.reason =
6172 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6173 self.state.event(
6174 "review",
6175 format!(
6176 "round {round}: reviewer {} revoted {}",
6177 r + 1,
6178 rv.vote.label()
6179 ),
6180 );
6181 }
6182 Err(e) => {
6183 rec.failed = Some(e.to_string());
6184 self.state.event(
6185 "review",
6186 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6187 );
6188 }
6189 }
6190 reconsideration.push(rec);
6191 }
6192 } else if initial_votes.len() > 1 {
6193 self.state.event(
6194 "review",
6195 format!(
6196 "round {round}: votes agreed ({}) — no reconsideration",
6197 initial_votes[0].label()
6198 ),
6199 );
6200 }
6201
6202 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6203 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6204 let defer_e2e =
6215 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6216 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6217 let reason =
6218 format!("{blocking} blocking finding(s) already required a fix this round");
6219 self.state.event(
6220 "verify",
6221 format!(
6222 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6223 {}); it will run once a round has none left",
6224 short(&head)
6225 ),
6226 );
6227 (Vec::new(), false, true, Some(reason))
6228 } else {
6229 let e2e_commands = self.state.config.verify.e2e.clone();
6230 let cache_dir = self.state.config.cache_dir();
6231 let context = format!("round {round}");
6232 let wait = self.repository_wait();
6233 let (e2e, verify_retried) = with_cache_lease(
6234 &mut self.state,
6235 cache_dir.as_deref(),
6236 "e2e",
6237 "e2e",
6238 &winner.worktree,
6239 &head,
6240 verify_timeout,
6241 &context,
6242 &wait,
6243 |state, budget| {
6244 let shell = shell.clone();
6245 let e2e_commands = e2e_commands.clone();
6246 let worktree = winner.worktree.clone();
6247 let context = context.clone();
6248 async move {
6249 run_e2e_with_retry(
6250 state,
6251 &shell,
6252 &e2e_commands,
6253 &worktree,
6254 budget,
6255 &context,
6256 )
6257 .await
6258 }
6259 },
6260 )
6261 .await?;
6262 (e2e, verify_retried, false, None)
6263 };
6264
6265 let expected = records.len();
6266 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6267 let incomplete = answered < expected;
6268 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6269 let policy = self.state.config.graph.incomplete_review;
6270 let clean = round_is_clean(
6271 blocking,
6272 e2e_ok,
6273 answered,
6274 expected,
6275 round_quota_missing,
6276 policy,
6277 );
6278
6279 let mut round_record = ReviewRound {
6280 round,
6281 head: head.clone(),
6282 verified_head: None,
6283 verified_at: None,
6284 reviews: records,
6285 e2e,
6286 verify_retried,
6287 e2e_deferred,
6288 e2e_defer_reason,
6289 fix: None,
6290 blocking,
6291 answered,
6292 expected,
6293 clean,
6294 progressed: false,
6295 vote_split,
6296 reconsideration,
6297 verdict: None,
6298 };
6299 round_record.verdict = ReviewVote::worst(
6303 round_record
6304 .final_votes()
6305 .into_iter()
6306 .map(|(_, _, vote)| vote),
6307 );
6308 if !matches!(
6319 round_record.e2e_status(),
6320 E2eStatus::Deferred | E2eStatus::NotConfigured
6321 ) {
6322 round_record.verified_head = Some(head.clone());
6323 round_record.verified_at = Some(Timestamp::now());
6324 }
6325 let this_round_verification = round_record.verification_summary(&head);
6326
6327 if incomplete {
6328 let missing: Vec<String> = round_record
6329 .reviews
6330 .iter()
6331 .filter(|r| r.failed.is_some())
6332 .map(|r| format!("review-{}", r.reviewer))
6333 .collect();
6334 self.state.event(
6335 "review",
6336 format!(
6337 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6338 missing.join(", ")
6339 ),
6340 );
6341 }
6342
6343 if clean {
6344 self.state.event(
6345 "review",
6346 if incomplete && policy == IncompleteReviewPolicy::Warn {
6347 format!(
6348 "round {round}: clean (warn policy, incomplete panel) — no \
6349 blocking findings from the seats that answered, verification green"
6350 )
6351 } else if incomplete {
6352 format!(
6353 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6354 quorum) — no blocking findings from the seats that answered, \
6355 verification green",
6356 expected - answered
6357 )
6358 } else {
6359 format!("round {round}: clean — no blocking findings, verification green")
6360 },
6361 );
6362 self.state.reviews.push(round_record);
6363 self.state.status = RunStatus::Gating;
6364 self.state.save()?;
6365 return Ok(());
6366 }
6367
6368 if incomplete && blocking == 0 && e2e_ok {
6376 self.state.reviews.push(round_record);
6377 self.state.save()?;
6378 if round == max_rounds {
6379 self.state.status = RunStatus::Blocked;
6380 self.state.event(
6381 "review",
6382 format!(
6383 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6384 refusing to call it clean",
6385 expected - answered
6386 ),
6387 );
6388 return Ok(());
6389 }
6390 continue;
6391 }
6392
6393 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6405 self.state.reviews.push(round_record);
6406 return self
6407 .stop_reviewing(
6408 "the round's own verification could not run",
6409 &shell,
6410 &winner.worktree,
6411 )
6412 .await;
6413 }
6414
6415 if round == max_rounds {
6416 self.state.reviews.push(round_record);
6417 return self
6418 .stop_reviewing(
6419 &format!(
6420 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6421 ),
6422 &shell,
6423 &winner.worktree,
6424 )
6425 .await;
6426 }
6427
6428 let blocking_findings: Vec<_> = all_findings
6431 .iter()
6432 .filter(|f| f.severity.blocks())
6433 .cloned()
6434 .collect();
6435 let fix_prompt = prompt::fix_in(
6436 &self.state.instruction,
6437 &blocking_findings,
6438 this_round_verification.as_ref(),
6439 round,
6440 max_rounds,
6441 &language,
6442 &self.state.config.graph.github_language,
6443 );
6444 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6445 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6446 let (job, seat, out) = self
6447 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6448 prompt: fix_prompt.clone(),
6449 spec,
6450 seat,
6451 cwd: winner.worktree.clone(),
6452 timeout,
6453 allow_write: true,
6454 sessions,
6455 artifacts: artifacts.clone(),
6456 stem: format!("fix-{round}"),
6457 handover: None,
6458 })
6459 .await;
6460 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6463 return self.stop_for_resource();
6464 }
6465 let agent_id = seat.agent.clone();
6466
6467 let mut fix = FixRecord {
6468 agent: agent_id,
6469 addressed: Vec::new(),
6470 rejected: Vec::new(),
6471 notes: String::new(),
6472 committed: false,
6473 failed: None,
6474 duration_ms: 0,
6475 continuation: None,
6476 };
6477 let mut continuation = ContinuationRecord::not_needed();
6478 let mut final_seat = seat.clone();
6479 match out {
6480 AgentOutcome::Ok(o) => {
6481 fix.duration_ms = o.duration_ms;
6482 let parsed = verdict::extract_json::<FixReport>(&o.text);
6483 let incomplete_reason = match &parsed {
6490 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6491 "the reply parsed, but it reported a command whose own CLI never \
6492 confirmed an exit status"
6493 .to_owned(),
6494 ),
6495 Ok(_) => None,
6496 Err(e) => Some(e.to_string()),
6497 };
6498 match incomplete_reason {
6499 None => {
6500 let report = parsed.expect("checked Ok above");
6501 fix.addressed = report.addressed;
6502 fix.rejected = report.rejected;
6503 fix.notes =
6504 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6505 }
6506 Some(reason) => {
6507 let (resumed_seat, resolved, failure, cont) = self
6508 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6509 .await;
6510 fix.duration_ms += cont.cumulative_wait_ms;
6511 continuation = cont;
6512 final_seat = resumed_seat;
6513 match resolved {
6514 Some(report) => {
6515 fix.addressed = report.addressed;
6516 fix.rejected = report.rejected;
6517 fix.notes = blind::sanitize_prose(
6518 &report.notes,
6519 &self.state.config.blind,
6520 );
6521 }
6522 None => fix.failed = failure,
6523 }
6524 }
6525 }
6526 }
6527 AgentOutcome::Dropped(o) => {
6529 fix.duration_ms = o.duration_ms;
6530 let why = o
6531 .dropped
6532 .as_ref()
6533 .map(|d| d.why.as_str())
6534 .unwrap_or("the CLI ended the stream without delivering its answer");
6535 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6536 }
6537 AgentOutcome::Quota(o) => {
6538 self.state.quota.push(QuotaLoss {
6539 seat: final_seat.key.clone(),
6540 node: "fix".to_owned(),
6541 at: Timestamp::now(),
6542 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6543 });
6544 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6545 }
6546 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6547 }
6548 fix.continuation = Some(continuation);
6549 self.state.seats.insert(final_seat.key.clone(), final_seat);
6550 let coordination = self.coordinate(&winner.worktree).await?;
6551 if let Ok(r) = git::rescue_commit(
6552 &winner.worktree,
6553 &format!("magi: review round {round} fixes (uncommitted work)"),
6554 )
6555 .await
6556 {
6557 self.state.note_withheld("fix", &r.withheld);
6558 }
6559 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6560 fix.committed = after != before;
6561 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6569 drop(coordination);
6570 let progressed = diff_after != patch;
6571 let commit_note = if fix.committed {
6572 "committed"
6573 } else {
6574 "NO new commit"
6575 };
6576 let tree_note = if progressed {
6577 "changed vs base"
6578 } else {
6579 "unchanged vs base"
6580 };
6581 self.state.event(
6582 "fix",
6583 match &fix.failed {
6584 Some(reason) => {
6590 format!(
6591 "round {round}: fixer's adoption report was lost ({reason}); \
6592 {commit_note}, tree {tree_note}"
6593 )
6594 }
6595 None => format!(
6596 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6597 {tree_note}{}",
6598 fix.addressed.len(),
6599 fix.rejected.len(),
6600 if continuation.outcome == ContinuationOutcome::Resumed {
6601 format!(
6602 " (adoption report recovered after {} continuation(s))",
6603 continuation.attempts
6604 )
6605 } else {
6606 String::new()
6607 },
6608 ),
6609 },
6610 );
6611 round_record.fix = Some(fix);
6612 round_record.progressed = progressed;
6613 self.state.reviews.push(round_record);
6614 self.state.save()?;
6615
6616 if matches!(
6629 continuation.outcome,
6630 ContinuationOutcome::Exhausted
6631 | ContinuationOutcome::QuotaLost
6632 | ContinuationOutcome::NoSession
6633 ) {
6634 return self
6635 .stop_reviewing(
6636 "the fixer's adoption report never came back, even after resuming its \
6637 own seat; refusing to start another round against the same worktree \
6638 while that is unresolved",
6639 &shell,
6640 &winner.worktree,
6641 )
6642 .await;
6643 }
6644
6645 let streak = self
6646 .state
6647 .reviews
6648 .iter()
6649 .rev()
6650 .take_while(|r| !r.progressed)
6651 .count();
6652 if streak >= STAGNANT_LIMIT {
6653 return self
6654 .stop_reviewing(
6655 &format!(
6656 "the tree has not moved against base for {streak} round(s) in a row"
6657 ),
6658 &shell,
6659 &winner.worktree,
6660 )
6661 .await;
6662 }
6663 }
6664 Ok(())
6665 }
6666
6667 fn record_contested_handoff(&mut self) {
6701 if self.state.contested_handoff.is_some() {
6702 return;
6703 }
6704 let Some(contested) = self
6705 .state
6706 .reviews
6707 .last()
6708 .and_then(ReviewRound::contested_handoff)
6709 else {
6710 return;
6711 };
6712 self.state.event(
6713 "review",
6714 format!(
6715 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6716 wait for the owner's approval",
6717 contested.findings.len(),
6718 contested.rejecters.len()
6719 ),
6720 );
6721 self.state.contested_handoff = Some(contested);
6722 }
6723
6724 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6725 let round_idx = self.state.reviews.len() - 1;
6726 let needs_catchup_run = matches!(
6734 self.state.reviews[round_idx].e2e_status(),
6735 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6736 );
6737 if needs_catchup_run {
6738 let round = self.state.reviews[round_idx].round;
6739 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6740 let commands = self.state.config.verify.e2e.clone();
6741 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6742 let cache_dir = self.state.config.cache_dir();
6743 let context = format!(
6744 "round {round}: verification unresolved, catching up before the final decision"
6745 );
6746 let wait = self.repository_wait();
6747 let (outcomes, verify_retried) = with_cache_lease(
6748 &mut self.state,
6749 cache_dir.as_deref(),
6750 "e2e",
6751 "e2e",
6752 worktree,
6753 &attempted_head,
6754 timeout,
6755 &context,
6756 &wait,
6757 |state, budget| {
6758 let shell = shell.to_vec();
6759 let commands = commands.clone();
6760 let context = context.clone();
6761 async move {
6762 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6763 .await
6764 }
6765 },
6766 )
6767 .await?;
6768 let last = &mut self.state.reviews[round_idx];
6769 last.e2e = outcomes;
6770 last.verify_retried = verify_retried;
6771 last.verified_head = Some(attempted_head);
6778 last.verified_at = Some(Timestamp::now());
6779 if verify_inconclusive(&last.e2e) {
6780 self.state.save()?;
6787 return Ok(());
6788 }
6789 last.e2e_deferred = false;
6790 }
6791 let last = &self.state.reviews[round_idx];
6792 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6793
6794 match last.e2e_status() {
6795 E2eStatus::Failed => {
6796 let red: Vec<String> = last
6797 .e2e
6798 .iter()
6799 .filter(|o| !o.ok())
6800 .map(|o| {
6801 format!(
6802 "`{}` -> {:?}\n{}",
6803 o.command,
6804 o.code,
6805 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6806 )
6807 })
6808 .collect();
6809 self.state
6810 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6811 self.state.status = RunStatus::Blocked;
6812 }
6813 E2eStatus::ResourceBlocked => {
6818 self.state.event(
6819 "review",
6820 format!(
6821 "{why}; e2e could not run (shared build cache unavailable); not \
6822 deciding yet"
6823 ),
6824 );
6825 }
6826 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6827 self.state.event(
6828 "review",
6829 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6830 );
6831 self.record_contested_handoff();
6832 self.state.status = RunStatus::Gating;
6833 }
6834 }
6835 self.state.save()?;
6836 Ok(())
6837 }
6838
6839 async fn gate(&mut self) -> Result<()> {
6842 if self.state.status == RunStatus::Failed
6854 || self
6855 .state
6856 .base_sync
6857 .as_ref()
6858 .is_some_and(|s| s.conflict.is_some())
6859 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6860 != Some(RunStatus::Gating)
6861 {
6862 return Ok(());
6863 }
6864 if self.state.gate_ran {
6865 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6876 self.state.status = RunStatus::Blocked;
6877 self.state.save()?;
6878 }
6879 return Ok(());
6880 }
6881 let Some(winner) = self.state.winner().cloned() else {
6882 return Ok(());
6883 };
6884 self.state.status = RunStatus::Gating;
6885 let mut outcomes = self.run_gate(&winner).await?;
6886 loop {
6887 if verify_inconclusive(&outcomes) {
6898 self.state.save()?;
6899 return Ok(());
6900 }
6901 if outcomes.iter().all(CommandOutcome::ok) {
6902 break;
6903 }
6904 match self.gate_fix_round(&winner, &outcomes).await? {
6905 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6906 GateFix::Stop => break,
6907 GateFix::Defer => {
6908 self.state.save()?;
6909 return Ok(());
6910 }
6911 }
6912 }
6913 let passed = outcomes.iter().all(CommandOutcome::ok);
6914 self.state.gate = outcomes;
6915 self.state.gate_ran = true;
6916 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6917 if !passed {
6918 self.state.status = RunStatus::Blocked;
6919 let spent = self.state.gate_fixes.len();
6920 self.state.event(
6921 "gate",
6922 if spent == 0 {
6923 "gate failed; not merging".to_owned()
6924 } else {
6925 format!("gate failed after {spent} gate-fix round(s); not merging")
6926 },
6927 );
6928 }
6929 self.state.save()?;
6930 Ok(())
6931 }
6932
6933 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6943 let commands = state.config.verify.pre_gate.clone();
6944 if commands.is_empty() {
6945 return;
6946 }
6947 let shell = state.config.shell();
6948 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
6949 let (outcomes, _) = run_commands(
6950 state,
6951 "pre_gate",
6952 "pre_gate",
6953 0,
6954 &shell,
6955 &commands,
6956 &winner.worktree,
6957 timeout,
6958 )
6959 .await;
6960 for o in &outcomes {
6961 if !o.ok() {
6962 tracing::warn!(
6963 "pre_gate `{}` failed ({:?}); the gate decides",
6964 o.command,
6965 o.code
6966 );
6967 }
6968 state.event(
6969 "pre_gate",
6970 format!(
6971 "`{}` -> {}",
6972 o.command,
6973 if o.ok() {
6974 "pass".to_owned()
6975 } else {
6976 format!(
6977 "FAIL ({:?})\n{}",
6978 o.code,
6979 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6980 )
6981 }
6982 ),
6983 );
6984 }
6985 state.pre_gate = outcomes;
6986 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
6987 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
6988 Ok(head) => {
6989 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
6990 state.pre_gate_commit = Some(head);
6991 }
6992 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
6993 },
6994 Ok(false) => {}
6995 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
6996 }
6997 if let Err(e) = state.save() {
6998 tracing::warn!("could not persist the pre_gate record: {e:#}");
6999 }
7000 }
7001
7002 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7005 let shell = self.state.config.shell();
7006 let gate_commands = self.state.config.verify.gate.clone();
7007 let has_pre_gate = !self.state.config.verify.pre_gate.is_empty();
7008 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
7009 Vec::new()
7010 } else {
7011 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7012 let cache_dir = self.state.config.cache_dir();
7013 let wait = self.repository_wait();
7014 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7015 let (outcomes, _) = with_cache_lease(
7016 &mut self.state,
7017 cache_dir.as_deref(),
7018 "gate",
7019 "gate",
7020 &winner.worktree,
7021 &head,
7022 timeout,
7023 "final gate",
7024 &wait,
7025 |state, budget| {
7026 let shell = shell.clone();
7027 let gate_commands = gate_commands.clone();
7028 let winner = winner.clone();
7029 async move {
7030 Self::run_pre_gate(state, &winner).await;
7031 let (outcomes, timed_out_pids) = run_commands(
7032 state,
7033 "gate",
7034 "gate",
7035 0,
7036 &shell,
7037 &gate_commands,
7038 &winner.worktree,
7039 budget,
7040 )
7041 .await;
7042 (outcomes, false, timed_out_pids)
7043 }
7044 },
7045 )
7046 .await?;
7047 outcomes
7048 };
7049 if outcomes.is_empty() {
7050 self.state.event(
7055 "gate",
7056 "no gate commands configured; nothing to check, passing",
7057 );
7058 }
7059 for o in &outcomes {
7060 self.state.event(
7061 "gate",
7062 format!(
7063 "`{}` -> {}",
7064 o.command,
7065 if o.ok() {
7066 "pass".to_owned()
7067 } else {
7068 format!(
7069 "FAIL ({:?})\n{}",
7070 o.code,
7071 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7072 )
7073 }
7074 ),
7075 );
7076 }
7077 Ok(outcomes)
7078 }
7079
7080 async fn gate_fix_round(
7093 &mut self,
7094 winner: &Candidate,
7095 outcomes: &[CommandOutcome],
7096 ) -> Result<GateFix> {
7097 let cap = self.state.config.graph.gate_fix_rounds;
7098 let spent = self.state.gate_fixes.len();
7099 if spent >= cap {
7100 if cap > 0 {
7101 self.state.event(
7102 "gate",
7103 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7104 );
7105 }
7106 return Ok(GateFix::Stop);
7107 }
7108 if !gate_fixable(outcomes) {
7109 self.state.event(
7110 "gate",
7111 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7112 command or similar); not spending a fix round on it",
7113 );
7114 return Ok(GateFix::Stop);
7115 }
7116 let min_free = self.state.config.disk.min_free_bytes;
7117 if min_free > 0 {
7118 match crate::disk::free_bytes(&winner.worktree) {
7119 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7120 Ok(free) => {
7121 self.state.event(
7122 "gate",
7123 format!(
7124 "only {free} bytes free ({min_free} required by `[disk] \
7125 min_free_bytes`); not spending a fix round on a failure the disk \
7126 may explain"
7127 ),
7128 );
7129 return Ok(GateFix::Stop);
7130 }
7131 Err(e) => {
7132 self.state.event(
7133 "gate",
7134 format!("free disk space could not be measured ({e:#}); no fix round"),
7135 );
7136 return Ok(GateFix::Stop);
7137 }
7138 }
7139 }
7140
7141 let attempt = spent + 1;
7142 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7143 let base = self.landing_base();
7144 let fix_prompt = prompt::gate_fix_in(
7145 &self.state.instruction,
7146 &failed,
7147 attempt,
7148 cap,
7149 &self.state.config.graph.language,
7150 &self.state.config.graph.github_language,
7151 );
7152 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7153 let sessions = self.state.config.graph.sessions;
7154 let artifacts = self.merge_repair_artifacts();
7155 self.state.event(
7156 "gate",
7157 format!("gate failed; gate-fix round {attempt} of {cap}"),
7158 );
7159 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7160 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7161 let (_, seat, out) = self
7162 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7163 prompt: fix_prompt.clone(),
7164 spec,
7165 seat,
7166 cwd: winner.worktree.clone(),
7167 timeout,
7168 allow_write: true,
7169 sessions,
7170 artifacts: artifacts.clone(),
7171 stem: format!("gate-fix-{attempt}"),
7172 handover: None,
7173 })
7174 .await;
7175 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7178 return Ok(GateFix::Defer);
7179 }
7180 let mut record = GateFixRecord {
7181 agent: seat.agent.clone(),
7182 failed,
7183 notes: String::new(),
7184 committed: false,
7185 error: None,
7186 };
7187 match out {
7188 AgentOutcome::Ok(o) => {
7189 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7192 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7193 }
7194 }
7195 AgentOutcome::Dropped(_) => {
7196 record.error = Some("the CLI dropped the stream".to_owned());
7197 }
7198 AgentOutcome::Quota(o) => {
7199 self.state.quota.push(QuotaLoss {
7200 seat: seat.key.clone(),
7201 node: "gate-fix".to_owned(),
7202 at: Timestamp::now(),
7203 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7204 });
7205 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7206 }
7207 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7208 }
7209 self.state.seats.insert(seat.key.clone(), seat);
7210 let coordination = self.coordinate(&winner.worktree).await?;
7211 if let Ok(r) = git::rescue_commit(
7212 &winner.worktree,
7213 &format!("magi: gate fix {attempt} (uncommitted work)"),
7214 )
7215 .await
7216 {
7217 self.state.note_withheld("gate-fix", &r.withheld);
7218 }
7219 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7220 record.committed = after != before;
7221 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7222 let note = record.error.clone();
7223 self.state.gate_fixes.push(record);
7224 self.state.save()?;
7225 if !changed {
7226 self.state.event(
7227 "gate",
7228 match note {
7229 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7230 None => format!("gate-fix round {attempt}: the tree did not change"),
7231 },
7232 );
7233 return Ok(GateFix::Stop);
7234 }
7235 self.state.event(
7236 "gate",
7237 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7238 );
7239
7240 drop(coordination);
7241 let commands = self.state.config.verify.e2e.clone();
7242 if !commands.is_empty() {
7243 let shell = self.state.config.shell();
7244 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7245 let cache_dir = self.state.config.cache_dir();
7246 let context = format!("gate-fix round {attempt}");
7247 let wait = self.repository_wait();
7248 let (e2e, _) = with_cache_lease(
7249 &mut self.state,
7250 cache_dir.as_deref(),
7251 "e2e",
7252 "e2e",
7253 &winner.worktree,
7254 &after,
7255 timeout,
7256 &context,
7257 &wait,
7258 |state, budget| {
7259 let shell = shell.clone();
7260 let commands = commands.clone();
7261 let context = context.clone();
7262 let worktree = winner.worktree.clone();
7263 async move {
7264 run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
7265 .await
7266 }
7267 },
7268 )
7269 .await?;
7270 if verify_inconclusive(&e2e) {
7271 return Ok(GateFix::Defer);
7272 }
7273 if e2e.iter().any(|o| !o.ok()) {
7274 self.state.event(
7275 "gate",
7276 format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
7277 );
7278 return Ok(GateFix::Stop);
7279 }
7280 }
7281 Ok(GateFix::Retry)
7282 }
7283
7284 async fn guarded_pr_message(
7291 &mut self,
7292 winner: &Candidate,
7293 facts: Option<&BranchFacts>,
7294 posting: bool,
7295 ) -> Result<Option<PrMessage>> {
7296 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7297 let head = match final_head.as_deref() {
7298 Some(h) => crate::pr_outcome::FinalHead::At(h),
7299 None => crate::pr_outcome::FinalHead::Unknown,
7300 };
7301 let summarized = if posting {
7303 Some(self.summarize_change(winner, final_head.as_deref()).await)
7304 } else {
7305 None
7306 };
7307 if self.state.resource_block.is_some() {
7312 self.state.event(
7313 "pr-summary",
7314 "a shared resource was busy; nothing was published, resume to summarize again",
7315 );
7316 return Ok(None);
7317 }
7318 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7319 anyhow::bail!(
7320 "the final commit moved during public description generation; retry before publishing"
7321 );
7322 }
7323 let note = match (&summarized, &self.state.change_summary) {
7324 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7325 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7326 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7327 (None, _) => ChangeNote::NotAsked,
7328 };
7329 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7330 if !posting {
7331 let identity = crate::scrub::Identity::current();
7332 return Ok(Some(PrMessage {
7333 title: crate::scrub::scrub(&pr.title, &identity),
7334 body: crate::scrub::scrub(&pr.body, &identity),
7335 }));
7336 }
7337 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7340 if let Some(known) = self
7341 .state
7342 .github_text
7343 .clone()
7344 .filter(|g| g.fingerprint == raw_fp)
7345 {
7346 return self.settle_withheld(winner, &pr, known).await;
7347 }
7348 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7349 let mut violations =
7350 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7351 let identity = crate::scrub::Identity::current();
7352 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7353 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7354 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7355 {
7356 violations.push(crate::github_text::Violation::SensitiveData);
7357 }
7358 let has_language = violations
7362 .iter()
7363 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7364 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7365 self.state.event(
7366 "github-text",
7367 if has_language {
7368 format!("description rejected: {violations:?}; requesting one rewrite")
7369 } else {
7370 format!("description rejected: {violations:?}; asking the owner")
7371 },
7372 );
7373 let mut rewritten = false;
7374 let mut rejected_rewrite = None;
7375 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7376 {
7377 let output = text;
7378 if let Some(title) = summary_title(&output) {
7379 let raw_body = summary_without_title(&output);
7382 let clean_title = crate::scrub::scrub(&title, &identity);
7383 let mut body = crate::scrub::scrub(&raw_body, &identity);
7384 let redacted = clean_title != title || body != raw_body;
7385 let title = clean_title;
7386 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7387 if !complete {
7388 self.state.event(
7389 "github-text",
7390 "description rewrite omitted required findings",
7391 );
7392 }
7393 if !body.trim().is_empty() {
7394 body.push_str("\n\n");
7395 body.push_str(&public_pr_evidence(&self.state, head));
7396 }
7397 let marker = format!("magi:run/{}", self.state.id);
7398 if !body.trim().is_empty() && !body.contains(&marker) {
7399 body.push_str(&format!("\n\n{marker}"));
7400 }
7401 let rewrite = PrMessage {
7402 title: title.clone(),
7403 body: body.clone(),
7404 };
7405 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7406 let mut failed = crate::github_text::check_for(
7407 &self.state.config.graph,
7408 &title,
7409 &body,
7410 redecision,
7411 );
7412 if (!crate::github_text::shareable(&title)
7413 || !crate::github_text::shareable(&body))
7414 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7415 {
7416 failed.push(crate::github_text::Violation::SensitiveData);
7417 }
7418 if complete
7419 && !title.trim().is_empty()
7420 && !body.trim().is_empty()
7421 && !failed.is_empty()
7422 {
7423 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7424 title: title.clone(),
7425 body: body.clone(),
7426 failed: failed
7427 .iter()
7428 .map(|v| crate::github_text::category(*v).to_owned())
7429 .collect(),
7430 });
7431 }
7432 if complete
7433 && !title.trim().is_empty()
7434 && !raw_body.trim().is_empty()
7435 && failed.is_empty()
7436 {
7437 decision = redecision;
7438 pr = PrMessage { title, body };
7439 rewritten = true;
7440 self.state
7441 .event("github-text", "description rewrite passed");
7442 if redacted {
7443 self.state.event(
7444 "github-text",
7445 "sensitive data removed from description rewrite",
7446 );
7447 }
7448 }
7449 }
7450 }
7451 if !rewritten {
7452 self.state.event(
7453 "github-text",
7454 "description rewrite unavailable or rejected; asking the owner",
7455 );
7456 if let Some(w) =
7457 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7458 {
7459 self.retire_github_text_question("the withheld text changed");
7462 let record = crate::run::GithubTextAsk {
7463 rewrite: rejected_rewrite,
7464 fingerprint: raw_fp,
7465 title: w.title,
7466 body: w.body,
7467 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7468 question: None,
7469 asks: 0,
7470 resolved: false,
7471 chosen_title: None,
7472 choice: None,
7473 };
7474 return self.settle_withheld(winner, &pr, record).await;
7475 }
7476 }
7477 }
7478 let (title, body) =
7479 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7480 Ok(Some(PrMessage { title, body }))
7481 }
7482
7483 async fn summarize_change(
7494 &mut self,
7495 winner: &Candidate,
7496 head: Option<&str>,
7497 ) -> std::result::Result<(), String> {
7498 let Some(head) = head else {
7499 return Err("the final commit could not be read".to_owned());
7500 };
7501 let remote = self.state.config.merge.remote.clone();
7502 let tracking = format!("{remote}/{}", self.state.base_branch);
7503 if !git::rev_exists(&winner.worktree, &tracking).await {
7504 return Err(format!("the remote base `{tracking}` could not be read"));
7505 }
7506 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7507 Ok(b) if !b.is_empty() => b,
7508 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7509 };
7510 let evidence = crate::pr_outcome::summary_evidence(
7511 &self.state,
7512 crate::pr_outcome::FinalHead::At(head),
7513 );
7514 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7515 if self
7516 .state
7517 .change_summary
7518 .as_ref()
7519 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7520 {
7521 return Ok(());
7522 }
7523 if self.state.change_summary.take().is_some() {
7524 self.state.event(
7525 "pr-summary",
7526 "the final diff or evidence changed; the earlier change summary was discarded",
7527 );
7528 }
7529 if let Err(e) = write_artifact(
7531 &self.state,
7532 &format!("public-description-{fingerprint}.md"),
7533 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7534 ) {
7535 tracing::warn!("could not save public description evidence: {e:#}");
7536 }
7537 let chain = match agent::pick_chain(
7538 &self.state.config.agents,
7539 self.state.config.roles.synthesizer.as_ref(),
7540 &agent::installed,
7541 "synthesizer",
7542 ) {
7543 Ok(chain) => chain,
7544 Err(e) => {
7545 tracing::warn!("no agent for the change summary: {e:#}");
7546 self.state.event("pr-summary", "no summary agent available");
7547 return Err("no agent was available to summarize it".to_owned());
7548 }
7549 };
7550 let subjects = git::subjects(&winner.worktree, &base, head)
7551 .await
7552 .unwrap_or_default();
7553 let stat = git::diff_stat(&winner.worktree, &base, head)
7554 .await
7555 .unwrap_or_default();
7556 let stat: String = stat.chars().take(6_000).collect();
7557 let mut prompt = crate::prompt::change_summary_in(
7558 &base,
7559 head,
7560 &subjects,
7561 stat.trim(),
7562 &self.state.config.graph.github_language,
7563 );
7564 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7565 let sessions = self.state.config.graph.sessions;
7566 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7567 let prompts = self.state.config.prompts.clone();
7568 let cache = self.state.config.cache_dir();
7569 let run = self.state.id.clone();
7570 for (i, spec) in chain.into_iter().enumerate() {
7571 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7572 let job = SeatJob {
7573 spec: spec.clone(),
7574 seat,
7575 cwd: winner.worktree.clone(),
7576 prompt: prompt.clone(),
7577 timeout,
7578 allow_write: false,
7579 sessions,
7580 artifacts: agent::artifacts_dir(&self.state.dir()),
7581 stem: if i == 0 {
7582 "change-summary".to_owned()
7583 } else {
7584 format!("change-summary-{}", spec.id)
7585 },
7586 handover: None,
7587 };
7588 let ctx = WaveCtx {
7589 run: &run,
7590 node: "pr-summary",
7591 prompts: &prompts,
7592 cache: cache.as_deref(),
7593 round: None,
7594 carry_seats: false,
7595 pauses: Some((&self.pause, &self.interrupt)),
7596 };
7597 let (seat, outcome) =
7598 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7599 self.state.seats.insert(seat.key.clone(), seat);
7600 let text = match outcome {
7601 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7602 AgentOutcome::ResourceBlocked(why) => {
7603 if self.state.resource_block.is_none() {
7606 self.state.resource_block = Some(crate::run::ResourceBlock {
7607 node: "pr-summary".to_owned(),
7608 kind: "shared resource".to_owned(),
7609 detail: why,
7610 at: Timestamp::now(),
7611 });
7612 }
7613 return Err("a shared resource was busy".to_owned());
7614 }
7615 _ => continue,
7616 };
7617 let blind = &self.state.config.blind;
7618 let Some(title) = summary_title(&text) else {
7619 continue;
7620 };
7621 let body: String = summary_without_title(&text)
7622 .lines()
7623 .filter(|l| !l.contains("magi:run/"))
7624 .collect::<Vec<_>>()
7625 .join("\n");
7626 let body = blind::sanitize_prose(body.trim(), blind);
7627 let title = blind::sanitize_prose(&title, blind);
7628 if body.trim().is_empty() || title.trim().is_empty() {
7629 continue;
7630 }
7631 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7632 continue;
7633 }
7634 if git::rev_parse(&winner.worktree, "HEAD")
7636 .await
7637 .ok()
7638 .as_deref()
7639 != Some(head)
7640 {
7641 return Err("the branch moved while the summary was written".to_owned());
7642 }
7643 if git::merge_base(&winner.worktree, &tracking, head)
7644 .await
7645 .ok()
7646 .as_deref()
7647 != Some(base.as_str())
7648 || crate::pr_outcome::summary_evidence(
7649 &self.state,
7650 crate::pr_outcome::FinalHead::At(head),
7651 ) != evidence
7652 {
7653 return Err("the summary evidence changed while it was written".to_owned());
7654 }
7655 self.state.change_summary = Some(crate::run::ChangeSummary {
7656 head: head.to_owned(),
7657 base,
7658 evidence: fingerprint,
7659 title,
7660 body,
7661 });
7662 self.state.event(
7663 "pr-summary",
7664 format!("change summary written by {} for the final diff", spec.id),
7665 );
7666 if let Err(e) = self.state.save() {
7667 tracing::warn!("could not save the change summary: {e:#}");
7668 }
7669 return Ok(());
7670 }
7671 self.state
7672 .event("pr-summary", "no agent produced a change summary");
7673 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7674 }
7675
7676 async fn rewrite_pr_text(
7684 &mut self,
7685 winner: &Candidate,
7686 pr: &PrMessage,
7687 violations: &[crate::github_text::Violation],
7688 ) -> Option<String> {
7689 let prompt = format!(
7690 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7691 pr.title, pr.body
7692 );
7693 let sessions = self.state.config.graph.sessions;
7694 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7695 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7697 let key = format!("impl-{}", winner.label);
7698 if let Some(spec) = self
7699 .state
7700 .config
7701 .agents
7702 .iter()
7703 .find(|a| a.id == winner.agent)
7704 .cloned()
7705 && let Some(existing) = self.state.seats.get(&key)
7706 && existing.agent == spec.id
7707 && agent::has_session(spec.kind, existing, sessions)
7708 {
7709 attempts.push((spec, existing.clone(), true));
7710 } else {
7711 let chain = agent::pick_chain(
7712 &self.state.config.agents,
7713 self.state.config.roles.language_judge.as_ref(),
7714 &agent::installed,
7715 "language_judge",
7716 );
7717 match chain {
7718 Ok(chain) => {
7719 for spec in chain {
7720 let seat = crate::fixer::seat_for(
7721 &mut self.state,
7722 "github-text-rewrite",
7723 &spec.id,
7724 );
7725 attempts.push((spec, seat, false));
7726 }
7727 }
7728 Err(e) => {
7729 tracing::warn!("no agent for the description rewrite: {e:#}");
7730 self.state
7731 .event("github-text", "no rewrite agent available");
7732 return None;
7733 }
7734 }
7735 }
7736 let prompts = self.state.config.prompts.clone();
7737 let cache = self.state.config.cache_dir();
7738 let run = self.state.id.clone();
7739 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7740 let job = SeatJob {
7741 spec: spec.clone(),
7742 seat,
7743 cwd: winner.worktree.clone(),
7744 prompt: prompt.clone(),
7745 timeout: retry_budget(timeout_full, resumed),
7746 allow_write: false,
7747 sessions,
7748 artifacts: agent::artifacts_dir(&self.state.dir()),
7749 stem: if i == 0 {
7750 "github-text-rewrite".to_owned()
7751 } else {
7752 format!("github-text-rewrite-{}", spec.id)
7753 },
7754 handover: None,
7755 };
7756 let ctx = WaveCtx {
7757 run: &run,
7758 node: "github-text",
7759 prompts: &prompts,
7760 cache: cache.as_deref(),
7761 round: None,
7762 carry_seats: false,
7763 pauses: Some((&self.pause, &self.interrupt)),
7764 };
7765 let (seat, outcome) =
7766 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7767 self.state.seats.insert(seat.key.clone(), seat);
7768 match outcome {
7769 AgentOutcome::Ok(o)
7770 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7771 {
7772 return Some(o.text);
7773 }
7774 AgentOutcome::ResourceBlocked(_) => return None,
7775 _ => {}
7776 }
7777 }
7778 None
7779 }
7780
7781 fn retire_github_text_question(&mut self, why: &str) {
7783 let Some(id) = self
7784 .state
7785 .github_text
7786 .as_ref()
7787 .and_then(|g| g.question.clone())
7788 else {
7789 return;
7790 };
7791 let _ = ask::Questions::open().update(&id, |q| {
7792 q.abandon(why);
7793 Ok(())
7794 });
7795 }
7796
7797 async fn settle_withheld(
7801 &mut self,
7802 winner: &Candidate,
7803 pr: &PrMessage,
7804 mut g: crate::run::GithubTextAsk,
7805 ) -> Result<Option<PrMessage>> {
7806 use crate::github_text as gt;
7807 let store = ask::Questions::open();
7808 let timeout = self.state.config.graph.answer_timeout;
7809 if !g.resolved {
7810 let mut standing = g
7811 .question
7812 .as_ref()
7813 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7814 if let Some(q) = standing.as_ref()
7815 && q.status.open()
7816 && gt::expired(q, timeout)
7817 {
7818 let why = format!("no answer within {}s of asking", timeout.max(1));
7819 standing = store
7820 .update(&q.id, |q| {
7821 q.abandon(&why);
7822 Ok(())
7823 })
7824 .ok()
7825 .map(|(q, ())| q);
7826 }
7827 let mut ask_again: Option<Option<Vec<String>>> = None;
7828 match standing {
7829 None if g.asks <= 1 => {
7832 g.asks = 0;
7833 ask_again = Some(None);
7834 }
7835 None => {
7836 g.resolved = true;
7837 g.chosen_title = None;
7838 g.choice = Some(GithubTextChoice::Fallback);
7839 }
7840 Some(q) if q.status.open() => {
7841 self.park_on_github_text(&g)?;
7842 return Ok(None);
7843 }
7844 Some(q) => match gt::read_reply(&q) {
7845 gt::Reply::Fallback => {
7846 g.resolved = true;
7847 g.chosen_title = None;
7848 g.choice = Some(GithubTextChoice::Fallback);
7849 }
7850 gt::Reply::Original => {
7851 g.resolved = true;
7852 g.chosen_title = None;
7853 g.choice = Some(GithubTextChoice::Original);
7854 }
7855 gt::Reply::Rewrite => {
7856 g.resolved = true;
7857 g.chosen_title = None;
7858 g.choice = Some(if g.rewrite.is_some() {
7859 GithubTextChoice::Rewrite
7860 } else {
7861 GithubTextChoice::Fallback
7862 });
7863 }
7864 gt::Reply::Title(text) => {
7865 let first = text
7866 .lines()
7867 .map(str::trim)
7868 .find(|l| !l.is_empty())
7869 .unwrap_or("");
7870 let decision = if first.is_empty() {
7871 None
7872 } else {
7873 let candidate = PrMessage {
7874 title: first.to_owned(),
7875 body: String::new(),
7876 };
7877 self.judge_pr_language(&winner.worktree, &candidate).await
7878 };
7879 match gt::vet_title(
7880 &text,
7881 decision,
7882 self.state.config.graph.github_english(),
7883 ) {
7884 Ok(title) => {
7885 g.resolved = true;
7886 g.chosen_title = Some(title);
7887 g.choice = Some(GithubTextChoice::MyTitle);
7888 }
7889 Err(cats) if g.asks < 2 => {
7890 ask_again = Some(Some(
7891 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7892 ))
7893 }
7894 Err(_) => {
7895 g.resolved = true;
7896 g.chosen_title = None;
7897 g.choice = Some(GithubTextChoice::Fallback);
7898 }
7899 }
7900 }
7901 },
7902 }
7903 if let Some(rejected) = ask_again {
7904 let w = gt::Withheld {
7907 title: g.title,
7908 body: g.body,
7909 categories: g.categories.clone(),
7910 };
7911 let artifact = match crate::run::write_artifact(
7914 &self.state,
7915 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7916 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7917 ) {
7918 Ok(p) => Some(p),
7919 Err(e) => {
7920 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7921 None
7922 }
7923 };
7924 let (detail, comparison) = gt::question_content(
7925 &w,
7926 &pr.title,
7927 &pr.body,
7928 &self.state.id,
7929 rejected.as_deref(),
7930 artifact.as_deref(),
7931 g.rewrite.as_ref(),
7932 );
7933 let mut q = ask::Question::new(
7934 self.state.id.clone(),
7935 gt::ASK_NODE.to_owned(),
7936 gt::ASK_SEAT.to_owned(),
7937 gt::question_summary(&self.state.config.graph.language, &w),
7938 detail,
7939 gt::question_choices(&w, g.rewrite.as_ref()),
7940 );
7941 q.posting_comparison = Some(comparison);
7942 q.answer_timeout = timeout;
7943 g.question = Some(q.id.clone());
7946 g.asks += 1;
7947 self.state.github_text = Some(g.clone());
7948 self.state.event(
7949 "github-text",
7950 format!("asking the owner about the withheld text ({})", q.short()),
7951 );
7952 self.state.save()?;
7953 store
7954 .put(&mut q)
7955 .context("file the posting-gate question")?;
7956 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7957 tracing::warn!(
7958 "could not notify about posting-gate question {}: {e:#}",
7959 q.short()
7960 );
7961 }
7962 self.park_on_github_text(&g)?;
7963 return Ok(None);
7964 }
7965 }
7966 let resolved = g.chosen_title.clone();
7967 let original = g.choice == Some(GithubTextChoice::Original);
7968 self.state.event(
7970 "github-text",
7971 if original {
7972 format!(
7973 "posting the original text for: {}",
7974 match (g.title, g.body) {
7975 (true, true) => "title, description",
7976 (true, false) => "title",
7977 _ => "description",
7978 }
7979 )
7980 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7981 "posting the saved rewrite title and description".to_owned()
7982 } else if resolved.is_some() {
7983 "posting the owner's replacement title".to_owned()
7984 } else {
7985 "posting the neutral text for the withheld fields".to_owned()
7986 },
7987 );
7988 self.state.github_text = Some(g.clone());
7989 self.state.save()?;
7990 if g.choice == Some(GithubTextChoice::Rewrite)
7991 && let Some(r) = g.rewrite.as_ref()
7992 {
7993 return Ok(Some(PrMessage {
7994 title: r.title.clone(),
7995 body: r.body.clone(),
7996 }));
7997 }
7998 let w = gt::Withheld {
7999 title: g.title,
8000 body: g.body,
8001 categories: g.categories.clone(),
8002 };
8003 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8004 if let Some(t) = resolved {
8005 title = t;
8006 }
8007 let english = gt::LanguageDecision {
8009 title_english: true,
8010 body_english: true,
8011 body_complete: true,
8012 };
8013 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8014 if original {
8015 if g.title {
8018 title = pr.title.clone();
8019 }
8020 if g.body {
8021 body = pr.body.clone();
8022 }
8023 }
8024 Ok(Some(PrMessage { title, body }))
8025 }
8026
8027 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8028 self.state.github_text = Some(g.clone());
8029 self.state.end_pass_parked(ParkCause::Question);
8030 self.state.parked = true;
8031 self.state.event(
8032 "github-text",
8033 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8034 );
8035 self.state.save()
8036 }
8037
8038 async fn judge_pr_language(
8041 &mut self,
8042 cwd: &Path,
8043 pr: &PrMessage,
8044 ) -> Option<crate::github_text::LanguageDecision> {
8045 self.state.config.roles.language_judge.as_ref()?;
8046 if !self.state.config.graph.language_gate() {
8047 return None;
8048 }
8049 let decision =
8050 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8051 self.state.event(
8052 "github-text",
8053 if decision.is_some() {
8054 "language decided by the language judge"
8055 } else {
8056 "language judge unavailable; using built-in heuristics"
8057 },
8058 );
8059 decision
8060 }
8061
8062 async fn merge(&mut self) -> Result<()> {
8063 if self
8078 .state
8079 .base_sync
8080 .as_ref()
8081 .is_some_and(|s| s.conflict.is_some())
8082 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8083 != Some(RunStatus::Gating)
8084 || !self.state.gate_status().ok()
8093 {
8094 return Ok(());
8095 }
8096 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8105 return Ok(());
8106 }
8107 let Some(winner) = self.state.winner().cloned() else {
8108 return Ok(());
8109 };
8110 let repo = self.state.repo.clone();
8111 let base = self.state.base_branch.clone();
8112 let mode = self.state.config.merge.mode;
8113 let style = self.state.config.merge.style;
8114 let facts = if is_review_run(&self.state) {
8115 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8116 let start = review_base(
8117 &repo,
8118 &self.state.config.merge.remote,
8119 &base,
8120 &self.state.base_commit,
8121 &winner.branch,
8122 )
8123 .await;
8124 branch_facts(&repo, &start, &winner.branch).await
8125 } else {
8126 None
8127 };
8128 let leaked = if is_review_run(&self.state) {
8131 leaked_subjects(&self.state, &winner.branch).await
8132 } else {
8133 Some(Vec::new())
8134 };
8135 let Some(pr) = self
8139 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8140 .await?
8141 else {
8142 return Ok(());
8143 };
8144 let message = pr.commit_message();
8145
8146 let outcome = match mode {
8147 MergeMode::None => MergeOutcome {
8148 mode,
8149 ok: true,
8150 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8151 empty: false,
8152 },
8153 MergeMode::Pr | MergeMode::Local
8154 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8155 {
8156 MergeOutcome {
8157 mode,
8158 ok: false,
8159 detail: empty_candidate_detail(&self.state, &base),
8160 empty: true,
8161 }
8162 }
8163 MergeMode::Local => {
8164 let on = git::current_branch(&repo).await?;
8165 if on.as_deref() != Some(base.as_str()) {
8166 MergeOutcome {
8167 mode,
8168 ok: false,
8169 detail: format!(
8170 "{} has {} checked out, not the base branch {base}",
8171 repo.display(),
8172 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8173 ),
8174 empty: false,
8175 }
8176 } else if !git::is_clean(&repo).await? {
8177 MergeOutcome {
8178 mode,
8179 ok: false,
8180 detail: format!("{} is dirty; refusing to merge", repo.display()),
8181 empty: false,
8182 }
8183 } else {
8184 let out = match style {
8185 MergeStyle::Merge => {
8186 git::merge_no_ff(&repo, &winner.branch, &message).await?
8187 }
8188 MergeStyle::Squash => {
8189 git::merge_squash(&repo, &winner.branch, &message).await?
8190 }
8191 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8192 };
8193 MergeOutcome {
8194 mode,
8195 ok: out.ok(),
8196 detail: if out.ok() { out.stdout } else { out.stderr },
8197 empty: false,
8198 }
8199 }
8200 }
8201 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8202 self.state.pr_publication_pending = true;
8203 self.state.save()?;
8204 let out =
8205 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8206 self.state.pr_publication_pending = out.is_err();
8207 MergeOutcome {
8208 mode,
8209 ok: out.is_ok(),
8210 detail: match out {
8211 Ok(url) => url,
8212 Err(e) => format!("{e:#}"),
8213 },
8214 empty: false,
8215 }
8216 }
8217 MergeMode::Pr => {
8218 let remote = self.state.config.merge.remote.clone();
8219 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8220 if !pushed.ok() {
8221 MergeOutcome {
8222 mode,
8223 ok: false,
8224 detail: pushed.stderr,
8225 empty: false,
8226 }
8227 } else {
8228 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8234 let out = match pr_merge_plan(found) {
8235 PrPlan::Create => {
8236 gh_pr_create(
8237 &winner.worktree,
8238 &base,
8239 &winner.branch,
8240 &pr.title,
8241 &pr.body,
8242 )
8243 .await
8244 }
8245 PrPlan::Adopt { url, title } => {
8246 self.state
8247 .event("merge", format!("Pr: adopted open pull request {url}"));
8248 if self.state.github_text.as_ref().is_some_and(|g| {
8249 g.resolved
8250 && g.choice == Some(GithubTextChoice::Rewrite)
8251 && g.rewrite.is_some()
8252 }) {
8253 land::set_pr_text_verbatim(
8256 &winner.worktree,
8257 &url,
8258 &pr.title,
8259 &pr.body,
8260 )
8261 .await
8262 .map(|()| url)
8263 } else {
8264 if title != pr.title
8265 && (!is_review_run(&self.state)
8266 || leaked
8267 .as_deref()
8268 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8269 && let Err(e) =
8270 if self.state.github_text.as_ref().is_some_and(|g| {
8271 g.resolved
8272 && g.title
8273 && g.choice == Some(GithubTextChoice::Original)
8274 }) {
8275 land::set_pr_title_verbatim(
8277 &winner.worktree,
8278 &url,
8279 &pr.title,
8280 )
8281 .await
8282 } else {
8283 land::set_pr_title(
8284 &mut self.state,
8285 &winner.worktree,
8286 &url,
8287 &pr.title,
8288 )
8289 .await
8290 }
8291 {
8292 tracing::warn!("could not refresh title of {url}: {e:#}");
8293 self.state
8294 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8295 }
8296 Ok(url)
8297 }
8298 }
8299 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8300 };
8301 match out {
8302 Ok(url) => MergeOutcome {
8303 mode,
8304 ok: true,
8305 detail: url,
8306 empty: false,
8307 },
8308 Err(e) => MergeOutcome {
8309 mode,
8310 ok: false,
8311 detail: e.to_string(),
8312 empty: false,
8313 },
8314 }
8315 }
8316 }
8317 };
8318
8319 self.state.status = match (mode, outcome.ok) {
8320 (MergeMode::None, _) => RunStatus::Ready,
8321 (_, true) => RunStatus::Merged,
8322 (_, false) => RunStatus::Blocked,
8323 };
8324 self.state.event(
8325 "merge",
8326 format!(
8327 "{:?}: {}",
8328 mode,
8329 outcome.detail.lines().next().unwrap_or("")
8330 ),
8331 );
8332 self.state.merge = Some(outcome);
8333 self.state.save()?;
8334
8335 if self.state.config.graph.land
8341 && mode == MergeMode::Pr
8342 && self.state.status == RunStatus::Merged
8343 {
8344 self.run_land().await?;
8345 }
8346 self.settle_questions();
8351 Ok(())
8352 }
8353
8354 async fn run_land(&mut self) -> Result<()> {
8365 crate::pr_head::ensure_owner(&self.state)?;
8366 let url = self
8367 .state
8368 .merge
8369 .as_ref()
8370 .map(|m| m.detail.clone())
8371 .unwrap_or_default();
8372 let url = url.lines().next().unwrap_or("").trim().to_owned();
8373 if !url.starts_with("http") {
8374 return Ok(());
8375 }
8376 loop {
8379 match land::land(&mut self.state, &url).await {
8380 Ok(_)
8381 if self
8382 .state
8383 .merge_repairs
8384 .last()
8385 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8386 {
8387 if self.repair_merge_review().await? {
8388 continue;
8389 }
8390 }
8391 Ok(pr) if self.state.parked => {
8392 let _ = pr;
8396 }
8397 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8398 self.state.status = RunStatus::Superseded;
8399 self.state.save()?;
8400 }
8401 Ok(pr) => {
8402 self.state.status = match pr.state {
8403 land::PrLifecycle::Merged => RunStatus::Merged,
8404 _ => RunStatus::Blocked,
8405 };
8406 if bump::should_release_bump(self.state.status)
8413 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8414 {
8415 self.state
8421 .event("bump", format!("release bump skipped: {e:#}"));
8422 }
8423 if self.state.status == RunStatus::Merged {
8426 crate::followup::after_merge(&mut self.state, &pr.url).await;
8427 }
8428 self.state.save()?;
8429 }
8430 Err(e) => {
8431 if e.is::<crate::repo_lock::ResourceBlocked>() {
8432 return Err(e);
8433 }
8434 self.state.status = RunStatus::Blocked;
8435 self.state.event("land", format!("gave up: {e}"));
8436 self.state.save()?;
8437 }
8438 }
8439 break;
8440 }
8441 Ok(())
8442 }
8443
8444 async fn repair_merge_review(&mut self) -> Result<bool> {
8447 use crate::run::MergeRepairPhase as Phase;
8448 crate::pr_head::ensure_owner(&self.state)?;
8449 let i = self.state.merge_repairs.len() - 1;
8450 let request = self.state.merge_repairs[i].clone();
8451 if request.phase == Phase::Fixing {
8452 let restored = match self.state.winner().cloned() {
8453 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8454 None => Err(anyhow::anyhow!("repair has no winning branch")),
8455 };
8456 let why = match restored {
8457 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8458 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8459 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8460 };
8461 self.hold_merge_repair(i, why)?;
8462 return Ok(false);
8463 }
8464 let failure = if i >= self.state.config.graph.land_rounds
8465 || self.state.config.graph.review_rounds == 0
8466 {
8467 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8468 } else {
8469 None
8470 };
8471 if let Some(why) = failure {
8472 self.hold_merge_repair(i, why)?;
8473 return Ok(false);
8474 }
8475 if request.phase == Phase::Requested {
8476 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8477 Ok(pr) => pr,
8478 Err(e) => {
8479 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8480 return Ok(false);
8481 }
8482 };
8483 self.state.merge_repairs[i].phase = Phase::Fixing;
8484 self.state.land_approval = None;
8485 self.state.land_armed_head = None;
8486 self.state.save()?;
8487 let evidence = serde_json::to_string_pretty(&request)?;
8490 let reason = format!(
8491 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8492 );
8493 let budget = self.state.config.graph.land_rounds;
8494 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8495 match result {
8496 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8497 Ok(land::Fixed::Failed(why)) => {
8498 self.hold_merge_repair(
8499 i,
8500 format!(
8501 "repair failed: {why}; inspect the branch and run an explicit review"
8502 ),
8503 )?;
8504 return Ok(false);
8505 }
8506 Err(e) => {
8507 self.hold_merge_repair(
8508 i,
8509 format!(
8510 "repair failed: {e:#}; inspect the branch and run an explicit review"
8511 ),
8512 )?;
8513 return Ok(false);
8514 }
8515 }
8516 self.begin_merge_repair_review(i)?;
8520 }
8521 self.review_loop().await?;
8522 self.gate().await?;
8523 if self.resource_halt()? || self.state.parked {
8524 return Ok(false);
8525 }
8526 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8527 != Some(RunStatus::Gating)
8528 || !self.state.gate_status().ok()
8529 {
8530 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8531 return Ok(false);
8532 }
8533 let winner = self
8536 .state
8537 .winner()
8538 .cloned()
8539 .context("repair has no winning branch")?;
8540 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8541 if self.state.gate_head.as_deref() != Some(after.as_str())
8542 || !git::is_clean(&winner.worktree).await?
8543 {
8544 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8545 return Ok(false);
8546 }
8547 let expected = self
8548 .state
8549 .pr_pushed_head
8550 .as_deref()
8551 .unwrap_or(&request.head)
8552 .to_owned();
8553 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8554 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8557 let refspec = format!("{after}:refs/heads/{public}");
8558 let push = git::git_raw(
8559 &winner.worktree,
8560 &["push", &lease, &self.state.config.merge.remote, &refspec],
8561 )
8562 .await;
8563 match push {
8564 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8565 Ok(out) => {
8566 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8567 return Ok(false);
8568 }
8569 Err(e) => {
8570 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8571 return Ok(false);
8572 }
8573 }
8574 self.state.merge_repairs[i].phase = Phase::Complete;
8575 self.state.status = RunStatus::Landing;
8576 self.state.event(
8577 "repair",
8578 "new review and gate completed; fresh owner merge decision required",
8579 );
8580 self.state.save()?;
8581 Ok(true)
8582 }
8583
8584 fn merge_repair_artifacts(&self) -> PathBuf {
8585 let artifacts = agent::artifacts_dir(&self.state.dir());
8586 match self
8587 .state
8588 .merge_repairs
8589 .last()
8590 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8591 {
8592 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8593 None => artifacts,
8594 }
8595 }
8596
8597 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8598 self.state.reviews.clear();
8599 self.state.gate.clear();
8600 self.state.gate_ran = false;
8601 self.state.gate_head = None;
8602 self.state.gate_fixes.clear();
8603 self.state.pre_gate.clear();
8604 self.state.pre_gate_commit = None;
8605 self.state.contested_handoff = None;
8606 self.state.land_reruns = None;
8607 self.state.land_approval = None;
8608 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8609 self.state.status = RunStatus::Reviewing;
8610 self.state.save()
8611 }
8612
8613 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8614 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8615 self.state.merge_repairs[index].failure = Some(why.clone());
8616 self.state.status = RunStatus::Blocked;
8617 self.state.event("repair", why);
8618 self.state.save()
8619 }
8620
8621 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8625 if let Some(existing) = self.state.seats.get(key)
8626 && existing.agent == agent
8627 {
8628 return existing.clone();
8629 }
8630 let fresh = if self.state.seats.contains_key(key) {
8633 handover_seat(key, agent, self.state.next_seat_seed())
8634 } else {
8635 SeatState::new(key, agent, self.state.seed)
8636 };
8637 self.state.seats.insert(key.to_owned(), fresh.clone());
8638 fresh
8639 }
8640
8641 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8647 match self.state.seats.get(key) {
8648 Some(s) if s.agent != spec.id => {
8649 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8650 }
8651 _ => spec,
8652 }
8653 }
8654
8655 fn view(&self, c: &Candidate) -> CandidateView {
8657 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8658 .unwrap_or_default();
8659 let (patch, _) = blind::sanitize_patch(
8660 &format!("candidate {} patch", c.label),
8661 &raw,
8662 &self.state.config.blind,
8663 );
8664 CandidateView {
8665 label: c.label,
8666 branch: c.branch.clone(),
8667 summary: c.summary.clone(),
8668 stat: c.stat.clone(),
8669 patch,
8670 }
8671 }
8672
8673 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8675 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8676 prompt::judge(
8677 "(see above)",
8678 &views,
8679 self.roles.judges.len(),
8680 base_short,
8681 "en",
8682 )
8683 }
8684
8685 fn vote_prompt_full(
8691 &self,
8692 j: usize,
8693 labels: &[char],
8694 language: &str,
8695 candidates: &[Candidate],
8696 base_short: &str,
8697 ) -> String {
8698 let mut text = format!(
8699 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8700 prompt::final_vote(labels, language),
8701 self.state.instruction,
8702 self.candidate_block(candidates, base_short)
8703 );
8704 if let Some(own) = self
8705 .state
8706 .judgements
8707 .get(j)
8708 .filter(|r| !r.ranking.is_empty())
8709 {
8710 let reasons = own
8711 .reasons
8712 .iter()
8713 .map(|(k, v)| format!("- {k}: {v}"))
8714 .collect::<Vec<_>>()
8715 .join("\n");
8716 text.push_str(&format!(
8717 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8718 own.ranking.iter().collect::<String>(),
8719 if reasons.is_empty() {
8720 ""
8721 } else {
8722 ", because:\n"
8723 }
8724 ));
8725 }
8726 if !self.state.deliberation.is_empty() {
8727 text.push_str("\n# What was argued before this vote\n");
8728 for t in self.transcript(&[], j) {
8729 text.push_str(&format!(
8730 "\n## {}{}\n\n{}\n",
8731 t.who,
8732 if t.is_self { " (you)" } else { "" },
8733 t.body.trim()
8734 ));
8735 }
8736 }
8737 text
8738 }
8739
8740 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8747 let mut turns = Vec::new();
8748 for j in &self.state.judgements {
8749 if j.ranking.is_empty() {
8750 continue;
8751 }
8752 let reasons = j
8753 .reasons
8754 .iter()
8755 .map(|(k, v)| format!("- {k}: {v}"))
8756 .collect::<Vec<_>>()
8757 .join("\n");
8758 turns.push(Turn {
8759 who: format!("Judge {} (opening ranking)", j.judge),
8760 is_self: j.judge == self_idx + 1,
8761 body: format!(
8762 "Ranked {}{}{reasons}",
8763 j.ranking.iter().collect::<String>(),
8764 if reasons.is_empty() {
8765 ""
8766 } else {
8767 ", because:\n"
8768 }
8769 ),
8770 });
8771 }
8772 for t in self
8773 .state
8774 .deliberation
8775 .iter()
8776 .flat_map(|r| r.turns.iter())
8777 .chain(current)
8778 {
8779 turns.push(Turn {
8780 who: format!("Judge {}", t.judge),
8781 is_self: t.judge == self_idx + 1,
8782 body: t.body.clone(),
8783 });
8784 }
8785 turns
8786 }
8787}
8788
8789fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8791 agent::has_session(spec.kind, seat, sessions)
8792}
8793
8794fn next_untried_in_roster<'a>(
8815 roster: &'a [AgentSpec],
8816 start: usize,
8817 tried: &BTreeSet<String>,
8818) -> Option<&'a AgentSpec> {
8819 roster
8820 .get(start + 1..)?
8821 .iter()
8822 .find(|s| !tried.contains(&s.id))
8823}
8824
8825fn pick_successor<'a>(
8846 roster: &'a [AgentSpec],
8847 start: usize,
8848 tried: &BTreeSet<String>,
8849 carried: Option<&BTreeSet<String>>,
8850 others: &BTreeSet<String>,
8851) -> Option<&'a AgentSpec> {
8852 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8853 roster
8854 .get(start + 1..)
8855 .and_then(|tail| {
8856 tail.iter()
8857 .filter(free)
8858 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8859 })
8860 .or_else(|| {
8861 carried
8862 .is_some()
8863 .then(|| roster.iter().find(free))
8864 .flatten()
8865 })
8866 .or_else(|| match carried {
8867 Some(c) => next_for_seat(roster, start, tried, c),
8868 None => next_untried_in_roster(roster, start, tried),
8869 })
8870}
8871
8872fn next_for_seat<'a>(
8882 roster: &'a [AgentSpec],
8883 start: usize,
8884 round_tried: &BTreeSet<String>,
8885 carried_failed: &BTreeSet<String>,
8886) -> Option<&'a AgentSpec> {
8887 roster
8888 .get(start + 1..)?
8889 .iter()
8890 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8891 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8892}
8893
8894fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8900 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8901 return spec;
8902 };
8903 let ok = |id: &str| !h.failed.contains(id);
8904 if let Some(last) = h.last_ok.as_deref()
8905 && ok(last)
8906 && let Some(s) = roster.iter().find(|s| s.id == last)
8907 {
8908 return s.clone();
8909 }
8910 if ok(&spec.id) {
8911 return spec;
8912 }
8913 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8914 roster
8915 .get(from + 1..)
8916 .into_iter()
8917 .flatten()
8918 .chain(roster.iter())
8919 .find(|s| ok(&s.id))
8920 .cloned()
8921 .unwrap_or(spec)
8922}
8923
8924pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8928 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8929}
8930
8931const TIMED_OUT: &str = "timed out";
8934
8935impl FailClass {
8936 fn of(out: &AgentOutcome) -> Option<Self> {
8938 match out {
8939 AgentOutcome::Ok(_) => None,
8940 AgentOutcome::Quota(_) => Some(Self::Quota),
8941 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8942 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8943 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8944 AgentOutcome::ResourceBlocked(_) => None,
8947 }
8948 }
8949
8950 fn stem_word(&self) -> &'static str {
8952 match self {
8953 Self::Quota => "quota",
8954 _ => "handover",
8955 }
8956 }
8957}
8958
8959fn failure_signature(msg: &str) -> String {
8963 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8964 if let Some(code) = line
8967 .strip_prefix("process exited with no usable output (exit code ")
8968 .and_then(|s| s.strip_suffix(')'))
8969 .and_then(|s| s.parse::<i32>().ok())
8970 {
8971 line = format!("exited with some({code}) and no usable output");
8972 } else if matches!(
8973 line.as_str(),
8974 "process exited with no usable output (no exit code reported)"
8975 | "process exited with no usable output (exit code unavailable)"
8976 ) {
8977 line = "exited with none and no usable output".to_owned();
8978 }
8979 let mut out = Vec::new();
8980 for word in line.split_whitespace() {
8981 if word.contains('/') || word.contains('\\') {
8982 out.push("<path>".to_owned());
8983 continue;
8984 }
8985 let mut w = String::new();
8986 let mut in_digits = false;
8987 for c in word.chars() {
8988 if c.is_ascii_digit() {
8989 if !in_digits {
8990 w.push('#');
8991 }
8992 in_digits = true;
8993 } else {
8994 in_digits = false;
8995 w.push(c);
8996 }
8997 }
8998 out.push(w);
8999 }
9000 out.join(" ").chars().take(120).collect()
9001}
9002
9003fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9010 match cur {
9011 FailClass::Quota | FailClass::Timeout => true,
9012 FailClass::Other(_) => prev != Some(cur),
9013 }
9014}
9015
9016fn fail_reason(out: &AgentOutcome) -> String {
9018 match out {
9019 AgentOutcome::Ok(_) => String::new(),
9020 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9021 AgentOutcome::Dropped(o) => format!(
9022 "the CLI dropped the stream ({})",
9023 o.dropped
9024 .as_ref()
9025 .map(|d| d.why.as_str())
9026 .unwrap_or("it ended without delivering its answer")
9027 ),
9028 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9029 e.lines().next().unwrap_or("").chars().take(160).collect()
9030 }
9031 }
9032}
9033
9034pub(crate) fn record_handover(
9037 state: &mut RunState,
9038 node: &str,
9039 seat: &str,
9040 from: &str,
9041 to: &str,
9042 class: &FailClass,
9043 reason: &str,
9044) {
9045 let message = if *class == FailClass::Quota {
9046 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9047 } else {
9048 format!("{seat}: handed over {from} -> {to} ({reason})")
9049 };
9050 state.event(node, message);
9051 state.handovers.push(Handover {
9052 at: Timestamp::now(),
9053 node: node.to_owned(),
9054 seat: seat.to_owned(),
9055 from: from.to_owned(),
9056 to: to.to_owned(),
9057 reason: reason.to_owned(),
9058 });
9059}
9060
9061fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9075 commands.iter().any(|c| c.exit_code.is_none())
9076}
9077
9078fn verified_noop_claim(
9091 usable: bool,
9092 commands: &[agent::CommandEvidence],
9093 text: &str,
9094) -> Option<String> {
9095 (usable && !has_unconfirmed_command(commands))
9096 .then(|| verdict::verified_noop(text))
9097 .flatten()
9098}
9099
9100fn short(commit: &str) -> String {
9101 commit.chars().take(7).collect()
9102}
9103
9104fn make_executable(path: &Path) -> Result<()> {
9105 #[cfg(unix)]
9106 {
9107 use std::os::unix::fs::PermissionsExt as _;
9108 let mut perms = std::fs::metadata(path)?.permissions();
9109 perms.set_mode(0o755);
9110 std::fs::set_permissions(path, perms)?;
9111 }
9112 #[cfg(not(unix))]
9113 {
9114 let _ = path;
9115 }
9116 Ok(())
9117}
9118
9119struct WaveCtx<'a> {
9126 run: &'a str,
9129 node: &'a str,
9131 prompts: &'a Prompts,
9132 cache: Option<&'a Path>,
9134 round: Option<usize>,
9137 carry_seats: bool,
9141 pauses: Option<(&'a Pause, &'a Pause)>,
9144}
9145
9146fn repository_wait(
9148 state: &RunState,
9149 pauses: Option<(&Pause, &Pause)>,
9150) -> crate::repo_lock::WaitPolicy {
9151 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9152 let inherited = crate::repo_lock::WaitPolicy::current();
9153 crate::repo_lock::WaitPolicy::new(
9154 Duration::from_secs(state.config.graph.timeout_resource_wait),
9155 move || {
9156 inherited.cancelled()
9157 || pauses
9158 .as_ref()
9159 .is_some_and(|(a, b)| a.parked() || b.parked())
9160 },
9161 )
9162}
9163
9164fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9165 if error.is::<crate::repo_lock::ResourceBlocked>() {
9166 state.resource_block = Some(crate::run::ResourceBlock {
9167 node: "repository".into(),
9168 kind: "repository coordination".into(),
9169 detail: error.to_string(),
9170 at: Timestamp::now(),
9171 });
9172 state.event("resource", error.to_string());
9173 }
9174}
9175
9176async fn coordinate_operation(
9177 state: &mut RunState,
9178 cwd: &Path,
9179 wait: &crate::repo_lock::WaitPolicy,
9180) -> Result<Option<crate::repo_lock::SeatLease>> {
9181 let result =
9182 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9183 .await;
9184 if let Err(e) = &result {
9185 record_repository_block(state, e);
9186 }
9187 result
9188}
9189
9190async fn run_one(
9192 job: SeatJob,
9193 sem: Arc<Semaphore>,
9194 ctx: &WaveCtx<'_>,
9195 state: &mut RunState,
9196 attempt: usize,
9197) -> (SeatState, AgentOutcome) {
9198 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9199 .await
9200 .pop()
9201 .expect("one job in, one result out");
9202 (seat, out)
9203}
9204
9205async fn wave(
9211 jobs: Vec<SeatJob>,
9212 sem: Arc<Semaphore>,
9213 ctx: &WaveCtx<'_>,
9214 state: &mut RunState,
9215 attempt: usize,
9216) -> Vec<(usize, SeatState, AgentOutcome)> {
9217 let WaveCtx {
9218 run,
9219 node,
9220 prompts,
9221 cache,
9222 round,
9223 carry_seats: _,
9224 pauses,
9225 } = *ctx;
9226 for job in &jobs {
9227 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9228 }
9229 if let Err(e) = state.save() {
9230 tracing::warn!("could not persist in-progress seats: {e:#}");
9235 }
9236 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9252 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9256 let cache_guard = if let Some(cache_dir) = cache {
9257 if jobs_had_a_writer {
9258 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9259 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9260 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9261 {
9262 Ok(Some(g)) => Some(g),
9263 Ok(None) => {
9266 state.resource_block = Some(crate::run::ResourceBlock {
9267 node: node.to_owned(),
9268 kind: "shared build cache".to_owned(),
9269 detail: "the run is parking".to_owned(),
9270 at: Timestamp::now(),
9271 });
9272 return block_wave(
9273 jobs,
9274 state,
9275 node,
9276 attempt,
9277 "parked while waiting for the shared build cache",
9278 );
9279 }
9280 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9283 Err(e) => {
9284 let why = format!("{e:#}");
9288 state.resource_block = Some(crate::run::ResourceBlock {
9289 node: node.to_owned(),
9290 kind: "shared build cache".to_owned(),
9291 detail: why.clone(),
9292 at: Timestamp::now(),
9293 });
9294 state.event(
9295 "resource",
9296 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9297 );
9298 return block_wave(jobs, state, node, attempt, &why);
9299 }
9300 }
9301 } else {
9302 None
9303 }
9304 } else {
9305 None
9306 };
9307 state.seats_spawning(node, attempt);
9310 if let Err(e) = state.save() {
9311 tracing::warn!("could not persist the start of a wave: {e:#}");
9312 }
9313 let slot_limit = resource_limit
9317 + jobs
9318 .iter()
9319 .map(|j| j.timeout)
9320 .max()
9321 .unwrap_or_default()
9322 .saturating_mul(jobs.len() as u32);
9323 let mut set = tokio::task::JoinSet::new();
9324 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9327 let overlay = prompts.overlay(node);
9328 let repository_wait = repository_wait(state, pauses);
9329 for (i, mut job) in jobs.into_iter().enumerate() {
9330 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9331 if cache.is_some() {
9332 job.prompt.push('\n');
9333 job.prompt
9334 .push_str(&prompt::build_cache_note(node, job.allow_write));
9335 }
9336 let sem = Arc::clone(&sem);
9337 let run = run.to_owned();
9338 let node = node.to_owned();
9339 let attachments = if node == "implement" {
9342 state.attachments.clone()
9343 } else {
9344 Vec::new()
9345 };
9346 let cache = cache
9357 .filter(|_| job.allow_write && cache_guard.is_some())
9358 .map(Path::to_path_buf);
9359 let spawned_tx = spawned_tx.clone();
9360 let repository_wait = repository_wait.clone();
9361 let coordination = state.config.coordination.clone();
9362 let fetched_base = state.fetched_base.clone();
9363 set.spawn(async move {
9364 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9368 Ok(p) => p,
9369 Err(_) => {
9370 return (
9371 i,
9372 job.seat,
9373 AgentOutcome::ResourceBlocked(format!(
9374 "no parallel slot became free within {}s",
9375 slot_limit.as_secs()
9376 )),
9377 );
9378 }
9379 };
9380 spawned_tx.send(job.seat.key.clone()).ok();
9381 let mut seat = job.seat;
9382 let out = repository_wait
9383 .scope(agent::invoke(
9384 &job.spec,
9385 &mut seat,
9386 &Invocation {
9387 coordination: coordination.as_ref(),
9388 fetched_base: fetched_base.as_ref(),
9389 cwd: &job.cwd,
9390 prompt: &job.prompt,
9391 timeout: job.timeout,
9392 allow_write: job.allow_write,
9393 unsandboxed: false,
9394 sessions: job.sessions,
9395 artifacts: &job.artifacts,
9396 stem: &job.stem,
9397 run: &run,
9398 node: &node,
9399 cache_dir: cache.as_deref(),
9400 attachments: &attachments,
9401 writable: &[],
9402 },
9403 ))
9404 .await;
9405 let out = match out {
9406 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9407 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9408 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9416 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9417 Ok(o) => AgentOutcome::Failed(format!(
9418 "process exited with no usable output ({})",
9419 crate::notices::exit_status(o.exit_code)
9420 )),
9421 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9422 AgentOutcome::ResourceBlocked(e.to_string())
9423 }
9424 Err(e) => AgentOutcome::Failed(e.to_string()),
9425 };
9426 (i, seat, out)
9427 });
9428 }
9429 drop(spawned_tx);
9430 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9431 loop {
9432 let joined = tokio::select! {
9433 Some(key) = spawned_rx.recv() => {
9434 state.seat_spawned(&key);
9436 if let Err(e) = state.save() {
9437 tracing::warn!("could not persist a seat's start: {e:#}");
9438 }
9439 continue;
9440 }
9441 joined = set.join_next() => joined,
9442 };
9443 let Some(joined) = joined else {
9444 break;
9445 };
9446 let (i, seat, out) = match joined {
9447 Ok(v) => v,
9448 Err(e) => {
9452 tracing::error!("agent task panicked: {e}");
9453 continue;
9454 }
9455 };
9456 state.seat_finished(&seat.key);
9457 if let AgentOutcome::ResourceBlocked(why) = &out
9458 && state.resource_block.is_none()
9459 {
9460 state.resource_block = Some(crate::run::ResourceBlock {
9461 node: node.to_owned(),
9462 kind: if why.contains("repository coordination") {
9463 "repository coordination"
9464 } else {
9465 "parallel slot"
9466 }
9467 .to_owned(),
9468 detail: why.clone(),
9469 at: Timestamp::now(),
9470 });
9471 state.event("resource", format!("{node}: no agent was started - {why}"));
9472 }
9473 record_jobs(state, node, round, &seat.key, &out);
9474 if let Err(e) = state.save() {
9475 tracing::warn!("could not persist a seat's completion: {e:#}");
9476 }
9477 if collected.len() <= i {
9478 collected.resize_with(i + 1, || None);
9479 }
9480 collected[i] = Some((i, seat, out));
9481 }
9482 if state
9488 .active
9489 .values()
9490 .any(|a| a.node == node && a.attempt == attempt)
9491 {
9492 state
9493 .active
9494 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9495 if let Err(e) = state.save() {
9496 tracing::warn!("could not persist the end of a wave: {e:#}");
9497 }
9498 }
9499 if let Some(cache_dir) = cache
9506 && jobs_had_a_writer
9507 {
9508 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9509 }
9510 if let Some(guard) = cache_guard {
9511 guard.release();
9512 }
9513 collected.into_iter().flatten().collect()
9514}
9515
9516fn record_jobs(
9527 state: &mut RunState,
9528 node: &str,
9529 round: Option<usize>,
9530 seat: &str,
9531 out: &AgentOutcome,
9532) {
9533 let commands: &[agent::CommandEvidence] = match out {
9534 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9535 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9536 };
9537 let checked_at = Timestamp::now();
9538 for c in commands {
9539 state.jobs.push(JobRecord {
9540 node: node.to_owned(),
9541 round,
9542 seat: seat.to_owned(),
9543 id: c.id.clone(),
9544 description: c.description.clone(),
9545 checked_at,
9546 status: match c.exit_code {
9547 Some(0) => JobStatus::Completed,
9548 Some(_) => JobStatus::Failed,
9549 None => JobStatus::Unknown,
9550 },
9551 exit_code: c.exit_code,
9552 result_summary: c.result_summary.clone(),
9553 source: c.source.clone(),
9554 });
9555 }
9556}
9557
9558fn round_is_clean(
9579 blocking: usize,
9580 e2e_ok: bool,
9581 answered: usize,
9582 expected: usize,
9583 quota_missing: usize,
9584 policy: IncompleteReviewPolicy,
9585) -> bool {
9586 if blocking != 0 || !e2e_ok {
9587 return false;
9588 }
9589 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9590 return true;
9591 }
9592 answered > 0 && expected - answered <= quota_missing
9593}
9594
9595fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9619 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9620 return Some(RunStatus::Gating);
9621 }
9622 let last = reviews.last()?;
9623 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9624 if reviews.len() < max_rounds && !stagnant {
9625 return None;
9626 }
9627 if last.incomplete() && last.blocking == 0 {
9628 return Some(RunStatus::Blocked);
9629 }
9630 if last.e2e_status() == E2eStatus::ResourceBlocked {
9631 return None;
9632 }
9633 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9634 RunStatus::Gating
9635 } else {
9636 RunStatus::Blocked
9637 })
9638}
9639
9640pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9655 if nudged {
9656 (full / 4).max(Duration::from_secs(120)).min(full)
9657 } else {
9658 full
9659 }
9660}
9661
9662#[allow(clippy::too_many_arguments)]
9691async fn ask_json_wave<T>(
9692 jobs: Vec<SeatJob>,
9693 sem: Arc<Semaphore>,
9694 retries: usize,
9695 roster: &[AgentSpec],
9696 ctx: &WaveCtx<'_>,
9697 losses: &mut Vec<QuotaLoss>,
9698 state: &mut RunState,
9699 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9700) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9701where
9702 T: serde::de::DeserializeOwned + Send + 'static,
9703{
9704 ask_wave_with(
9705 jobs,
9706 sem,
9707 retries,
9708 roster,
9709 ctx,
9710 losses,
9711 state,
9712 &|text: &str| {
9713 let v = verdict::extract_json::<T>(text)?;
9714 validate(&v)?;
9715 Ok(v)
9716 },
9717 )
9718 .await
9719}
9720
9721#[allow(clippy::too_many_arguments)]
9730async fn ask_wave_with<T>(
9731 jobs: Vec<SeatJob>,
9732 sem: Arc<Semaphore>,
9733 retries: usize,
9734 roster: &[AgentSpec],
9735 ctx: &WaveCtx<'_>,
9736 losses: &mut Vec<QuotaLoss>,
9737 state: &mut RunState,
9738 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9739) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9740where
9741 T: Send + 'static,
9742{
9743 let n = jobs.len();
9744 let originals: Vec<SeatJob> = jobs;
9745 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9746 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9747 let mut nudges: Vec<usize> = vec![0; n];
9754 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9758 let mut tried: Vec<BTreeSet<String>> = specs
9759 .iter()
9760 .map(|s| BTreeSet::from([s.id.clone()]))
9761 .collect();
9762 let starts: Vec<usize> = specs
9763 .iter()
9764 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9765 .collect();
9766 let carry = ctx.carry_seats && !roster.is_empty();
9767 let carried: Vec<BTreeSet<String>> = originals
9770 .iter()
9771 .map(|j| {
9772 state
9773 .seat_history
9774 .get(&j.seat.key)
9775 .filter(|_| carry)
9776 .map(|h| h.failed.clone())
9777 .unwrap_or_default()
9778 })
9779 .collect();
9780 let mut prev: Vec<Option<FailClass>> = originals
9781 .iter()
9782 .map(|j| {
9783 state
9784 .seat_history
9785 .get(&j.seat.key)
9786 .filter(|_| carry)
9787 .and_then(|h| h.last_fail.clone())
9788 })
9789 .collect();
9790 let next_agent =
9791 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9792 let others: BTreeSet<String> = specs
9793 .iter()
9794 .enumerate()
9795 .filter(|(j, _)| *j != i)
9796 .map(|(_, s)| s.id.clone())
9797 .collect();
9798 pick_successor(
9799 roster,
9800 starts[i],
9801 tried,
9802 carry.then(|| &carried[i]),
9803 &others,
9804 )
9805 .cloned()
9806 };
9807 let mut fresh: Vec<Option<String>> = vec![None; n];
9808 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9809 let mut pending: Vec<usize> = (0..n).collect();
9810
9811 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9816 for round in 0..max_rounds {
9817 if pending.is_empty() {
9818 break;
9819 }
9820 let mut batch = Vec::with_capacity(pending.len());
9821 let mut renudged: Vec<&str> = Vec::new();
9822 for &i in &pending {
9823 let src = &originals[i];
9824 let full: &str = match &src.handover {
9829 Some(h) if specs[i].id != src.spec.id => h,
9830 _ => &src.prompt,
9831 };
9832 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9835 (
9836 full.to_owned(),
9837 src.timeout,
9838 format!("{}-{word}-{}", src.stem, specs[i].id),
9839 )
9840 } else if nudges[i] == 0 {
9841 (full.to_owned(), src.timeout, src.stem.clone())
9842 } else {
9843 renudged.push(src.seat.key.as_str());
9844 let why = done[i]
9845 .as_ref()
9846 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9847 .unwrap_or_else(|| "no parsable answer".to_owned());
9848 let nudge = prompt::nudge(&why);
9849 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9850 let prompt = if nudged {
9851 nudge
9852 } else {
9853 format!("{full}\n\n---\n\n{nudge}")
9854 };
9855 (
9856 prompt,
9857 retry_budget(src.timeout, nudged),
9858 format!("{}-retry{}", src.stem, nudges[i]),
9859 )
9860 };
9861 batch.push(SeatJob {
9862 spec: specs[i].clone(),
9863 seat: seats[i].clone(),
9864 cwd: src.cwd.clone(),
9865 prompt,
9866 timeout,
9867 allow_write: src.allow_write,
9868 sessions: src.sessions,
9869 artifacts: src.artifacts.clone(),
9870 stem,
9871 handover: None,
9872 });
9873 }
9874
9875 if !renudged.is_empty() {
9876 state.event(
9877 ctx.node,
9878 format!("retry {round}: re-asking {}", renudged.join(", ")),
9879 );
9880 }
9881 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9882 let mut still = Vec::new();
9883 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9884 seats[i] = seat;
9885 if let AgentOutcome::ResourceBlocked(why) = &out {
9888 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9889 continue;
9890 }
9891 let class = FailClass::of(&out);
9892 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9895 && nudges[i] < retries
9896 && !roster.is_empty();
9897 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9898 let next = should_hand_over(prev[i].as_ref(), &cur)
9899 .then(|| next_agent(i, &tried[i], &specs))
9900 .flatten();
9901 if carry {
9902 let h = state
9903 .seat_history
9904 .entry(originals[i].seat.key.clone())
9905 .or_default();
9906 h.failed.insert(specs[i].id.clone());
9907 h.last_fail = Some(cur.clone());
9908 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9909 h.last_ok = None;
9910 }
9911 if let Err(e) = state.save() {
9914 tracing::warn!("could not persist a seat's failure history: {e:#}");
9915 }
9916 }
9917 if let Some(next) = next {
9918 record_handover(
9919 state,
9920 ctx.node,
9921 &originals[i].seat.key,
9922 &specs[i].id,
9923 &next.id,
9924 &cur,
9925 &fail_reason(&out),
9926 );
9927 tried[i].insert(next.id.clone());
9928 prev[i] = Some(cur.clone());
9929 seats[i] =
9930 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9931 specs[i] = next;
9932 fresh[i] = Some(cur.stem_word().to_owned());
9933 nudges[i] = 0;
9934 done[i] = Some(Err(anyhow::anyhow!(
9935 "handed over after: {}",
9936 fail_reason(&out)
9937 )));
9938 still.push(i);
9939 continue;
9940 }
9941 }
9942 if carry
9943 && !nudge_first
9944 && let Some(cur) = class.clone()
9945 {
9946 let h = state
9948 .seat_history
9949 .entry(originals[i].seat.key.clone())
9950 .or_default();
9951 h.failed.insert(specs[i].id.clone());
9952 h.last_fail = Some(cur);
9953 }
9954 let parsed = match out {
9955 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9956 AgentOutcome::Quota(o) => {
9957 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9958 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9959 }
9960 AgentOutcome::Dropped(o) => {
9964 let why = o
9965 .dropped
9966 .as_ref()
9967 .map(|d| d.why.as_str())
9968 .unwrap_or("the CLI ended the stream without delivering its answer");
9969 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9970 }
9971 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9972 Err(anyhow::anyhow!(e))
9973 }
9974 };
9975 let quota = class == Some(FailClass::Quota);
9976 let failed = parsed.is_err();
9977 done[i] = Some(parsed);
9978 if carry && !failed {
9979 let h = state
9980 .seat_history
9981 .entry(originals[i].seat.key.clone())
9982 .or_default();
9983 h.failed.remove(&specs[i].id);
9984 h.last_ok = Some(specs[i].id.clone());
9985 h.last_fail = None;
9986 }
9987 let agent_failure = class.is_some()
9995 && !nudge_first
9996 && !roster.is_empty()
9997 && next_agent(i, &tried[i], &specs).is_some();
9998 if failed && !quota && !agent_failure && nudges[i] < retries {
9999 nudges[i] += 1;
10000 still.push(i);
10001 }
10002 }
10003 pending = still;
10004 }
10005
10006 for (i, q) in last_quota.into_iter().enumerate() {
10009 if let Some(reset) = q {
10010 losses.push(QuotaLoss {
10011 seat: originals[i].seat.key.clone(),
10012 node: ctx.node.to_owned(),
10013 at: Timestamp::now(),
10014 reset,
10015 });
10016 }
10017 }
10018
10019 seats
10020 .into_iter()
10021 .zip(done)
10022 .zip(nudges)
10023 .map(|((seat, res), attempts)| {
10024 (
10025 seat,
10026 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10027 attempts,
10028 )
10029 })
10030 .collect()
10031}
10032
10033fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10035 results
10036 .iter()
10037 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10038}
10039
10040fn block_wave(
10044 jobs: Vec<SeatJob>,
10045 state: &mut RunState,
10046 node: &str,
10047 attempt: usize,
10048 why: &str,
10049) -> Vec<(usize, SeatState, AgentOutcome)> {
10050 state.resource_wait = None;
10051 state
10052 .active
10053 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10054 if let Err(e) = state.save() {
10055 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10056 }
10057 jobs.into_iter()
10058 .enumerate()
10059 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10060 .collect()
10061}
10062
10063async fn acquire_cache_lease(
10076 state: &mut RunState,
10077 cache_dir: &Path,
10078 owner: &crate::cache::Owner,
10079 budget: Duration,
10080 context: &str,
10081 cancel: &(dyn Fn() -> bool + Sync),
10082) -> Result<Option<crate::cache::Guard>> {
10083 let home = crate::run::home();
10084 let started = Instant::now();
10085 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10086 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10087 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10088 Err(e) => {
10089 state.event(
10090 "verify",
10091 format!("{context}: could not check the shared build cache: {e:#}"),
10092 );
10093 if let Err(e2) = state.save() {
10094 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10095 }
10096 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10097 }
10098 };
10099 state.event(
10100 "verify",
10101 format!(
10102 "{context}: waiting for the shared build cache at {} ({})",
10103 cache_dir.display(),
10104 busy.describe()
10105 ),
10106 );
10107 state.resource_wait = Some(crate::run::ResourceWait {
10110 node: context.to_owned(),
10111 kind: "shared build cache".to_owned(),
10112 holder: busy.describe(),
10113 since: Timestamp::now(),
10114 limit_secs: budget.as_secs(),
10115 });
10116 if let Err(e) = state.save() {
10117 tracing::warn!("could not persist a cache wait: {e:#}");
10118 }
10119 let remaining = budget.saturating_sub(started.elapsed());
10120 let waited = crate::cache::wait_for_unless(
10121 &home,
10122 cache_dir,
10123 owner,
10124 remaining,
10125 Duration::from_secs(5),
10126 cancel,
10127 )
10128 .await;
10129 state.resource_wait = None;
10130 match waited {
10131 Ok(g) => Ok(g),
10132 Err(e) => {
10133 state.event("verify", format!("{context}: {e:#}"));
10134 if let Err(e2) = state.save() {
10135 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10136 }
10137 Err(e)
10138 }
10139 }
10140}
10141
10142#[derive(Debug)]
10147struct CacheCheckError(String);
10148
10149impl std::fmt::Display for CacheCheckError {
10150 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10151 f.write_str(&self.0)
10152 }
10153}
10154
10155impl std::error::Error for CacheCheckError {}
10156
10157#[allow(clippy::too_many_arguments)]
10178async fn with_cache_lease<'s, F, Fut>(
10179 state: &'s mut RunState,
10180 cache_dir: Option<&Path>,
10181 node: &str,
10182 seat: &str,
10183 worktree: &Path,
10184 head: &str,
10185 budget: Duration,
10186 context: &str,
10187 wait: &crate::repo_lock::WaitPolicy,
10188 body: F,
10189) -> Result<(Vec<CommandOutcome>, bool)>
10190where
10191 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10192 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10193{
10194 let started = Instant::now();
10197 let home = crate::run::home();
10198 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10199 let guard = if let Some(cache_dir) = cache_dir {
10200 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10201 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10202 wait.cancelled()
10203 })
10204 .await
10205 .and_then(|g| {
10206 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10207 }) {
10208 Ok(g) => Some(g),
10209 Err(e) => {
10210 return Ok((
10211 vec![CommandOutcome {
10212 command: "(waiting for the shared build cache)".into(),
10213 code: None,
10214 output_tail: e.to_string(),
10215 duration_ms: started.elapsed().as_millis() as u64,
10216 resource_blocked: true,
10217 }],
10218 false,
10219 ));
10220 }
10221 }
10222 } else {
10223 None
10224 };
10225 let _coordination = coordinate_operation(state, worktree, wait).await?;
10226 if let Some(cache_dir) = cache_dir {
10227 let identity = crate::cache::Identity::new(worktree, head);
10228 if let Err(e) = crate::cache::ensure_fresh(
10229 &home,
10230 cache_dir,
10231 &identity,
10232 &state.config.verify.cargo_manifests(),
10233 ) {
10234 state.event(
10235 "verify",
10236 format!(
10237 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10238 worktree.display(),
10239 short(head)
10240 ),
10241 );
10242 if let Some(guard) = guard {
10243 guard.release();
10244 }
10245 return Ok((
10246 vec![CommandOutcome {
10247 command: "(confirming the shared build cache is fresh)".into(),
10248 code: None,
10249 output_tail: e.to_string(),
10250 duration_ms: started.elapsed().as_millis() as u64,
10251 resource_blocked: true,
10252 }],
10253 false,
10254 ));
10255 }
10256 }
10257 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10258 if !timed_out_pids.is_empty() {
10259 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10260 }
10261 drop(_coordination);
10264 if let Some(guard) = guard {
10265 guard.release();
10266 }
10267 Ok((outcomes, retried))
10268}
10269
10270async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10282 wait_for_pids_with(
10283 pids,
10284 crate::proc::pid_alive,
10285 LEASE_RELEASE_POLL,
10286 LEASE_RELEASE_MAX_WAIT,
10287 )
10288 .await;
10289}
10290
10291async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10297 pids: &[u32],
10298 alive: F,
10299 poll: Duration,
10300 max_wait: Duration,
10301) {
10302 let deadline = Instant::now() + max_wait;
10303 loop {
10304 if pids.iter().all(|&pid| !alive(pid)) {
10305 return;
10306 }
10307 if Instant::now() >= deadline {
10308 return;
10309 }
10310 tokio::time::sleep(poll).await;
10311 }
10312}
10313
10314fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10321 outcomes.iter().any(|o| o.resource_blocked)
10322}
10323
10324enum GateFix {
10326 Retry,
10328 Stop,
10331 Defer,
10334}
10335
10336fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10344 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10345 red.peek().is_some()
10346 && red.all(|o| {
10347 !o.resource_blocked
10348 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10349 && !o.output_tail.trim().is_empty()
10350 })
10351}
10352
10353fn e2e_outcome_label(o: &CommandOutcome) -> String {
10357 if o.ok() {
10358 return "pass".to_owned();
10359 }
10360 let reason = if o.build_failed() {
10361 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10362 } else {
10363 format!("FAIL ({:?})", o.code)
10364 };
10365 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10366}
10367
10368async fn run_e2e_with_retry(
10376 state: &mut RunState,
10377 shell: &[String],
10378 commands: &[String],
10379 worktree: &Path,
10380 timeout: Duration,
10381 context: &str,
10382) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10383 let (mut e2e, mut timed_out_pids) = run_commands(
10384 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10385 )
10386 .await;
10387 for o in &e2e {
10388 state.event(
10389 "verify",
10390 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10391 );
10392 }
10393 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10397 if verify_retried {
10398 state.event(
10399 "verify",
10400 format!(
10401 "{context}: verify could not build/link, not a test result — retrying once \
10402 before concluding"
10403 ),
10404 );
10405 let retried = run_commands(
10406 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10407 )
10408 .await;
10409 e2e = retried.0;
10410 timed_out_pids.extend(retried.1);
10413 for o in &e2e {
10414 state.event(
10415 "verify",
10416 format!(
10417 "{context}: retry `{}` -> {}",
10418 o.command,
10419 e2e_outcome_label(o)
10420 ),
10421 );
10422 }
10423 }
10424 (e2e, verify_retried, timed_out_pids)
10425}
10426
10427#[allow(clippy::too_many_arguments)]
10443async fn run_commands(
10444 state: &mut RunState,
10445 node: &str,
10446 task: &str,
10447 attempt: usize,
10448 shell: &[String],
10449 commands: &[String],
10450 cwd: &Path,
10451 timeout: Duration,
10452) -> (Vec<CommandOutcome>, Vec<u32>) {
10453 if commands.is_empty() {
10454 return (Vec::new(), Vec::new());
10459 }
10460 let mut out = Vec::new();
10461 let mut timed_out_pids = Vec::new();
10462 let total = commands.len();
10463 for (idx, command) in commands.iter().enumerate() {
10464 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10465 if let Err(e) = state.save() {
10466 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10467 }
10468 let started = Instant::now();
10469 let mut cmd = tokio::process::Command::new(&shell[0]);
10470 cmd.quiet();
10471 cmd.args(&shell[1..])
10472 .arg(command)
10473 .current_dir(cwd)
10474 .stdin(std::process::Stdio::null())
10475 .stdout(std::process::Stdio::piped())
10476 .stderr(std::process::Stdio::piped())
10477 .kill_on_drop(true);
10478 let spawned = cmd.spawn();
10479 let (code, body) = match spawned {
10480 Ok(child) => {
10481 let pid = child.id();
10486 match tokio::time::timeout(timeout, child.wait_with_output()).await {
10487 Ok(Ok(o)) => {
10488 let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
10489 body.push_str(&String::from_utf8_lossy(&o.stderr));
10490 (o.status.code(), body)
10491 }
10492 Ok(Err(e)) => (None, format!("failed to run: {e}")),
10493 Err(_) => {
10494 if let Some(pid) = pid {
10495 timed_out_pids.push(pid);
10496 }
10497 (None, format!("timed out after {}s", timeout.as_secs()))
10498 }
10499 }
10500 }
10501 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10502 };
10503 out.push(CommandOutcome {
10504 command: command.clone(),
10505 code,
10506 output_tail: tail(&body, OUTPUT_TAIL),
10507 duration_ms: started.elapsed().as_millis() as u64,
10508 resource_blocked: false,
10509 });
10510 }
10511 state.task_finished(task);
10512 if let Err(e) = state.save() {
10513 tracing::warn!("could not persist the end of {task}: {e:#}");
10514 }
10515 (out, timed_out_pids)
10516}
10517
10518fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10530 let repo = repo.display();
10531 match style {
10532 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10533 MergeStyle::Squash => {
10534 let subject = message
10537 .lines()
10538 .next()
10539 .unwrap_or(branch)
10540 .replace(['\\', '"', '$', '`'], "");
10541 format!(
10542 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10543 )
10544 }
10545 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10546 }
10547}
10548
10549const PR_TITLE_MAX: usize = 240;
10560
10561fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10572 let line = queue::first_line(text)?;
10573 let chars: Vec<char> = line.chars().collect();
10574 let (kept, mark) = if chars.len() <= max {
10575 (line.to_owned(), "")
10576 } else if let Some(end) = sentence_end(&chars, max) {
10577 (chars[..end].iter().collect::<String>(), "")
10578 } else {
10579 let room = max.saturating_sub(3);
10580 let cut = if chars[room].is_whitespace() {
10583 room
10584 } else {
10585 chars[..room]
10586 .iter()
10587 .rposition(|c| c.is_whitespace())
10588 .unwrap_or(room)
10589 };
10590 let head: String = chars[..cut].iter().collect();
10591 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10592 (head.to_owned(), "...")
10593 };
10594 if kept.is_empty() {
10595 return None;
10596 }
10597 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10598 return None;
10599 }
10600 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10601 return None;
10602 }
10603 Some(format!("{kept}{mark}"))
10604}
10605
10606fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10610 const MIN: usize = 20;
10611 for i in MIN..max.min(chars.len()) {
10612 if !matches!(chars[i], '.' | '!' | '?') {
10613 continue;
10614 }
10615 let Some(&next) = chars.get(i + 1) else {
10616 continue;
10617 };
10618 if !next.is_whitespace() {
10619 continue;
10620 }
10621 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10622 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10623 continue;
10624 }
10625 let word: String = chars[..i]
10626 .iter()
10627 .rev()
10628 .take_while(|c| !c.is_whitespace())
10629 .collect::<Vec<_>>()
10630 .into_iter()
10631 .rev()
10632 .collect();
10633 let word = word.to_ascii_lowercase();
10634 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10635 continue;
10636 }
10637 let end = chars[..i]
10638 .iter()
10639 .rposition(|c| !c.is_whitespace())
10640 .map_or(i, |p| p + 1);
10641 return Some(end);
10642 }
10643 None
10644}
10645
10646struct PrMessage {
10651 title: String,
10652 body: String,
10653}
10654
10655impl PrMessage {
10656 fn commit_message(&self) -> String {
10660 format!("{}\n\n{}", self.title, self.body)
10661 }
10662}
10663
10664fn title_marker(line: &str) -> Option<&str> {
10666 let line = line.trim();
10667 let head = line.get(..6)?;
10668 head.eq_ignore_ascii_case("title:")
10669 .then(|| line[6..].trim())
10670}
10671
10672fn summary_title(summary: &str) -> Option<String> {
10677 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10678 let raw = title_marker(first)?;
10679 if raw.is_empty() {
10680 return None;
10681 }
10682 let title = queue::title_from(raw, PR_TITLE_MAX);
10683 let lower = title.to_ascii_lowercase();
10684 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10685 return None;
10686 }
10687 Some(title)
10688}
10689
10690const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10692
10693pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10695
10696fn is_review_run(state: &RunState) -> bool {
10700 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10701}
10702
10703fn review_title(state: &RunState) -> Option<String> {
10708 english_subject(
10709 state.reviewed_commits.as_ref()?.first()?,
10710 state.config.graph.github_english(),
10711 )
10712}
10713
10714fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10717 let raw = raw.trim();
10718 if raw.is_empty() {
10719 return None;
10720 }
10721 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10722 return None;
10723 }
10724 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10725 return None;
10726 }
10727 let title = queue::title_from(raw, PR_TITLE_MAX);
10728 let lower = title.to_ascii_lowercase();
10729 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10730 return None;
10731 }
10732 Some(title)
10733}
10734
10735#[derive(Debug, Clone, PartialEq, Eq)]
10738struct BranchFacts {
10739 commits: Vec<(String, String)>,
10741 stat: String,
10743}
10744
10745const STAT_FILE_LINES: usize = 25;
10747const COMMITS_MAX_CHARS: usize = 20_000;
10749
10750async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10753 let commits = git::commit_log(repo, base, branch).await.ok()?;
10754 if commits.is_empty() {
10755 return None;
10756 }
10757 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10758 let lines: Vec<&str> = stat.lines().collect();
10759 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10760 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10761 let more = format!(" ... {omitted} more file(s)");
10762 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10763 kept.push(&more);
10764 kept.push(lines[lines.len() - 1]);
10765 kept.join("\n")
10766 } else {
10767 lines.join("\n")
10768 };
10769 Some(BranchFacts { commits, stat })
10770}
10771
10772fn markdown_safe(text: &str) -> String {
10775 text.replace("</details>", "</details>")
10776 .replace("\x60\x60\x60", "~~~")
10777}
10778
10779fn neutral_title(state: &RunState, winner: char) -> String {
10780 format!(
10781 "chore: land candidate {} of run {}",
10782 winner.to_ascii_uppercase(),
10783 state.id
10784 )
10785}
10786
10787pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
10793 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
10794 ""
10795 } else {
10796 pr_title
10797 }
10798}
10799
10800pub fn landing_subject_source(state: &RunState) -> String {
10804 if is_review_run(state) {
10805 let winner = state.candidates.first().map_or('A', |c| c.label);
10806 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
10807 }
10808 state.instruction.clone()
10809}
10810
10811fn summary_without_title(summary: &str) -> String {
10814 let mut lines = summary.trim().lines().peekable();
10815 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
10816 lines.next();
10817 }
10818 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
10819}
10820
10821#[cfg(test)]
10833fn pr_message(state: &RunState, winner: char) -> PrMessage {
10834 pr_message_with(state, winner, None)
10835}
10836
10837#[cfg(test)]
10840fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
10841 let raw = pr_message_raw(
10842 state,
10843 winner,
10844 facts,
10845 crate::pr_outcome::FinalHead::Unreported,
10846 ChangeNote::NotAsked,
10847 );
10848 let id = crate::scrub::Identity::current();
10849 PrMessage {
10850 title: crate::scrub::scrub(&raw.title, &id),
10851 body: crate::scrub::scrub(&raw.body, &id),
10852 }
10853}
10854
10855#[derive(Debug, Clone, Copy)]
10857enum ChangeNote<'a> {
10858 NotAsked,
10860 Summary(&'a crate::run::ChangeSummary),
10862 Unavailable(&'a str),
10864}
10865
10866fn pr_message_raw(
10867 state: &RunState,
10868 winner: char,
10869 facts: Option<&BranchFacts>,
10870 head: crate::pr_outcome::FinalHead<'_>,
10871 change: ChangeNote<'_>,
10872) -> PrMessage {
10873 if !matches!(change, ChangeNote::NotAsked) {
10874 let (title, mut body) = match change {
10875 ChangeNote::Summary(c) => (
10876 c.title.clone(),
10877 format!("## Summary\n\n{}\n\n", c.body.trim()),
10878 ),
10879 ChangeNote::Unavailable(why) => (
10880 neutral_title(state, winner),
10881 format!(
10882 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
10883 ),
10884 ),
10885 ChangeNote::NotAsked => unreachable!(),
10886 };
10887 body.push_str(&public_pr_evidence(state, head));
10888 body.push_str(&format!(
10889 "\n---\nmagi:run/{} magi:candidate-{}\n",
10890 state.id,
10891 winner.to_ascii_lowercase()
10892 ));
10893 return PrMessage { title, body };
10894 }
10895 pr_preview(state, winner, facts, head)
10896}
10897
10898fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
10901 let mut body = crate::pr_outcome::review_rounds(state);
10902 body.push_str(&crate::pr_outcome::final_verification(state, head));
10903 let open = state.last_round_findings();
10904 if !open.is_empty() {
10905 body.push_str("## Open review findings\n\n");
10906 for f in open {
10907 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
10908 }
10909 body.push('\n');
10910 }
10911 if let Some(c) = state.change_summary.as_ref()
10912 && head == crate::pr_outcome::FinalHead::At(&c.head)
10913 {
10914 body.push_str(&format!(
10915 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
10916 c.base, c.head
10917 ));
10918 }
10919 body
10920}
10921
10922fn pr_preview(
10924 state: &RunState,
10925 winner: char,
10926 facts: Option<&BranchFacts>,
10927 head: crate::pr_outcome::FinalHead<'_>,
10928) -> PrMessage {
10929 let summary = state
10930 .candidates
10931 .iter()
10932 .find(|c| c.label == winner)
10933 .map(|c| c.summary.as_str())
10934 .unwrap_or_default();
10935 let review = is_review_run(state);
10938 let title = if review {
10939 facts
10940 .and_then(|f| {
10941 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
10942 })
10943 .or_else(|| review_title(state))
10944 .or_else(|| {
10945 state
10946 .candidates
10947 .iter()
10948 .find(|c| c.label == winner)
10949 .filter(|c| !c.branch.starts_with("magi/"))
10950 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
10951 })
10952 .unwrap_or_else(|| neutral_title(state, winner))
10953 } else {
10954 summary_title(summary).unwrap_or_else(|| {
10955 english_title(
10956 &state.instruction,
10957 PR_TITLE_MAX,
10958 state.config.graph.github_english(),
10959 )
10960 .unwrap_or_else(|| neutral_title(state, winner))
10961 })
10962 };
10963
10964 let mut body = String::new();
10965 let what = summary_without_title(summary);
10966 if !what.is_empty() {
10967 body.push_str("## Summary\n\n");
10968 body.push_str(&what);
10969 body.push_str("\n\n");
10970 }
10971
10972 body.push_str(&crate::pr_outcome::review_rounds(state));
10974 body.push_str(&crate::pr_outcome::final_verification(state, head));
10975 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
10976
10977 let open = state.open_findings();
10978 if !open.is_empty() {
10979 body.push_str("## Open review findings\n\n");
10980 for f in &open {
10981 body.push_str(&format!(
10982 "- `{}` [{:?}] {}\n",
10983 f.id, f.severity, "Context retained in local review records."
10984 ));
10985 }
10986 body.push('\n');
10987 }
10988
10989 if let Some(fix) = fix
10990 && !fix.rejected.is_empty()
10991 {
10992 body.push_str("## Declined by the fixer\n\n");
10993 for r in &fix.rejected {
10994 body.push_str(&format!(
10995 "- `{}`: {}\n",
10996 r.id, "Reason retained in local review records."
10997 ));
10998 }
10999 body.push('\n');
11000 }
11001
11002 if review {
11003 body.push_str("## Commits under review\n\n");
11005 if let Some(facts) = facts {
11006 let mut left = COMMITS_MAX_CHARS;
11007 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11008 let mut entry = format!("- {}\n", markdown_safe(subject));
11009 for l in markdown_safe(text).lines() {
11010 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11011 }
11012 if left == 0 {
11013 body.push_str(&format!(
11014 "- ... {} more commit(s)\n",
11015 facts.commits.len() - i
11016 ));
11017 break;
11018 }
11019 if entry.len() > left {
11020 let mut end = left;
11023 while !entry.is_char_boundary(end) {
11024 end -= 1;
11025 }
11026 entry.truncate(end);
11027 entry.push_str("\n ... (truncated)\n");
11028 left = 0;
11029 } else {
11030 left -= entry.len();
11031 }
11032 body.push_str(&entry);
11033 }
11034 if !facts.stat.trim().is_empty() {
11035 body.push_str(&format!(
11036 "\n## Diff stat\n\n```\n{}\n```\n",
11037 markdown_safe(facts.stat.trim())
11038 ));
11039 }
11040 } else {
11041 match &state.reviewed_commits {
11042 Some(subjects) => {
11043 for s in subjects {
11044 body.push_str(&format!("- {}\n", s.trim()));
11045 }
11046 }
11047 None => {
11048 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11051 body.push_str(rest.trim());
11052 body.push('\n');
11053 }
11054 }
11055 }
11056 } else {
11057 let task = state.instruction.trim();
11058 let task = if task.is_empty() {
11059 "(empty task)"
11060 } else {
11061 task
11062 };
11063 body.push_str(&format!(
11064 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11065 task.replace("</details>", "</details>")
11066 ));
11067 }
11068
11069 body.push_str(&format!(
11070 "\n---\nmagi:run/{} magi:candidate-{}\n",
11071 state.id,
11072 winner.to_ascii_lowercase()
11073 ));
11074
11075 PrMessage { title, body }
11076}
11077
11078fn seeded_instruction(state: &RunState) -> String {
11082 match refs::describe(&state.seeds) {
11083 Some(facts) => format!(
11084 "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
11085 Candidates start from the unmerged branch named above, when there \
11086 is one, and carry any unmerged commit named by sha as a \
11087 cherry-pick. Check that this is what the task meant before \
11088 building on it.",
11089 state.instruction
11090 ),
11091 None => state.instruction.clone(),
11092 }
11093}
11094
11095async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11099 let base = &state.base_branch;
11100 let mut against = base.clone();
11101 if mode == MergeMode::Pr {
11102 let remote = &state.config.merge.remote;
11106 let tracking = format!("{remote}/{base}");
11107 let fetched = git::fetch(repo, remote, base).await;
11108 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11109 against = tracking;
11110 } else {
11111 return false;
11112 }
11113 }
11114 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11115}
11116
11117fn empty_candidate_detail(state: &RunState, base: &str) -> String {
11120 let mut detail = format!(
11121 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11122 nothing to open a pull request for"
11123 );
11124 match refs::describe(&state.seeds) {
11125 Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
11126 None => detail.push_str(
11127 "\nThe task names no existing branch or commit; if it means to land work \
11128 that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
11129 ),
11130 }
11131 detail
11132}
11133
11134#[derive(Debug, PartialEq, Eq)]
11137enum PrPlan {
11138 Create,
11139 Adopt { url: String, title: String },
11140 Stop(String),
11141}
11142
11143fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11146 match found {
11147 Ok(land::OpenPr::None) => PrPlan::Create,
11148 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11149 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11150 "several open pull requests exist for this branch, not picking one: {}",
11151 urls.join(" ")
11152 )),
11153 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11154 }
11155}
11156
11157pub(crate) async fn gh_pr_create(
11159 cwd: &Path,
11160 base: &str,
11161 head: &str,
11162 title: &str,
11163 body: &str,
11164) -> Result<String> {
11165 let out = tokio::process::Command::new("gh")
11166 .args([
11167 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11168 ])
11169 .current_dir(cwd)
11170 .quiet()
11171 .stdin(std::process::Stdio::null())
11172 .output()
11173 .await
11174 .context("spawn gh")?;
11175 if out.status.success() {
11176 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11177 } else {
11178 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11179 }
11180}
11181
11182async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11184 match std::fs::symlink_metadata(path) {
11185 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11186 git::worktree_remove_clean(repo, path).await?;
11189 return Ok(());
11190 }
11191 Err(e) => return Err(e.into()),
11192 Ok(_) => {}
11193 }
11194 if !git::is_clean(path).await? {
11195 bail!(
11196 "worktree has uncommitted work; refusing removal: {}",
11197 path.display()
11198 );
11199 }
11200 if !git::worktree_remove_clean(repo, path).await? {
11201 bail!("worktree removal refused: {}", path.display());
11202 }
11203 Ok(())
11204}
11205
11206pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11215 if crate::pr_recovery::sources_retained(home, &state.id) {
11216 return Ok(Vec::new());
11217 }
11218 if !state.candidate_preservation.is_empty() {
11219 bail!("candidate preservation incomplete; refusing cleanup");
11220 }
11221 let repo = state.repo.clone();
11222 let wait = repository_wait(state, None);
11225 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11226 Ok(lease) => lease,
11227 Err(e) => {
11228 state.save_under(home)?;
11229 return Err(e);
11230 }
11231 };
11232 let root = state.worktree_root();
11233 let winner = state.tally.as_ref().map(|t| t.winner);
11234 let mut removed = Vec::new();
11235
11236 for i in 0..state.candidates.len() {
11237 let c = state.candidates[i].clone();
11238 let is_winner = Some(c.label) == winner;
11239 if is_winner && !drop_winner {
11240 continue;
11241 }
11242 let existed = c.worktree.exists();
11243 remove_preserved_worktree(&repo, &c.worktree).await?;
11246 if existed {
11247 removed.push(c.worktree.to_string_lossy().into_owned());
11248 }
11249 let handed_over = state.released_branches.contains(&c.branch);
11252 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11253 if !git::branch_delete(&repo, &c.branch).await? {
11254 bail!("candidate branch removal refused");
11255 }
11256 removed.push(c.branch.clone());
11257 }
11258 state.candidates[i].folded = true;
11259 }
11260
11261 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11262 let path = name.path();
11263 let keep = !drop_winner
11264 && winner.is_some_and(|w| {
11265 path.file_name()
11266 .is_some_and(|n| n == format!("cand-{w}").as_str())
11267 });
11268 if keep {
11269 continue;
11270 }
11271 remove_preserved_worktree(&repo, &path).await?;
11272 removed.push(path.to_string_lossy().into_owned());
11273 }
11274
11275 remove_if_empty(&root);
11284
11285 if state.enabled_worktree_config && drop_winner {
11286 git::release_worktree_config(&repo).await.ok();
11290 state.enabled_worktree_config = false;
11291 }
11292 state.save_under(home)?;
11293 Ok(removed)
11294}
11295
11296fn remove_if_empty(dir: &Path) {
11307 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11308 std::fs::remove_dir(dir).ok();
11309 }
11310}
11311
11312pub fn worst_open(state: &RunState) -> Option<Severity> {
11314 state
11315 .reviews
11316 .last()?
11317 .reviews
11318 .iter()
11319 .flat_map(|r| r.findings.iter())
11320 .map(|f| f.severity)
11321 .max()
11322}
11323
11324async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11328 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11329 git::acquire_worktree_config(repo).await?;
11330 state.enabled_worktree_config = true;
11331 }
11332 Ok(())
11333}
11334
11335#[cfg(test)]
11336mod tests {
11337 #[tokio::test]
11338 async fn launcher_base_fetch_requires_remote_success() {
11339 let tmp = tempfile::tempdir().unwrap();
11340 let repo = tmp.path().join("repo");
11341 let remote = tmp.path().join("remote");
11342 std::fs::create_dir(&repo).unwrap();
11343 std::fs::create_dir(&remote).unwrap();
11344 git::git(&remote, &["init", "--bare", "-b", "main"])
11345 .await
11346 .unwrap();
11347 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11348 git::git(
11349 &repo,
11350 &[
11351 "-c",
11352 "user.name=Fixture",
11353 "-c",
11354 "user.email=fixture@example.invalid",
11355 "commit",
11356 "--allow-empty",
11357 "-m",
11358 "fixture",
11359 ],
11360 )
11361 .await
11362 .unwrap();
11363 git::git(
11364 &repo,
11365 &["remote", "add", "origin", remote.to_str().unwrap()],
11366 )
11367 .await
11368 .unwrap();
11369 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11370 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11371 assert_eq!(
11372 resolve_base(&repo, "main", "origin", None)
11373 .await
11374 .unwrap()
11375 .commit,
11376 head
11377 );
11378 #[cfg(unix)]
11379 {
11380 let policy: crate::config::Coordination = toml::from_str(
11381 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11382 ).unwrap();
11383 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11384 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11385 let held = crate::repo_lock::SeatLease::operation_with(
11386 &repo,
11387 Some(&policy),
11388 &crate::repo_lock::WaitPolicy::default(),
11389 )
11390 .await
11391 .unwrap();
11392 assert!(held.is_some());
11393 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11394 tokio::pin!(fetch);
11395 assert!(
11396 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11397 .await
11398 .is_err()
11399 );
11400 assert!(
11401 !repo.join(".git/FETCH_HEAD").exists(),
11402 "no fetch before admission"
11403 );
11404 drop(held);
11405 let fetched = fetch.await.unwrap();
11406 assert_eq!(fetched.commit, head);
11407 assert_eq!(fetched.remote, "origin");
11408 assert_eq!(fetched.branch, "main");
11409 }
11410 git::git(
11412 &repo,
11413 &[
11414 "remote",
11415 "set-url",
11416 "origin",
11417 tmp.path().join("missing").to_str().unwrap(),
11418 ],
11419 )
11420 .await
11421 .unwrap();
11422 let error = resolve_base(&repo, "main", "origin", None)
11423 .await
11424 .unwrap_err()
11425 .to_string();
11426 assert!(error.contains("cannot read origin/main"));
11427 assert!(error.contains("refusing to branch off"));
11428 }
11429
11430 #[test]
11431 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11432 let leaked = vec!["chore(deps): update a crate".to_owned()];
11433 let own = "fix(daemon): apply a chosen action";
11434 assert!(should_retitle("", own, &leaked));
11435 assert!(should_retitle(
11436 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11437 own,
11438 &leaked
11439 ));
11440 assert!(should_retitle(
11441 "chore: land candidate A of run 1",
11442 own,
11443 &leaked
11444 ));
11445 assert!(should_retitle(
11446 "magi: candidate A (uncommitted work)",
11447 own,
11448 &leaked
11449 ));
11450 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11451 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11452 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11453 }
11454
11455 #[test]
11456 fn pr_merge_plan_creates_adopts_or_stops() {
11457 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11458 assert_eq!(
11459 pr_merge_plan(Ok(land::OpenPr::One {
11460 url: "u".into(),
11461 title: "t".into()
11462 })),
11463 PrPlan::Adopt {
11464 url: "u".into(),
11465 title: "t".into()
11466 }
11467 );
11468 let PrPlan::Stop(many) =
11469 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11470 else {
11471 panic!("many must stop");
11472 };
11473 assert!(many.contains('a') && many.contains('b'));
11474 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11475 panic!("a failed lookup must stop");
11476 };
11477 assert!(err.contains("bad token"));
11478 }
11479
11480 use super::*;
11481 use crate::run::GateStatus;
11482 use std::collections::BTreeMap;
11483 use std::time::Duration;
11484
11485 fn conductor() -> AgentSpec {
11486 AgentSpec {
11487 id: "conductor".to_owned(),
11488 kind: crate::config::AgentKind::Command,
11489 model: None,
11490 command: vec!["true".to_owned()],
11491 extra_args: Vec::new(),
11492 env: BTreeMap::new(),
11493 prompt_delivery: None,
11494 }
11495 }
11496
11497 fn spec(id: &str) -> AgentSpec {
11498 AgentSpec {
11499 id: id.to_owned(),
11500 kind: crate::config::AgentKind::Command,
11501 model: None,
11502 command: vec!["true".to_owned()],
11503 extra_args: Vec::new(),
11504 env: BTreeMap::new(),
11505 prompt_delivery: None,
11506 }
11507 }
11508
11509 fn ids(xs: &[&str]) -> BTreeSet<String> {
11510 xs.iter().map(|s| (*s).to_owned()).collect()
11511 }
11512
11513 #[test]
11514 fn pick_successor_skips_an_agent_another_seat_holds() {
11515 let roster = [spec("a"), spec("b"), spec("c")];
11517 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11518 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11519 }
11520
11521 #[test]
11522 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11523 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11525 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11526 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11527 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11528 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11529 }
11530
11531 #[test]
11532 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11533 let roster = [spec("a"), spec("b")];
11534 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11535 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11536 let carried = ids(&["b"]);
11537 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11538 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11539 }
11540
11541 #[test]
11542 fn pick_successor_returns_none_without_any_untried_successor() {
11543 let roster = [spec("a"), spec("b")];
11544 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11545 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11546 let carried = ids(&["a"]);
11547 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11548 }
11549
11550 #[test]
11551 fn pick_successor_rescue_avoids_another_seats_occupant() {
11552 let roster = [spec("a"), spec("b"), spec("c")];
11554 let carried = ids(&["b", "c"]);
11555 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11556 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11557 }
11558
11559 #[test]
11560 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11561 let roster = [spec("a"), spec("b"), spec("c")];
11562 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11563 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11564 }
11565
11566 #[test]
11567 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11568 let roster = [spec("a"), spec("b"), spec("c")];
11569 let failed = ids(&["a", "b", "c"]);
11570 let mut tried = ids(&["b"]);
11572 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11573 assert_eq!(first.id, "a");
11574 tried.insert(first.id.clone());
11575 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11576 assert_eq!(second.id, "c");
11577 tried.insert(second.id.clone());
11578 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11579 }
11580
11581 #[test]
11582 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11583 let roster = [spec("a"), spec("b")];
11584 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11585 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11586 }
11587
11588 #[test]
11589 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11590 let roster = [spec("a"), spec("b"), spec("c")];
11591 let h = SeatHistory {
11592 failed: ids(&["a"]),
11593 last_ok: Some("b".to_owned()),
11594 last_fail: None,
11595 };
11596 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
11597 let gone = SeatHistory {
11599 last_ok: Some("zzz".to_owned()),
11600 ..h.clone()
11601 };
11602 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
11603 let failed = SeatHistory {
11604 failed: ids(&["a", "b"]),
11605 last_ok: Some("b".to_owned()),
11606 last_fail: None,
11607 };
11608 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
11609 }
11610
11611 #[test]
11612 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
11613 let first = SeatState::new("review-1", "alpha", 7);
11614 let next = handover_seat("review-1", "gamma", 7);
11615 assert_ne!(first.claude_session, next.claude_session);
11616 }
11617
11618 #[test]
11619 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
11620 let mut state = state_with_summary("x", "y");
11621 let a = handover_seat("review-1", "beta", state.next_seat_seed());
11622 let b = handover_seat("review-1", "beta", state.next_seat_seed());
11623 assert_ne!(a.claude_session, b.claude_session);
11624 }
11625
11626 #[test]
11627 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
11628 let roster = [spec("a"), spec("b")];
11629 let h = SeatHistory {
11630 failed: ids(&["a", "b"]),
11631 ..SeatHistory::default()
11632 };
11633 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
11634 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
11635 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
11636 }
11637
11638 #[test]
11645 fn failure_signature_ignores_numbers_and_paths() {
11646 assert_eq!(
11647 failure_signature("exited with Some(2) and no usable output"),
11648 failure_signature("exited with Some(137) and no usable output")
11649 );
11650 assert_eq!(
11651 failure_signature("process exited with no usable output (exit code 2)"),
11652 failure_signature("process exited with no usable output (exit code 137)")
11653 );
11654 for (old, new) in [
11655 (
11656 "exited with Some(2) and no usable output",
11657 "process exited with no usable output (exit code 2)",
11658 ),
11659 (
11660 "exited with None and no usable output",
11661 "process exited with no usable output (exit code unavailable)",
11662 ),
11663 ] {
11664 let previous = FailClass::Other(failure_signature(old));
11665 let current = FailClass::Other(failure_signature(new));
11666 assert_eq!(previous, current, "resumption retains the failure class");
11667 assert!(!should_hand_over(Some(&previous), ¤t));
11668 }
11669 assert_eq!(
11670 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
11671 failure_signature("cannot open /var/x.txt: denied")
11672 );
11673 assert_ne!(failure_signature("boom"), failure_signature("bang"));
11674 }
11675
11676 #[test]
11677 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
11678 let other = FailClass::Other("x".into());
11679 assert!(should_hand_over(None, &FailClass::Quota));
11680 assert!(should_hand_over(Some(&other), &FailClass::Quota));
11681 assert!(should_hand_over(
11682 Some(&FailClass::Timeout),
11683 &FailClass::Timeout
11684 ));
11685 assert!(should_hand_over(None, &other));
11686 assert!(!should_hand_over(Some(&other), &other));
11687 assert!(should_hand_over(
11688 Some(&other),
11689 &FailClass::Other("y".into())
11690 ));
11691 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
11693 assert!(should_hand_over(Some(&FailClass::Quota), &other));
11694 }
11695
11696 #[test]
11697 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
11698 let a = SeatState::new("judge-1", "alpha", 7);
11699 let b = handover_seat("judge-1", "beta", 7);
11700 assert_ne!(a.claude_session, b.claude_session);
11701 assert_eq!(b.turns, 0);
11702 }
11703
11704 #[test]
11705 fn a_timeout_is_classified_apart_from_other_failures() {
11706 assert_eq!(
11707 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
11708 Some(FailClass::Timeout)
11709 );
11710 assert!(matches!(
11711 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
11712 Some(FailClass::Other(_))
11713 ));
11714 }
11715
11716 #[test]
11717 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
11718 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11719 let tried = BTreeSet::from(["beta".to_owned()]);
11720 let next = next_untried_in_roster(&roster, 1, &tried);
11723 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
11724 }
11725
11726 #[test]
11727 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
11728 let roster = vec![spec("alpha"), spec("beta")];
11729 let tried = BTreeSet::from(["beta".to_owned()]);
11730 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11734 }
11735
11736 #[test]
11737 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
11738 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
11739 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
11740 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
11744 }
11745
11746 #[test]
11747 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
11748 let roster = vec![spec("a"), spec("a"), spec("b")];
11749 let tried = BTreeSet::from(["a".to_owned()]);
11750 let next = next_untried_in_roster(&roster, 0, &tried);
11751 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11752 }
11753
11754 #[test]
11755 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
11756 let roster = vec![spec("a"), spec("b")];
11757 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
11758 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
11759 }
11760
11761 #[test]
11762 fn remove_if_empty_only_ever_takes_a_bare_directory() {
11763 let dir = tempfile::tempdir().unwrap();
11764 let bay = dir.path().join("ffff");
11765
11766 remove_if_empty(&bay);
11768 assert!(!bay.exists());
11769
11770 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
11773 remove_if_empty(&bay);
11774 assert!(bay.exists(), "non-empty directory must survive");
11775
11776 std::fs::remove_dir(bay.join("cand-A")).unwrap();
11778 remove_if_empty(&bay);
11779 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
11780 }
11781
11782 #[test]
11791 fn a_full_panel_that_found_nothing_is_clean() {
11792 assert!(round_is_clean(
11793 0,
11794 true,
11795 2,
11796 2,
11797 0,
11798 IncompleteReviewPolicy::Block
11799 ));
11800 }
11801
11802 #[test]
11803 fn a_missing_seat_is_never_clean_under_the_default_policy() {
11804 assert!(!round_is_clean(
11805 0,
11806 true,
11807 1,
11808 2,
11809 0,
11810 IncompleteReviewPolicy::Block
11811 ));
11812 }
11813
11814 #[test]
11815 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
11816 assert!(!round_is_clean(
11817 1,
11818 true,
11819 1,
11820 2,
11821 0,
11822 IncompleteReviewPolicy::Warn
11823 ));
11824 }
11825
11826 #[test]
11827 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
11828 assert!(round_is_clean(
11829 0,
11830 true,
11831 1,
11832 2,
11833 0,
11834 IncompleteReviewPolicy::Warn
11835 ));
11836 }
11837
11838 #[test]
11839 fn a_full_panel_with_an_open_finding_is_not_clean() {
11840 assert!(!round_is_clean(
11841 1,
11842 true,
11843 2,
11844 2,
11845 0,
11846 IncompleteReviewPolicy::Block
11847 ));
11848 }
11849
11850 #[test]
11851 fn a_full_panel_with_a_red_e2e_is_not_clean() {
11852 assert!(!round_is_clean(
11853 0,
11854 false,
11855 2,
11856 2,
11857 0,
11858 IncompleteReviewPolicy::Block
11859 ));
11860 }
11861
11862 #[test]
11869 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
11870 assert!(round_is_clean(
11873 0,
11874 true,
11875 1,
11876 2,
11877 1,
11878 IncompleteReviewPolicy::Block
11879 ));
11880 }
11881
11882 #[test]
11883 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
11884 assert!(!round_is_clean(
11887 0,
11888 true,
11889 1,
11890 2,
11891 0,
11892 IncompleteReviewPolicy::Block
11893 ));
11894 }
11895
11896 #[test]
11897 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
11898 assert!(!round_is_clean(
11899 1,
11900 true,
11901 1,
11902 2,
11903 1,
11904 IncompleteReviewPolicy::Block
11905 ));
11906 assert!(!round_is_clean(
11907 0,
11908 false,
11909 1,
11910 2,
11911 1,
11912 IncompleteReviewPolicy::Block
11913 ));
11914 }
11915
11916 #[test]
11917 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
11918 assert!(!round_is_clean(
11922 0,
11923 true,
11924 0,
11925 2,
11926 2,
11927 IncompleteReviewPolicy::Block
11928 ));
11929 }
11930
11931 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
11932 CommandOutcome {
11933 command: "test".to_owned(),
11934 code,
11935 output_tail: String::new(),
11936 duration_ms: 0,
11937 resource_blocked,
11938 }
11939 }
11940
11941 #[test]
11942 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
11943 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
11944 assert!(
11945 !verify_inconclusive(&[outcome(Some(1), false)]),
11946 "an ordinary failure is still evidence about the patch"
11947 );
11948 assert!(verify_inconclusive(&[outcome(None, true)]));
11949 assert!(
11950 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
11951 "one inconclusive outcome taints the whole batch"
11952 );
11953 assert!(!verify_inconclusive(&[]));
11954 }
11955
11956 #[tokio::test]
11957 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
11958 let calls = std::sync::atomic::AtomicUsize::new(0);
11962 let started = Instant::now();
11963 wait_for_pids_with(
11964 &[123],
11965 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
11966 Duration::from_millis(5),
11967 Duration::from_secs(5),
11968 )
11969 .await;
11970 assert!(
11971 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
11972 "must keep checking rather than deciding on the first answer"
11973 );
11974 assert!(
11975 started.elapsed() < Duration::from_secs(1),
11976 "must return the moment it is confirmed dead, not wait out the ceiling"
11977 );
11978 }
11979
11980 #[tokio::test]
11981 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
11982 let started = Instant::now();
11983 wait_for_pids_with(
11984 &[123],
11985 |_| true, Duration::from_millis(5),
11987 Duration::from_millis(30),
11988 )
11989 .await;
11990 let elapsed = started.elapsed();
11991 assert!(
11992 elapsed >= Duration::from_millis(30),
11993 "must not give up before its own ceiling: {elapsed:?}"
11994 );
11995 assert!(
11996 elapsed < Duration::from_secs(1),
11997 "must not wait past its own ceiling either: {elapsed:?}"
11998 );
11999 }
12000
12001 #[tokio::test]
12002 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12003 let started = Instant::now();
12004 wait_for_pids_with(
12005 &[],
12006 |_| true,
12007 Duration::from_secs(5),
12008 Duration::from_secs(5),
12009 )
12010 .await;
12011 assert!(
12012 started.elapsed() < Duration::from_millis(200),
12013 "an empty pid list has nothing to confirm"
12014 );
12015 }
12016
12017 fn review_round(
12023 clean: bool,
12024 blocking: usize,
12025 answered: usize,
12026 expected: usize,
12027 progressed: bool,
12028 e2e_ok: bool,
12029 ) -> ReviewRound {
12030 ReviewRound {
12031 round: 1,
12032 head: "h".to_owned(),
12033 verified_head: None,
12034 verified_at: None,
12035 reviews: Vec::new(),
12036 e2e: vec![CommandOutcome {
12037 command: "test".to_owned(),
12038 code: Some(if e2e_ok { 0 } else { 1 }),
12039 output_tail: String::new(),
12040 duration_ms: 0,
12041 resource_blocked: false,
12042 }],
12043 verify_retried: false,
12044 e2e_deferred: false,
12045 e2e_defer_reason: None,
12046 fix: None,
12047 blocking,
12048 answered,
12049 expected,
12050 clean,
12051 progressed,
12052 vote_split: false,
12053 reconsideration: Vec::new(),
12054 verdict: None,
12055 }
12056 }
12057
12058 #[test]
12059 fn review_conclusion_is_none_when_nothing_has_run() {
12060 assert_eq!(review_conclusion(&[], 3), None);
12061 }
12062
12063 #[test]
12064 fn review_conclusion_is_none_while_rounds_remain() {
12065 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12066 assert_eq!(review_conclusion(&rounds, 3), None);
12067 }
12068
12069 #[test]
12070 fn review_conclusion_is_gating_once_a_round_is_clean() {
12071 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12072 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12073 }
12074
12075 #[test]
12076 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12077 let rounds = vec![
12078 review_round(false, 1, 2, 2, true, true),
12079 review_round(false, 1, 2, 2, true, true),
12080 ];
12081 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12082 }
12083
12084 #[test]
12085 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12086 let rounds = vec![
12087 review_round(false, 1, 2, 2, true, true),
12088 review_round(false, 1, 2, 2, true, false),
12089 ];
12090 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12091 }
12092
12093 #[test]
12094 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12095 let mut blocked = review_round(false, 1, 2, 2, true, false);
12102 blocked.e2e[0].resource_blocked = true;
12103 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12104 assert_eq!(review_conclusion(&rounds, 2), None);
12105 }
12106
12107 #[test]
12108 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12109 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12111 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12112 }
12113
12114 #[test]
12115 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12116 let rounds = vec![
12117 review_round(false, 1, 2, 2, false, true),
12118 review_round(false, 1, 2, 2, false, true),
12119 ];
12120 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12121 }
12122
12123 fn secs(n: u64) -> Duration {
12124 Duration::from_secs(n)
12125 }
12126
12127 #[tokio::test]
12130 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12131 ask_test_home();
12132 let tmp = tempfile::tempdir().unwrap();
12133 let repo = tmp.path().join("repo");
12134 std::fs::create_dir_all(&repo).unwrap();
12135 init_repo(&repo);
12136 let g = |args: &[&str]| {
12137 let out = std::process::Command::new("git")
12138 .args(args)
12139 .current_dir(&repo)
12140 .quiet()
12141 .output()
12142 .unwrap();
12143 assert!(
12144 out.status.success(),
12145 "{}",
12146 String::from_utf8_lossy(&out.stderr)
12147 );
12148 };
12149 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12150 g(&["checkout", "-b", "historical"]);
12151 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12152 g(&["add", "."]);
12153 g(&["commit", "-m", "old alternative"]);
12154 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12155 g(&["checkout", "-b", "magi/task-selected", &base]);
12156 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12157 g(&["add", "."]);
12158 g(&["commit", "-m", "measurement infrastructure"]);
12159 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12160 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12161 g(&["add", "."]);
12162 g(&["commit", "-m", "dependent implementation"]);
12163 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12164 g(&["checkout", "main"]);
12165 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12166 g(&["add", "."]);
12167 g(&["commit", "-m", "unrelated base change"]);
12168 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12169 let origin = tmp.path().join("origin.git");
12170 g(&[
12171 "clone",
12172 "--bare",
12173 repo.to_str().unwrap(),
12174 origin.to_str().unwrap(),
12175 ]);
12176 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12177 g(&["fetch", "origin"]);
12178 let selection = crate::pr_recovery::Selection {
12179 url: "https://example.test/project/pull/1".into(),
12180 branch: "magi/task-selected".into(),
12181 head: head.clone(),
12182 instruction: "preserve measured behavior".into(),
12183 approved_at: jiff::Timestamp::now(),
12184 };
12185 let mut template = runner_at(RunStatus::Reviewing);
12186 template.state.repo = repo.clone();
12187 template.state.base_commit = current.clone();
12188 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12189 template.state.origin = Some(crate::run::Origin::queue("selected"));
12190 template.state.pr_head = Some(selection.branch.clone());
12191 template.state.pr_recovery = Some(selection.clone());
12192 let mut runner =
12193 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12194 .await
12195 .unwrap();
12196 let wt = runner.state.winner().unwrap().worktree.clone();
12197 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12198 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12199 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12200 assert!(runner.state.seeds.is_empty());
12201 assert_eq!(runner.state.instruction, "preserve measured behavior");
12202 runner.sync_to_base().await.unwrap();
12203 assert_eq!(
12204 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12205 "prerequisite"
12206 );
12207 assert_eq!(
12208 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12209 "measured implementation"
12210 );
12211 assert_eq!(
12212 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12213 "current main change"
12214 );
12215 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12216 assert_eq!(
12217 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12218 head
12219 );
12220
12221 let mut partial = runner_at(RunStatus::Reviewing);
12223 partial.state.repo = repo.clone();
12224 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12225 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12226 run: Some("exit 1".into()),
12227 ..Default::default()
12228 }];
12229 let selection = runner.state.pr_recovery.clone().unwrap();
12230 partial.state.pr_recovery = Some(selection.clone());
12231 partial.state.pr_head = Some(selection.branch.clone());
12232 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12233 let id = partial.state.id.clone();
12234 let error = match Runner::open_selected(
12235 &repo,
12236 partial.state,
12237 partial.roles,
12238 1,
12239 runner.state.base_commit.clone(),
12240 selection,
12241 )
12242 .await
12243 {
12244 Ok(_) => panic!("setup must fail"),
12245 Err(e) => e,
12246 };
12247 assert_eq!(
12248 error
12249 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12250 .unwrap()
12251 .run,
12252 id
12253 );
12254 let failed = RunState::load(&id).unwrap();
12255 assert_eq!(failed.status, RunStatus::Blocked);
12256 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12257 assert!(Runner::resume(&id).is_err());
12258 runner.state.status = RunStatus::Ready;
12259 runner.state.driver_exited = true;
12260 runner.state.save().unwrap();
12261 let mut retry = runner_at(RunStatus::Reviewing);
12262 retry.state.repo = repo.clone();
12263 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12264 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12265 reselected.approved_at = jiff::Timestamp::now();
12266 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12267 retry.state.pr_head = Some(reselected.branch.clone());
12268 retry.state.pr_recovery = Some(reselected.clone());
12269 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12270 .await
12271 .unwrap();
12272 let restarted = Runner::open_selected(
12273 &repo,
12274 retry.state,
12275 retry.roles,
12276 1,
12277 runner.state.base_commit.clone(),
12278 reselected,
12279 )
12280 .await
12281 .unwrap();
12282 assert_eq!(
12283 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12284 .await
12285 .unwrap(),
12286 head
12287 );
12288 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12289 assert_eq!(
12290 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12291 head
12292 );
12293 }
12294
12295 #[tokio::test]
12296 #[cfg(unix)]
12297 async fn fallback_and_interrupted_restoration_require_coordination() {
12298 use std::os::fd::AsRawFd as _;
12299 ask_test_home();
12300 let tmp = tempfile::tempdir().unwrap();
12301 let mut runner = registered_runner(tmp.path()).await;
12302 runner.state.config.graph.timeout_resource_wait = 1;
12303 let winner = runner.state.candidates[0].clone();
12304 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12305 let leftover = winner.worktree.join("leftover");
12306 std::fs::write(&leftover, "untrusted edits").unwrap();
12307 let next = tmp.path().join("next-fixer");
12308 let mut failed = spec("first");
12309 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12310 let mut successor = spec("second");
12311 successor.command = vec![
12312 "sh".into(),
12313 "-c".into(),
12314 format!("touch '{}'; echo done", next.display()),
12315 ];
12316 let attempts = vec![
12317 (failed.clone(), "first".into()),
12318 (successor.clone(), "second".into()),
12319 ];
12320 let lock_path = runner.state.repo.join(".git/updater.lock");
12321 let external = std::fs::File::create(&lock_path).unwrap();
12322 assert_eq!(
12323 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12324 0
12325 );
12326 let artifacts = tmp.path().join("artifacts");
12327 let build = |spec, seat| SeatJob {
12328 spec,
12329 seat,
12330 cwd: winner.worktree.clone(),
12331 prompt: "fixture".into(),
12332 timeout: Duration::from_secs(10),
12333 allow_write: true,
12334 sessions: false,
12335 artifacts: artifacts.clone(),
12336 stem: "fix".into(),
12337 handover: None,
12338 };
12339 let (_, _, out) = runner
12340 .ask_fixer_among(
12341 attempts.clone(),
12342 "graft",
12343 None,
12344 Some((&winner.worktree, &head)),
12345 build,
12346 )
12347 .await;
12348 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12349 assert!(leftover.exists(), "contention must prevent clean");
12350 assert!(!next.exists());
12351 assert!(runner.state.handovers.is_empty());
12352 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12354 from_run: "earlier".into(),
12355 generation: 1,
12356 mode: crate::recovery::Mode::VerifyRepair,
12357 reasons: Vec::new(),
12358 reviewed_head: None,
12359 rounds: Vec::new(),
12360 open_findings: Vec::new(),
12361 evidence: None,
12362 criteria_inherited: true,
12363 repair: crate::recovery::Repair::Started {
12364 before: head.clone(),
12365 },
12366 });
12367 assert!(
12368 runner
12369 .recovery_repair(&winner)
12370 .await
12371 .unwrap_err()
12372 .is::<crate::repo_lock::ResourceBlocked>()
12373 );
12374 assert!(matches!(
12375 runner.state.recovery.as_ref().unwrap().repair,
12376 crate::recovery::Repair::Started { .. }
12377 ));
12378 assert!(leftover.exists());
12379 drop(external);
12380 runner.recovery_repair(&winner).await.unwrap();
12381 assert!(!leftover.exists());
12382 assert!(matches!(
12383 runner.state.recovery.as_ref().unwrap().repair,
12384 crate::recovery::Repair::Done { .. }
12385 ));
12386 std::fs::remove_file(&lock_path).unwrap();
12388 std::fs::create_dir(&lock_path).unwrap();
12389 std::fs::write(&leftover, "untrusted edits").unwrap();
12390 let (_, _, out) = runner
12391 .ask_fixer_among(
12392 attempts,
12393 "graft",
12394 None,
12395 Some((&winner.worktree, &head)),
12396 build,
12397 )
12398 .await;
12399 assert!(matches!(out, AgentOutcome::Failed(_)));
12400 assert!(leftover.exists());
12401 assert!(!next.exists());
12402 assert_eq!(
12403 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12404 head
12405 );
12406 }
12407
12408 #[cfg(unix)]
12409 async fn registered_runner(root: &Path) -> Runner {
12410 ask_test_home();
12411 let repo = root.join("repo");
12412 std::fs::create_dir_all(&repo).unwrap();
12413 init_repo(&repo);
12414 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12415 let mut runner = runner_at(RunStatus::Prep);
12416 runner.state.repo = repo.clone();
12417 runner.state.config.coordination = crate::config::test_coordination();
12418 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12419 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12420 runner.state.config.graph.advise = false;
12421 runner.roles.implementers = vec![spec("implement")];
12422 runner.roles.judges = vec![spec("judge")];
12423 runner.prep().await.unwrap();
12424 runner
12425 }
12426
12427 #[tokio::test]
12428 #[cfg(unix)]
12429 async fn parked_implementer_persists_completed_work() {
12430 for registered in [false, true] {
12431 let tmp = tempfile::tempdir().unwrap();
12432 let mut runner = registered_runner(tmp.path()).await;
12433 if !registered {
12434 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12435 }
12436 let marker = tmp.path().join("started");
12437 let release = tmp.path().join("release");
12438 let mut implementer = spec("implement");
12439 implementer.command = vec![
12440 "sh".into(),
12441 "-c".into(),
12442 format!(
12443 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12444 marker.display(),
12445 release.display()
12446 ),
12447 ];
12448 runner.roles.implementers = vec![implementer.clone()];
12449 runner.state.config.agents = vec![implementer];
12450 let pause = runner.pause.clone();
12451 let (result, ()) = tokio::join!(runner.implement(), async {
12452 tokio::time::timeout(Duration::from_secs(10), async {
12453 while !marker.exists() {
12454 tokio::time::sleep(Duration::from_millis(10)).await;
12455 }
12456 })
12457 .await
12458 .unwrap();
12459 pause.park();
12460 std::fs::write(&release, "go").unwrap();
12461 });
12462 result.unwrap();
12463 let candidate = &runner.state.candidates[0];
12464 assert!(candidate.commits > 0, "{candidate:?}");
12465 assert!(candidate.summary.contains("saved-result"));
12466 assert_eq!(candidate.files, 1);
12467 let before = candidate.commits;
12468 runner.implement().await.unwrap();
12469 assert_eq!(runner.state.candidates[0].commits, before);
12470 }
12471 }
12472
12473 #[tokio::test]
12474 #[cfg(unix)]
12475 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12476 let tmp = tempfile::tempdir().unwrap();
12477 let mut runner = registered_runner(tmp.path()).await;
12478 let winner = runner.state.candidates[0].clone();
12479 let cache = tmp.path().join("cache");
12480 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12481 runner.state.config.graph.timeout_resource_wait = 0;
12482 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12483 let crate::cache::AcquireOutcome::Acquired(held) =
12484 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12485 else {
12486 panic!("cache must be free")
12487 };
12488 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12489 from_run: "earlier".into(),
12490 generation: 1,
12491 mode: crate::recovery::Mode::VerifyRepair,
12492 reasons: Vec::new(),
12493 reviewed_head: None,
12494 rounds: Vec::new(),
12495 open_findings: Vec::new(),
12496 evidence: None,
12497 criteria_inherited: true,
12498 repair: crate::recovery::Repair::NotStarted,
12499 });
12500 runner.recovery_repair(&winner).await.unwrap();
12501 assert!(matches!(
12502 runner.state.recovery.as_ref().unwrap().repair,
12503 crate::recovery::Repair::WaitingForRestore { .. }
12504 ));
12505 drop(held);
12506 runner.state.resource_block = None;
12507 runner.state.config.graph.timeout_resource_wait = 1;
12508 runner.recovery_repair(&winner).await.unwrap();
12509 assert!(matches!(
12510 runner.state.recovery.as_ref().unwrap().repair,
12511 crate::recovery::Repair::Done { .. }
12512 ));
12513 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12514 }
12515
12516 #[tokio::test]
12517 #[cfg(unix)]
12518 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12519 use std::os::fd::AsRawFd as _;
12520 for graft in [false, true] {
12521 let tmp = tempfile::tempdir().unwrap();
12522 let mut runner = registered_runner(tmp.path()).await;
12523 runner.state.config.graph.timeout_resource_wait = 1;
12524 let winner = runner.state.candidates[0].clone();
12525 let marker = tmp.path().join("fixer-started");
12526 let mut fixer = spec("fixer");
12527 fixer.command = vec![
12528 "sh".into(),
12529 "-c".into(),
12530 format!("echo started >> '{}'; echo done", marker.display()),
12531 ];
12532 runner.roles.fixer = Some(vec![fixer]);
12533 if graft {
12534 let mut loser = winner.clone();
12535 loser.label = 'B';
12536 loser.index = 1;
12537 runner.state.candidates.push(loser);
12538 runner.state.config.graph.graft_runner_up = true;
12539 runner.state.tally = Some(Tally {
12540 first_choice: BTreeMap::new(),
12541 borda: BTreeMap::new(),
12542 winner: winner.label,
12543 rankings: 1,
12544 unanimous_initial: true,
12545 deliberated: false,
12546 changed_votes: 0,
12547 unanimous_final: true,
12548 tie_break: None,
12549 judges: 1,
12550 present: 1,
12551 quorum: 1,
12552 met_quorum: true,
12553 uncontested: None,
12554 });
12555 runner.state.judgements.push(crate::run::Judgement {
12556 judge: 1,
12557 seat: "judge-1".into(),
12558 agent: "judge".into(),
12559 ranking: vec![winner.label, 'B'],
12560 reasons: BTreeMap::new(),
12561 confidence: None,
12562 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12563 order: vec![0, 1],
12564 failed: None,
12565 duration_ms: 0,
12566 });
12567 } else {
12568 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12569 from_run: "earlier".into(),
12570 generation: 1,
12571 mode: crate::recovery::Mode::VerifyRepair,
12572 reasons: Vec::new(),
12573 reviewed_head: None,
12574 rounds: Vec::new(),
12575 open_findings: Vec::new(),
12576 evidence: None,
12577 criteria_inherited: true,
12578 repair: crate::recovery::Repair::NotStarted,
12579 });
12580 }
12581 let reader = crate::repo_lock::SeatLease::for_checkout(
12582 &winner.worktree,
12583 false,
12584 crate::config::test_coordination().as_ref(),
12585 )
12586 .await
12587 .unwrap();
12588 let pause = runner.pause.clone();
12589 let (result, ()) = tokio::join!(
12590 async {
12591 if graft {
12592 runner.graft().await
12593 } else {
12594 runner.recovery_repair(&winner).await
12595 }
12596 },
12597 async {
12598 tokio::time::sleep(Duration::from_millis(100)).await;
12599 pause.park();
12600 }
12601 );
12602 result.unwrap();
12603 assert!(!marker.exists(), "a blocked seat must not spawn");
12604 runner.state = RunState::load(&runner.state.id).unwrap();
12606 if graft {
12607 assert_eq!(
12608 runner.state.graft.as_ref().unwrap().outcome,
12609 GraftOutcome::WaitingForRestore
12610 );
12611 } else {
12612 assert!(matches!(
12613 runner.state.recovery.as_ref().unwrap().repair,
12614 crate::recovery::Repair::WaitingForRestore { .. }
12615 ));
12616 }
12617 drop(reader);
12618 runner.pause = Pause::new();
12619 runner.state.resource_block = None;
12620 let external =
12622 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12623 assert_eq!(
12624 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12625 0
12626 );
12627 let leftover = winner.worktree.join("leftover");
12628 std::fs::write(&leftover, "earlier fallback edits").unwrap();
12629 let result = if graft {
12630 runner.graft().await
12631 } else {
12632 runner.recovery_repair(&winner).await
12633 };
12634 assert!(
12635 result
12636 .unwrap_err()
12637 .is::<crate::repo_lock::ResourceBlocked>()
12638 );
12639 assert!(leftover.exists());
12640 assert!(!marker.exists());
12641 drop(external);
12642 runner.state.resource_block = None;
12643 let result = if graft {
12644 runner.graft().await
12645 } else {
12646 runner.recovery_repair(&winner).await
12647 };
12648 result.unwrap();
12649 assert!(!leftover.exists(), "restore must precede retry");
12650 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
12651 if graft {
12652 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12653 } else {
12654 assert!(matches!(
12655 runner.state.recovery.as_ref().unwrap().repair,
12656 crate::recovery::Repair::Done { .. }
12657 ));
12658 }
12659 }
12660 }
12661
12662 #[tokio::test]
12663 #[cfg(unix)]
12664 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
12665 use std::os::fd::AsRawFd as _;
12666 let tmp = tempfile::tempdir().unwrap();
12667 let mut runner = registered_runner(tmp.path()).await;
12668 runner.state.config.graph.timeout_resource_wait = 1;
12669 let winner = runner.state.candidates[0].clone();
12670 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12671 runner.state.tally = Some(Tally {
12672 first_choice: BTreeMap::new(),
12673 borda: BTreeMap::new(),
12674 winner: winner.label,
12675 rankings: 0,
12676 unanimous_initial: false,
12677 deliberated: false,
12678 changed_votes: 0,
12679 unanimous_final: false,
12680 tie_break: None,
12681 judges: 0,
12682 present: 0,
12683 quorum: 0,
12684 met_quorum: true,
12685 uncontested: Some("fixture".into()),
12686 });
12687 runner.state.graft = Some(GraftRecord {
12688 pre: pre.clone(),
12689 post: None,
12690 lines: None,
12691 limit: None,
12692 agent: None,
12693 outcome: GraftOutcome::Running,
12694 requested: Vec::new(),
12695 adoption: None,
12696 adoption_error: None,
12697 });
12698 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
12699 git::commit_all(&winner.worktree, "unfinished graft")
12700 .await
12701 .unwrap();
12702 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12703 let leftover = winner.worktree.join("leftover");
12704 std::fs::write(&leftover, "untrusted edits").unwrap();
12705 let path = runner.state.repo.join(".git/updater.lock");
12706 let external = std::fs::File::create(&path).unwrap();
12707 assert_eq!(
12708 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12709 0
12710 );
12711 assert!(
12712 runner
12713 .graft()
12714 .await
12715 .unwrap_err()
12716 .is::<crate::repo_lock::ResourceBlocked>()
12717 );
12718 assert!(matches!(
12719 runner.state.graft.as_ref().unwrap().outcome,
12720 GraftOutcome::Running
12721 ));
12722 assert_eq!(
12723 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12724 unfinished
12725 );
12726 assert!(leftover.exists());
12727 drop(external);
12728 std::fs::remove_file(&path).unwrap();
12729 std::fs::create_dir(&path).unwrap();
12730 assert!(runner.graft().await.is_err());
12731 assert!(matches!(
12732 runner.state.graft.as_ref().unwrap().outcome,
12733 GraftOutcome::Running
12734 ));
12735 assert_eq!(
12736 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12737 unfinished
12738 );
12739 assert!(leftover.exists());
12740 std::fs::remove_dir(&path).unwrap();
12741 runner.graft().await.unwrap();
12742 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
12743 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
12744 assert!(!leftover.exists());
12745 }
12746
12747 #[tokio::test]
12748 #[cfg(unix)]
12749 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
12750 use std::os::fd::AsRawFd as _;
12751 let tmp = tempfile::tempdir().unwrap();
12752 let mut runner = registered_runner(tmp.path()).await;
12753 runner.state.prep_complete = Some(false);
12754 runner.state.config.graph.timeout_resource_wait = 1;
12755 runner.state.save().unwrap();
12756 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12757 assert_eq!(
12758 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12759 0
12760 );
12761 runner.execute().await.unwrap();
12762 let disk = RunState::load(&runner.state.id).unwrap();
12763 assert!(disk.driver_exited);
12764 assert_eq!(disk.driver_pid, None);
12765 assert_eq!(runner.state.status, RunStatus::Blocked);
12766 assert_eq!(
12767 runner.state.resource_block.as_ref().unwrap().kind,
12768 "repository coordination"
12769 );
12770 assert!(runner.state.prep_failure.is_none());
12771 assert!(runner.state.seats.is_empty());
12772 runner.pause.park();
12773 tokio::time::timeout(Duration::from_secs(10), runner.execute())
12776 .await
12777 .unwrap()
12778 .unwrap();
12779 assert!(runner.state.parked);
12780 assert!(runner.state.prep_failure.is_none());
12781 let disk = RunState::load(&runner.state.id).unwrap();
12782 assert!(disk.driver_exited);
12783 assert_eq!(disk.driver_pid, None);
12784 }
12785
12786 #[tokio::test]
12787 #[cfg(unix)]
12788 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
12789 use std::os::fd::AsRawFd as _;
12790 let tmp = tempfile::tempdir().unwrap();
12791 let mut runner = registered_runner(tmp.path()).await;
12792 runner.state.prep_complete = Some(false);
12793 runner.state.config.graph.timeout_resource_wait = 30;
12794 runner.state.save().unwrap();
12795 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
12796 assert_eq!(
12797 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12798 0
12799 );
12800 assert!(
12801 tokio::time::timeout(Duration::from_millis(100), runner.execute())
12802 .await
12803 .is_err()
12804 );
12805 let disk = RunState::load(&runner.state.id).unwrap();
12806 assert!(disk.driver_exited);
12807 assert_eq!(disk.driver_pid, None);
12808 assert!(disk.status.resumable());
12809 assert!(runner.state.driver_exited);
12810 assert_eq!(runner.state.driver_pid, None);
12811 runner.state.save().unwrap();
12812 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
12813 }
12814
12815 #[tokio::test]
12816 #[cfg(unix)]
12817 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
12818 let tmp = tempfile::tempdir().unwrap();
12819 let mut runner = registered_runner(tmp.path()).await;
12820 let winner = runner.state.candidates[0].clone();
12821 let cache = tmp.path().join("cache");
12822 let started = tmp.path().join("pre-gate-started");
12823 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
12824 runner.state.config.verify.gate =
12825 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
12826 let home = crate::run::home();
12827 let owner =
12828 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
12829 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
12830 crate::cache::AcquireOutcome::Acquired(g) => g,
12831 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
12832 };
12833 let mut gate = Box::pin(runner.run_gate(&winner));
12834 assert!(
12835 tokio::time::timeout(Duration::from_millis(150), &mut gate)
12836 .await
12837 .is_err()
12838 );
12839 assert!(!started.exists(), "pre-gate must wait for the cache too");
12840 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
12844 let editor = crate::repo_lock::SeatLease::checkout_with(
12845 &winner.worktree,
12846 true,
12847 crate::config::test_coordination().as_ref(),
12848 &wait,
12849 )
12850 .await
12851 .unwrap();
12852 drop(editor);
12853 guard.release();
12854 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12855 .await
12856 .unwrap()
12857 .unwrap();
12858 drop(gate);
12859 assert!(outcomes.iter().all(CommandOutcome::ok));
12860 assert!(started.exists());
12861 }
12862
12863 #[tokio::test]
12864 #[cfg(unix)]
12865 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
12866 use std::os::fd::AsRawFd as _;
12867
12868 ask_test_home();
12869 let tmp = tempfile::tempdir().unwrap();
12870 let repo = tmp.path().join("repo");
12871 std::fs::create_dir_all(&repo).unwrap();
12872 init_repo(&repo);
12873 let common = repo.join(".git");
12874 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
12875 let mut runner = runner_at(RunStatus::Prep);
12876 runner.state.repo = repo.clone();
12877 runner.state.config.coordination = crate::config::test_coordination();
12878 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12879 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
12880 runner.state.config.graph.advise = false;
12881 runner.roles.implementers = vec![spec("implement")];
12882 runner.roles.judges = vec![spec("judge")];
12883 runner.prep().await.unwrap();
12884 let winner = runner.state.candidates[0].clone();
12885 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
12888 let completed = tmp.path().join("seat-completed");
12889 runner.roles.implementers[0].command = vec![
12890 "sh".into(),
12891 "-c".into(),
12892 format!("touch '{}'; echo completed", completed.display()),
12893 ];
12894 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
12895 assert_eq!(
12896 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12897 0
12898 );
12899 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12900 let mut implement = Box::pin(runner.implement());
12901 tokio::select! {
12902 result = &mut implement => panic!("rescue must wait: {result:?}"),
12903 result = tokio::time::timeout(Duration::from_secs(10), async {
12904 while !completed.exists() {
12905 tokio::time::sleep(Duration::from_millis(20)).await;
12906 }
12907 }) => result.unwrap(),
12908 }
12909 assert!(
12910 tokio::time::timeout(Duration::from_millis(200), &mut implement)
12911 .await
12912 .is_err()
12913 );
12914 assert_eq!(
12915 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12916 before
12917 );
12918 assert_eq!(
12919 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12920 0
12921 );
12922 tokio::time::timeout(Duration::from_secs(10), &mut implement)
12923 .await
12924 .unwrap()
12925 .unwrap();
12926 drop(implement);
12927 assert_ne!(
12928 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12929 before
12930 );
12931 assert!(git::is_clean(&winner.worktree).await.unwrap());
12932
12933 let started = tmp.path().join("pre-gate-started");
12934 let proceed = tmp.path().join("pre-gate-proceed");
12935 let checked = tmp.path().join("gate-ran");
12936 runner.state.config.verify.pre_gate = vec![format!(
12937 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
12938 started.display(),
12939 proceed.display()
12940 )];
12941 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
12942 assert_eq!(
12943 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12944 0
12945 );
12946 let mut gate = Box::pin(runner.run_gate(&winner));
12947 assert!(
12948 tokio::time::timeout(Duration::from_millis(200), &mut gate)
12949 .await
12950 .is_err()
12951 );
12952 assert!(!started.exists());
12953 assert!(!checked.exists());
12954 assert!(git::is_clean(&winner.worktree).await.unwrap());
12955 assert_eq!(
12956 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
12957 0
12958 );
12959 tokio::select! {
12960 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
12961 result = tokio::time::timeout(Duration::from_secs(10), async {
12962 while !started.exists() {
12963 tokio::time::sleep(Duration::from_millis(20)).await;
12964 }
12965 }) => result.unwrap(),
12966 }
12967 assert_eq!(
12968 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12969 -1,
12970 "mechanical edits must hold the real lock"
12971 );
12972 std::fs::write(proceed, "ready").unwrap();
12973 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
12974 .await
12975 .unwrap()
12976 .unwrap();
12977 drop(gate);
12978 assert!(outcomes.iter().all(CommandOutcome::ok));
12979 assert!(checked.exists());
12980 assert!(runner.state.pre_gate_commit.is_some());
12981 assert!(git::is_clean(&winner.worktree).await.unwrap());
12982 assert_eq!(
12983 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12984 0,
12985 "gate completion must release coordination"
12986 );
12987 drop(outsider);
12988 std::fs::remove_file(common.join("updater.lock")).unwrap();
12989 std::fs::create_dir(common.join("updater.lock")).unwrap();
12990 std::fs::remove_file(&started).unwrap();
12991 std::fs::remove_file(&checked).unwrap();
12992 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12993 let error = runner.run_gate(&winner).await.unwrap_err();
12994 assert!(
12995 error
12996 .to_string()
12997 .contains("acquire mandatory repository lock")
12998 );
12999 assert!(
13000 !started.exists(),
13001 "failed acquisition must prevent pre-gate"
13002 );
13003 assert!(!checked.exists(), "failed acquisition must prevent gate");
13004 assert!(git::is_clean(&winner.worktree).await.unwrap());
13005 assert_eq!(
13006 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13007 head
13008 );
13009 }
13010
13011 #[tokio::test]
13012 #[cfg(unix)]
13013 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13014 use std::os::fd::AsRawFd as _;
13015 use std::time::Duration;
13016
13017 ask_test_home();
13018 let tmp = tempfile::tempdir().unwrap();
13019 let repo = tmp.path().join("repo");
13020 std::fs::create_dir_all(&repo).unwrap();
13021 init_repo(&repo);
13022 let common = repo.join(".git");
13023 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13024 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13025 assert_eq!(
13026 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13027 0
13028 );
13029 let mut runner = runner_at(RunStatus::Prep);
13030 runner.state.repo = repo.clone();
13031 runner.state.config.coordination = crate::config::test_coordination();
13032 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13033 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13034 runner.state.config.graph.advise = true;
13035 runner.state.config.graph.advisors = 1;
13036 runner.roles.implementers = vec![spec("implement")];
13037 runner.roles.judges = vec![spec("judge")];
13038 let root = runner.state.worktree_root();
13039 let started = tmp.path().join("setup-started");
13040 let proceed = tmp.path().join("setup-proceed");
13041 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13042 run: Some(format!(
13043 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13044 started.display(),
13045 proceed.display()
13046 )),
13047 ..Default::default()
13048 }];
13049 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13050 let config = git::git(&repo, &["config", "--local", "--list"])
13051 .await
13052 .unwrap();
13053 let mut prep = Box::pin(runner.prep());
13054 assert!(
13055 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13056 .await
13057 .is_err()
13058 );
13059 assert!(
13060 !root.exists(),
13061 "contention must prevent even checkout creation"
13062 );
13063 assert!(!started.exists(), "contention must prevent setup");
13064 assert_eq!(
13065 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13066 branches
13067 );
13068 assert_eq!(
13069 git::git(&repo, &["config", "--local", "--list"])
13070 .await
13071 .unwrap(),
13072 config
13073 );
13074 drop(lock);
13075 tokio::select! {
13076 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13077 result = tokio::time::timeout(Duration::from_secs(10), async {
13078 while !started.exists() {
13079 tokio::time::sleep(Duration::from_millis(20)).await;
13080 }
13081 }) => result.unwrap(),
13082 }
13083 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13084 assert_eq!(
13085 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13086 -1
13087 );
13088 assert!(
13089 tokio::time::timeout(
13090 Duration::from_millis(100),
13091 crate::repo_lock::SeatLease::for_checkout(
13092 &repo,
13093 true,
13094 crate::config::test_coordination().as_ref()
13095 )
13096 )
13097 .await
13098 .is_err(),
13099 "preparation must also exclude local editors"
13100 );
13101 std::fs::write(proceed, "ready").unwrap();
13102 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13103 .await
13104 .unwrap()
13105 .unwrap();
13106 drop(prep);
13107 assert_eq!(runner.state.prep_complete, Some(true));
13108 assert!(root.join("judge-1").exists());
13109 assert!(root.join("advisor-1").exists());
13110 assert_eq!(
13111 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13112 0
13113 );
13114 }
13115
13116 #[tokio::test]
13117 #[cfg(unix)]
13118 async fn preparation_lock_failure_prevents_repository_changes() {
13119 ask_test_home();
13120 let tmp = tempfile::tempdir().unwrap();
13121 let repo = tmp.path().join("repo");
13122 std::fs::create_dir_all(&repo).unwrap();
13123 init_repo(&repo);
13124 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13125 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13128 let mut runner = runner_at(RunStatus::Prep);
13129 runner.state.repo = repo.clone();
13130 runner.state.config.coordination = crate::config::test_coordination();
13131 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13132 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13133 runner.roles.implementers = vec![spec("implement")];
13134 let error = runner.prep().await.unwrap_err();
13135 assert!(
13136 error
13137 .to_string()
13138 .contains("acquire mandatory repository lock")
13139 );
13140 assert!(!runner.state.worktree_root().exists());
13141 assert!(runner.state.candidates.is_empty());
13142 assert!(!repo.join(".git/config.worktree").exists());
13143 assert_eq!(
13144 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13145 "* main"
13146 );
13147 }
13148
13149 #[tokio::test]
13150 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13151 ask_test_home();
13152 let tmp = tempfile::tempdir().unwrap();
13153 let repo = tmp.path().join("repo");
13154 std::fs::create_dir_all(&repo).unwrap();
13155 init_repo(&repo);
13156 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13157 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13158 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13159 git::git(&repo, &["add", "."]).await.unwrap();
13160 git::git(&repo, &["commit", "-m", "required seed"])
13161 .await
13162 .unwrap();
13163 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13164 git::git(&repo, &["checkout", "main"]).await.unwrap();
13165 let mut runner = runner_at(RunStatus::Prep);
13166 runner.state.repo = repo.clone();
13167 runner.state.base_commit = base.clone();
13168 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13169 runner.state.config.graph.advise = false;
13170 runner.state.config.blind.commit_msg_hook = false;
13171 runner.state.seeds = vec![refs::Seed {
13172 token: seed.clone(),
13173 sha: seed.clone(),
13174 kind: refs::SeedKind::Unmerged,
13175 branch: false,
13176 detail: String::new(),
13177 }];
13178 runner.roles.implementers = vec![spec("one"), spec("two")];
13179 runner.roles.judges = vec![spec("judge")];
13180 let label = blind::assign_labels(2, runner.state.seed)[0];
13181 let branch = runner.state.branch_for(label);
13182 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13183 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13184 .await
13185 .unwrap();
13186 runner.state.candidates.push(Candidate {
13187 index: 0,
13188 label,
13189 agent: "one".into(),
13190 branch,
13191 worktree,
13192 summary: String::new(),
13193 stat: String::new(),
13194 files: 0,
13195 commits: 0,
13196 empty: false,
13197 failed: None,
13198 verified_noop: None,
13199 duration_ms: 0,
13200 folded: false,
13201 });
13202 runner.state.save().unwrap();
13203 runner.state = RunState::load(&runner.state.id).unwrap();
13205 assert_eq!(runner.state.prep_complete, Some(false));
13206 runner.prep().await.unwrap();
13207 assert_eq!(runner.state.prep_complete, Some(true));
13208 assert_eq!(runner.state.candidates.len(), 2);
13209 for candidate in &runner.state.candidates {
13210 assert_eq!(
13211 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13212 git::tree_of(&repo, &seed).await.unwrap()
13213 );
13214 assert_eq!(
13215 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13216 "required commit"
13217 );
13218 }
13219 assert!(runner.state.worktree_root().join("judge-1").exists());
13220 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13221 runner.prep().await.unwrap();
13222 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13223
13224 runner.state.prep_complete = Some(false);
13225 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13226 run: Some(format!(
13227 "test -f '{}'",
13228 tmp.path().join("dependency-ready").display()
13229 )),
13230 ..Default::default()
13231 }];
13232 assert!(runner.prep().await.is_err());
13233 assert!(runner.state.prep_failure.is_some());
13234 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13235 runner.state = RunState::load(&runner.state.id).unwrap();
13236 assert!(
13237 runner.prep().await.is_err(),
13238 "correcting the cause alone must not auto-retry"
13239 );
13240 let failed_at = runner.state.events.last().unwrap().at;
13241 assert!(
13242 runner.retry_preparation(failed_at).is_err(),
13243 "old approvals cannot unlock the hold"
13244 );
13245 let mut task = crate::queue::Task::new(
13246 "retry".into(),
13247 "retry setup".into(),
13248 repo.clone(),
13249 crate::queue::Source::Human,
13250 );
13251 task.start(runner.state.id.clone());
13252 task.hold_machine(Some("setup failed".into()));
13253 task.release_operator("cli", None);
13254 runner
13255 .retry_preparation(task.resumption.as_ref().unwrap().at)
13256 .unwrap();
13257 runner.prep().await.unwrap();
13258 assert!(runner.state.prep_failure.is_none());
13259 assert_eq!(runner.state.prep_complete, Some(true));
13260 assert_eq!(runner.state.candidates.len(), 2);
13261 assert!(runner.state.worktree_root().join("judge-1").exists());
13262 for candidate in &runner.state.candidates {
13263 assert_eq!(
13264 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13265 git::tree_of(&repo, &seed).await.unwrap()
13266 );
13267 }
13268 }
13269
13270 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13271 let mut heads = Vec::new();
13272 for candidate in candidates {
13273 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13274 }
13275 heads
13276 }
13277
13278 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13279 ask_test_home();
13280 let tmp = tempfile::tempdir().unwrap();
13281 let repo = tmp.path().join("repo");
13282 std::fs::create_dir(&repo).unwrap();
13283 init_repo(&repo);
13284 let mut runner = runner_at(RunStatus::Reviewing);
13285 runner.state.repo = repo.clone();
13286 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13287 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13288 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13289 let branch = format!("candidate-{label}");
13290 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13291 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13292 .await
13293 .unwrap();
13294 if label == 'B' {
13295 std::fs::write(worktree.join("change"), "winner").unwrap();
13296 git::rescue_commit(&worktree, "candidate change")
13297 .await
13298 .unwrap();
13299 }
13300 runner.state.candidates.push(Candidate {
13301 index,
13302 label,
13303 agent: "alpha".into(),
13304 branch,
13305 worktree,
13306 summary: "completed reply".into(),
13307 stat: String::new(),
13308 files: usize::from(label == 'B'),
13309 commits: usize::from(label == 'B'),
13310 empty: label == 'A',
13311 failed: None,
13312 verified_noop: None,
13313 duration_ms: 1,
13314 folded: false,
13315 });
13316 }
13317 runner.state.judge_skipped = true;
13318 runner.state.tally = Some(Tally {
13319 first_choice: BTreeMap::new(),
13320 borda: BTreeMap::new(),
13321 winner: 'B',
13322 rankings: 0,
13323 unanimous_initial: false,
13324 deliberated: false,
13325 changed_votes: 0,
13326 unanimous_final: false,
13327 tie_break: None,
13328 judges: 0,
13329 present: 0,
13330 quorum: 0,
13331 met_quorum: true,
13332 uncontested: Some("only B produced a change".into()),
13333 });
13334 runner.state.save().unwrap();
13335 (tmp, runner)
13336 }
13337
13338 #[tokio::test]
13339 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
13340 let (_tmp, mut runner) = preservation_resume_fixture().await;
13341 runner.implement().await.unwrap();
13343 assert!(runner.state.candidate_preservation.is_empty());
13344 assert!(
13345 !runner
13346 .state
13347 .dir()
13348 .join("artifacts/candidate-A-recovery")
13349 .exists()
13350 );
13351 runner.fold_losers().await.unwrap();
13352 assert!(!runner.state.candidates[0].worktree.exists());
13353 for schema in [14, crate::run::SCHEMA] {
13354 runner.state.schema = schema;
13355 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13356 runner.state.save().unwrap();
13357 runner.state = RunState::load(&runner.state.id).unwrap();
13358 runner.implement().await.unwrap();
13359 assert!(runner.state.candidate_preservation.is_empty());
13360 assert!(runner.state.candidates[0].folded);
13361 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13362 }
13363 }
13364
13365 #[tokio::test]
13366 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
13367 let (_tmp, mut runner) = preservation_resume_fixture().await;
13368 let wt = runner.state.candidates[0].worktree.clone();
13369 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
13370 runner.state.schema = 14;
13371 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13372 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13373 let tally = serde_json::to_value(&runner.state.tally).unwrap();
13374 for _ in 0..2 {
13375 assert!(runner.implement().await.is_err());
13376 assert_eq!(runner.state.status, RunStatus::Blocked);
13377 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13378 assert!(
13379 runner.state.candidates[0].empty,
13380 "old verdict must remain unusable"
13381 );
13382 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
13383 assert!(runner.state.judge_skipped);
13384 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
13385 assert!(
13386 crate::run::read_artifact(&runner.state, "cand-A.patch")
13387 .unwrap()
13388 .contains("owned edits")
13389 );
13390 assert!(wt.exists());
13391 assert_eq!(
13392 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
13393 .await
13394 .unwrap(),
13395 1
13396 );
13397 runner.state = RunState::load(&runner.state.id).unwrap();
13398 }
13399 }
13400
13401 #[tokio::test]
13402 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
13403 let (_tmp, mut runner) = preservation_resume_fixture().await;
13404 let wt = runner.state.candidates[0].worktree.clone();
13405 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
13406 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
13407 let index_lock = PathBuf::from(
13408 git::git(
13409 &wt,
13410 &[
13411 "rev-parse",
13412 "--path-format=absolute",
13413 "--git-path",
13414 "index.lock",
13415 ],
13416 )
13417 .await
13418 .unwrap(),
13419 );
13420 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13421 runner.state.schema = 14;
13422 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
13423 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13424 assert!(runner.implement().await.is_err());
13425 assert!(runner.after_implement().is_err());
13426 assert!(runner.fold_losers().await.is_err());
13427 assert_eq!(
13428 std::fs::read_to_string(wt.join("recovered")).unwrap(),
13429 "failed seat edits"
13430 );
13431 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13432 runner.state = RunState::load(&runner.state.id).unwrap();
13433 std::fs::remove_file(index_lock).unwrap();
13434 assert!(runner.implement().await.is_err());
13436 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
13437 assert!(runner.state.candidates[0].empty);
13438 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
13439 assert!(
13440 crate::run::read_artifact(&runner.state, "cand-A.patch")
13441 .unwrap()
13442 .contains("failed seat edits")
13443 );
13444 assert!(wt.exists());
13445 }
13446
13447 #[tokio::test]
13448 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
13449 let (_tmp, mut runner) = preservation_resume_fixture().await;
13450 let repo = runner.state.repo.clone();
13451 let common = git::git(
13452 &repo,
13453 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
13454 )
13455 .await
13456 .unwrap();
13457 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
13458 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
13459 assert!(runner.fold_losers().await.is_err());
13460 assert!(!runner.state.candidates[0].worktree.exists());
13461 assert!(!runner.state.candidates[0].folded);
13462 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
13463 runner.state = RunState::load(&runner.state.id).unwrap();
13464 std::fs::remove_file(branch_lock).unwrap();
13465 let judge = runner.state.worktree_root().join("judge-1");
13466 git::worktree_add_detached(&repo, &judge, "main")
13467 .await
13468 .unwrap();
13469 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
13470 runner.roles.judges = vec![spec("judge")];
13471 assert!(runner.fold_losers().await.is_err());
13472 runner.state = RunState::load(&runner.state.id).unwrap();
13473 assert!(
13474 runner.state.candidates[0].folded,
13475 "candidate progress saved before judge failure"
13476 );
13477 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
13478 assert!(judge.join("uncommitted").exists());
13479 std::fs::remove_file(judge.join("uncommitted")).unwrap();
13480 runner.fold_losers().await.unwrap();
13481 assert!(!judge.exists());
13482 runner.fold_losers().await.unwrap();
13483 }
13484
13485 #[tokio::test]
13486 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
13487 let (_tmp, mut runner) = preservation_resume_fixture().await;
13488 runner.state.candidates.retain(|c| c.label == 'B');
13489 runner.state.tally = None;
13490 runner.state.judge_skipped = false;
13491 runner.state.prep_complete = Some(true);
13492 runner.state.config.graph.advise = false;
13493 runner.state.config.graph.selected_implementer = Some("alpha".into());
13494 let wt = runner.state.candidates[0].worktree.clone();
13495 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
13496 runner
13497 .state
13498 .candidate_preservation
13499 .insert('B', "awaiting persistence".into());
13500 let index_lock = git::git(
13501 &wt,
13502 &[
13503 "rev-parse",
13504 "--path-format=absolute",
13505 "--git-path",
13506 "index.lock",
13507 ],
13508 )
13509 .await
13510 .unwrap();
13511 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13512 runner.state.save().unwrap();
13513 assert!(runner.execute_graph().await.is_err());
13514 assert_eq!(runner.state.status, RunStatus::Blocked);
13515 assert!(runner.state.candidate_preservation.contains_key(&'B'));
13516 std::fs::remove_file(index_lock).unwrap();
13517 for _ in 0..2 {
13520 runner.state = RunState::load(&runner.state.id).unwrap();
13521 runner.pause.park();
13522 runner.execute_graph().await.unwrap();
13523 assert_eq!(runner.state.status, RunStatus::Implementing);
13524 assert!(runner.state.candidate_preservation.is_empty());
13525 assert_eq!(runner.state.candidates[0].commits, 2);
13526 assert!(runner.state.tally.is_none());
13527 }
13528 runner.implement().await.unwrap();
13529 assert_eq!(runner.state.status, RunStatus::Judging);
13530 runner.state.status = RunStatus::Blocked;
13532 runner.state.candidates[0].failed = Some("implementation failed".into());
13533 runner.state.save().unwrap();
13534 let error = runner.execute_graph().await.unwrap_err();
13535 assert!(
13536 error
13537 .to_string()
13538 .contains("selected implementation is held")
13539 );
13540 }
13541
13542 #[tokio::test]
13543 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
13544 let (_tmp, mut runner) = preservation_resume_fixture().await;
13545 let repo = runner.state.repo.clone();
13546 let candidate = runner.state.candidates[0].clone();
13547 let registered = git::worktree_holding(&repo, &candidate.branch)
13550 .await
13551 .unwrap()
13552 .unwrap();
13553 assert_eq!(
13554 registered.canonicalize().unwrap(),
13555 candidate.worktree.canonicalize().unwrap()
13556 );
13557 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13558 assert_eq!(
13559 git::worktree_holding(&repo, &candidate.branch)
13560 .await
13561 .unwrap(),
13562 Some(registered)
13563 );
13564 assert!(
13565 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
13566 "the stale registration initially keeps the branch checked out"
13567 );
13568 runner.fold_losers().await.unwrap();
13569 assert!(runner.state.candidates[0].folded);
13570 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13571 assert_eq!(
13572 git::worktree_holding(&repo, &candidate.branch)
13573 .await
13574 .unwrap(),
13575 None
13576 );
13577 runner.state = RunState::load(&runner.state.id).unwrap();
13578 runner.fold_losers().await.unwrap();
13579 }
13580
13581 #[tokio::test]
13582 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
13583 let (_tmp, mut runner) = preservation_resume_fixture().await;
13584 let repo = runner.state.repo.clone();
13585 let candidate = runner.state.candidates[0].clone();
13586 let winner = runner.state.candidates[1].clone();
13587 let registered = git::worktree_holding(&repo, &candidate.branch)
13588 .await
13589 .unwrap()
13590 .unwrap();
13591 assert_eq!(
13592 registered.canonicalize().unwrap(),
13593 candidate.worktree.canonicalize().unwrap()
13594 );
13595 std::fs::remove_dir_all(&candidate.worktree).unwrap();
13596 assert_eq!(
13597 git::worktree_holding(&repo, &candidate.branch)
13598 .await
13599 .unwrap(),
13600 Some(registered)
13601 );
13602 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
13603
13604 for _ in 0..2 {
13605 fold_run(&mut runner.state, false, &crate::run::home())
13606 .await
13607 .unwrap();
13608 runner.state = RunState::load(&runner.state.id).unwrap();
13609 assert!(runner.state.candidates[0].folded);
13610 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
13611 assert_eq!(
13612 git::worktree_holding(&repo, &candidate.branch)
13613 .await
13614 .unwrap(),
13615 None
13616 );
13617 assert!(!runner.state.candidates[1].folded);
13618 assert!(winner.worktree.exists());
13619 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
13620 }
13621 }
13622
13623 #[tokio::test]
13624 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
13625 ask_test_home();
13626 let tmp = tempfile::tempdir().unwrap();
13627 let repo = tmp.path().join("repo");
13628 std::fs::create_dir(&repo).unwrap();
13629 init_repo(&repo);
13630 let mut runner = runner_at(RunStatus::Implementing);
13631 runner.state.repo = repo.clone();
13632 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13633 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13634 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13635 let branch = format!("candidate-{label}");
13636 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
13637 git::worktree_add_branch(&repo, &wt, &branch, "main")
13638 .await
13639 .unwrap();
13640 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
13641 git::git(&wt, &["add", "README.md"]).await.unwrap();
13642 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
13643 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
13644 runner.state.candidates.push(Candidate {
13645 index,
13646 label,
13647 agent: "alpha".to_owned(),
13648 branch,
13649 worktree: wt,
13650 summary: "completed implementation".to_owned(),
13651 stat: String::new(),
13652 files: 0,
13653 commits: 0,
13654 empty: false,
13655 failed: None,
13656 verified_noop: None,
13657 duration_ms: 42,
13658 folded: false,
13659 });
13660 runner
13661 .state
13662 .candidate_preservation
13663 .insert(label, "awaiting persistence".to_owned());
13664 }
13665 let wt = runner.state.candidates[0].worktree.clone();
13666 let index_lock = git::git(
13667 &wt,
13668 &[
13669 "rev-parse",
13670 "--path-format=absolute",
13671 "--git-path",
13672 "index.lock",
13673 ],
13674 )
13675 .await
13676 .unwrap();
13677 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
13678 runner.state.save().unwrap();
13679 assert!(runner.persist_candidates().await.is_err());
13680 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13681 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
13682 assert_eq!(
13683 runner.state.candidates[1].commits, 1,
13684 "other results are still collected"
13685 );
13686 assert!(
13687 !runner.state.candidates[0].empty,
13688 "failed rescue is not no change"
13689 );
13690 assert_eq!(
13691 runner.state.candidates[0].summary,
13692 "completed implementation"
13693 );
13694 assert!(runner.after_implement().is_err());
13695 let home = tmp.path().join("fold-home");
13696 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
13697 assert!(wt.join("new-file").exists());
13698 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
13699 assert!(recovery.join("staged.patch").exists());
13700 assert!(recovery.join("unstaged.patch").exists());
13701 assert_eq!(
13702 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
13703 "untracked edits\n"
13704 );
13705 runner.state.begin_pass();
13706 runner.state.end_pass_parked(ParkCause::Upgrade {
13707 from: "1.0".into(),
13708 to: Some("1.1".into()),
13709 });
13710 runner.state.save().unwrap();
13711 runner.state = RunState::load(&runner.state.id).unwrap();
13712 assert!(matches!(
13713 runner.state.passes.last().unwrap().end,
13714 Some(crate::run::PassEnd::Parked {
13715 cause: ParkCause::Upgrade { .. }
13716 })
13717 ));
13718 std::fs::remove_file(index_lock).unwrap();
13719 runner.implement().await.unwrap();
13721 assert!(runner.state.candidate_preservation.is_empty());
13722 assert!(
13723 runner
13724 .state
13725 .candidates
13726 .iter()
13727 .all(|c| c.commits == 1 && !c.empty)
13728 );
13729 assert!(git::is_clean(&wt).await.unwrap());
13730 assert!(
13731 crate::run::read_artifact(&runner.state, "cand-A.patch")
13732 .unwrap()
13733 .contains("later tracked edits")
13734 );
13735 }
13736
13737 #[tokio::test]
13738 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
13739 ask_test_home();
13740 let tmp = tempfile::tempdir().unwrap();
13741 let repo = tmp.path().join("repo");
13742 std::fs::create_dir(&repo).unwrap();
13743 init_repo(&repo);
13744 let wt = tmp.path().join("candidate");
13745 git::worktree_add_branch(&repo, &wt, "candidate", "main")
13746 .await
13747 .unwrap();
13748 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13749 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
13750 assert!(wt.join("uncommitted").exists());
13751 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13752 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13753 let mut runner = runner_at(RunStatus::Implementing);
13754 runner.state.repo = repo.clone();
13755 runner.state.base_commit = "missing-base".to_owned();
13756 runner.state.candidates.push(Candidate {
13757 index: 0,
13758 label: 'A',
13759 agent: "alpha".into(),
13760 branch: "candidate".into(),
13761 worktree: wt.clone(),
13762 summary: "completed".into(),
13763 stat: String::new(),
13764 files: 0,
13765 commits: 0,
13766 empty: false,
13767 failed: None,
13768 verified_noop: None,
13769 duration_ms: 1,
13770 folded: false,
13771 });
13772 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
13773 assert!(
13774 fold_run(&mut runner.state, true, &tmp.path().join("home"))
13775 .await
13776 .is_err()
13777 );
13778 assert!(!runner.state.candidates[0].folded);
13779 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
13780 std::fs::remove_file(wt.join("uncommitted")).unwrap();
13781 runner
13782 .state
13783 .candidate_preservation
13784 .insert('A', "pending".into());
13785 assert!(runner.persist_candidates().await.is_err());
13786 assert!(!runner.state.candidates[0].empty);
13787 assert!(!runner.state.candidates[0].folded);
13788 assert!(wt.exists());
13789 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13790 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
13791 std::fs::create_dir_all(&patch_path).unwrap();
13792 assert!(
13793 runner.persist_candidates().await.is_err(),
13794 "artifact failure stays pending"
13795 );
13796 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13797 assert!(!runner.state.candidates[0].empty);
13798 std::fs::remove_dir(patch_path).unwrap();
13799 runner.persist_candidates().await.unwrap();
13800 assert!(
13801 runner.state.candidates[0].empty,
13802 "only successful inspection confirms no change"
13803 );
13804 }
13805
13806 fn init_repo(dir: &Path) {
13807 let run = |args: &[&str]| {
13808 let out = std::process::Command::new("git")
13809 .args(args)
13810 .current_dir(dir)
13811 .quiet()
13812 .output()
13813 .expect("spawn git");
13814 assert!(
13815 out.status.success(),
13816 "git {args:?} failed: {}",
13817 String::from_utf8_lossy(&out.stderr)
13818 );
13819 };
13820 run(&["init", "-b", "main"]);
13821 run(&["config", "user.name", "magi test"]);
13822 run(&["config", "user.email", "magi@example.com"]);
13823 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
13824 run(&["add", "-A"]);
13825 run(&["commit", "-m", "init"]);
13826 }
13827
13828 fn ask_test_home() {
13836 crate::run::pin_test_home();
13837 }
13838
13839 fn runner_at(status: RunStatus) -> Runner {
13842 let mut state = RunState::new(
13843 PathBuf::from("/nonexistent/repo"),
13844 "main".to_owned(),
13845 "deadbeef".to_owned(),
13846 "task".to_owned(),
13847 Config::default(),
13848 );
13849 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
13852 state.id = format!(
13853 "20000101-000000-{:04x}",
13854 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
13855 );
13856 state.status = status;
13857 Runner {
13858 state,
13859 roles: ResolvedRoles {
13860 implementers: Vec::new(),
13861 judges: Vec::new(),
13862 reviewers: Vec::new(),
13863 fixer: None,
13864 conductor: conductor(),
13865 implementer_roster: Vec::new(),
13866 judge_roster: Vec::new(),
13867 reviewer_roster: Vec::new(),
13868 },
13869 sem: Arc::new(Semaphore::new(1)),
13870 pause: Pause::new(),
13871 interrupt: Pause::new(),
13872 }
13873 }
13874
13875 #[test]
13876 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
13877 crate::run::pin_test_home();
13878 let mut runner = runner_at(RunStatus::Gating);
13879 runner.state.save().unwrap();
13880 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
13883 assert!(runner.claim_driver().is_err());
13884 assert!(runner.state.driver_exited);
13885 assert_eq!(runner.state.driver_pid, None);
13886 let disk = RunState::load(&runner.state.id).unwrap();
13887 assert!(disk.driver_exited);
13888 assert_eq!(disk.driver_pid, None);
13889 assert_eq!(disk.status, RunStatus::Gating);
13890 }
13891
13892 #[tokio::test]
13893 async fn an_early_graph_error_releases_its_driver() {
13894 crate::run::pin_test_home();
13895 let mut runner = runner_at(RunStatus::Blocked);
13896 runner.state.config.graph.selected_implementer = Some("selected".into());
13897 runner.state.save().unwrap();
13898 assert!(runner.execute().await.is_err());
13899 let disk = RunState::load(&runner.state.id).unwrap();
13900 assert!(disk.driver_exited);
13901 assert_eq!(disk.driver_pid, None);
13902 assert_eq!(disk.status, RunStatus::Blocked);
13903 }
13904
13905 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
13906 crate::run::MergeRepair {
13907 question: "repair-question".into(),
13908 head: "old-head".into(),
13909 pr: "https://github.com/o/r/pull/42".into(),
13910 reviews: vec![review_round(false, 1, 1, 1, false, true)],
13911 gate: vec![outcome(Some(0), false)],
13912 gate_head: Some("old-head".into()),
13913 gate_fixes: Vec::new(),
13914 pre_gate: Vec::new(),
13915 phase,
13916 failure: None,
13917 }
13918 }
13919
13920 #[tokio::test]
13921 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
13922 crate::run::pin_test_home();
13923 let mut runner = runner_at(RunStatus::Landing);
13924 runner.state.merge_repairs.push(merge_repair_request(
13925 crate::run::MergeRepairPhase::Requested,
13926 ));
13927 runner.state.save().unwrap();
13928 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
13929 assert!(
13930 runner
13931 .execute()
13932 .await
13933 .unwrap_err()
13934 .to_string()
13935 .contains("already running")
13936 );
13937 assert_eq!(
13938 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13939 crate::run::MergeRepairPhase::Requested
13940 );
13941 drop(claim);
13942 let mut disk = runner.state.clone();
13943 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
13944 disk.save().unwrap();
13945 assert!(
13946 runner
13947 .execute()
13948 .await
13949 .unwrap_err()
13950 .to_string()
13951 .contains("decision changed")
13952 );
13953 assert_eq!(
13954 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
13955 crate::run::MergeRepairPhase::Complete
13956 );
13957 }
13958
13959 #[test]
13960 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
13961 crate::run::pin_test_home();
13962 let mut runner = runner_at(RunStatus::Landing);
13963 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
13964 runner.state.reviews = request.reviews.clone();
13965 runner.state.gate = request.gate.clone();
13966 runner.state.gate_ran = true;
13967 runner.state.land_approval = Some(crate::run::LandApproval {
13968 question: "old".into(),
13969 head: "old-head".into(),
13970 });
13971 runner.state.merge_repairs.push(request);
13972 runner.begin_merge_repair_review(0).unwrap();
13973 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
13974 assert!(!runner.state.gate_status().ok());
13975 assert!(runner.state.land_approval.is_none());
13976 let disk = RunState::load(&runner.state.id).unwrap();
13977 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
13978 assert!(disk.merge_repairs[0].gate[0].ok());
13979 assert_eq!(
13980 disk.merge_repairs[0].phase,
13981 crate::run::MergeRepairPhase::Reviewing
13982 );
13983 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
13984 assert!(disk.reviews.is_empty());
13985 }
13986
13987 #[tokio::test]
13988 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
13989 crate::run::pin_test_home();
13990 for phase in [
13991 crate::run::MergeRepairPhase::Fixing,
13992 crate::run::MergeRepairPhase::Requested,
13993 ] {
13994 let mut runner = runner_at(RunStatus::Landing);
13995 runner.state.config.graph.land_rounds = 0;
13996 runner.state.merge_repairs.push(merge_repair_request(phase));
13997 runner.state.save().unwrap();
13998 assert!(!runner.repair_merge_review().await.unwrap());
13999 assert_eq!(runner.state.status, RunStatus::Blocked);
14000 assert_eq!(
14001 runner.state.merge_repairs[0].phase,
14002 crate::run::MergeRepairPhase::Failed
14003 );
14004 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14005 assert!(
14006 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14007 "interrupted"
14008 } else {
14009 "budget exhausted"
14010 })
14011 );
14012 assert!(runner.state.merge.is_none());
14013 }
14014 }
14015
14016 #[test]
14020 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14021 crate::run::pin_test_home();
14022 let mut runner = runner_at(RunStatus::Implementing);
14023 let interrupt = Pause::new();
14024 runner.watch_interrupt(interrupt.clone());
14025
14026 interrupt.park_because("task a1b2 asked to run first");
14027
14028 assert!(runner.park_here().expect("park_here"));
14029 assert!(runner.state.parked);
14030 let last = runner.state.events.last().expect("a park event");
14031 assert_eq!(last.node, "park");
14032 assert!(
14033 last.message.contains("task a1b2 asked to run first"),
14034 "expected the interrupt reason in {:?}",
14035 last.message
14036 );
14037 }
14038
14039 #[test]
14042 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14043 crate::run::pin_test_home();
14044 let mut runner = runner_at(RunStatus::Implementing);
14045 runner.state.begin_pass();
14046 let shutdown = Pause::new();
14047 runner.on_pause(shutdown.clone());
14048 let upgrade = ParkCause::Upgrade {
14049 from: "0.121.0".to_owned(),
14050 to: Some("0.121.1".to_owned()),
14051 };
14052 shutdown.park_for(upgrade.clone());
14053 shutdown.park_for(ParkCause::Operator);
14054
14055 assert!(runner.park_here().expect("park_here"));
14056 assert_eq!(
14057 runner.state.passes[0].end,
14058 Some(crate::run::PassEnd::Parked { cause: upgrade })
14059 );
14060 let last = runner.state.events.last().expect("a park event");
14061 assert!(last.message.contains("0.121.0"), "{}", last.message);
14062 }
14063
14064 #[test]
14072 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14073 crate::run::pin_test_home();
14074 let mut runner = runner_at(RunStatus::Implementing);
14075 let shutdown = Pause::new();
14076 runner.on_pause(shutdown.clone());
14077 let interrupt = Pause::new();
14078 runner.watch_interrupt(interrupt.clone());
14079
14080 assert!(!runner.park_here().expect("park_here"));
14082 assert!(!runner.state.parked);
14083
14084 interrupt.park_because("test");
14086 assert!(!shutdown.parked());
14087 assert!(runner.park_here().expect("park_here"));
14088 }
14089
14090 #[tokio::test]
14104 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14105 crate::run::pin_test_home();
14106 let mut runner = runner_at(RunStatus::Implementing);
14107 let interrupt = Pause::new();
14108 runner.watch_interrupt(interrupt.clone());
14109
14110 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14111 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14112
14113 let node = async move {
14117 started_tx.send(()).expect("send started");
14118 finish_rx.await.expect("recv finish");
14119 "node finished"
14120 };
14121
14122 let interrupter = async move {
14123 started_rx.await.expect("recv started");
14124 interrupt.park_because("higher-priority task waiting");
14126 tokio::task::yield_now().await;
14130 finish_tx.send(()).expect("send finish");
14131 };
14132
14133 let (node_result, ()) = tokio::join!(node, interrupter);
14134 assert_eq!(
14135 node_result, "node finished",
14136 "the in-flight call ran to completion"
14137 );
14138
14139 assert!(runner.park_here().expect("park_here"));
14142 assert!(runner.state.parked);
14143 }
14144
14145 #[test]
14151 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14152 crate::run::pin_test_home();
14153 let mut runner = runner_at(RunStatus::Judging);
14154 runner.state.config.agents = vec![conductor()];
14158 runner.state.candidates = vec![Candidate {
14159 index: 0,
14160 label: 'A',
14161 agent: "alpha".to_owned(),
14162 branch: "magi/x/A".to_owned(),
14163 worktree: PathBuf::from("/nonexistent/worktree"),
14164 summary: "did the thing".to_owned(),
14165 stat: "1 file changed".to_owned(),
14166 files: 1,
14167 commits: 1,
14168 empty: false,
14169 failed: None,
14170 verified_noop: None,
14171 duration_ms: 1234,
14172 folded: false,
14173 }];
14174 let run_id = runner.state.id.clone();
14175
14176 let interrupt = Pause::new();
14177 runner.watch_interrupt(interrupt.clone());
14178 interrupt.park_because("task c3d4 asked to run first");
14179 assert!(runner.park_here().expect("park_here"));
14180
14181 let resumed = Runner::resume(&run_id).expect("resume");
14182 assert_eq!(resumed.state.candidates.len(), 1);
14183 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14184 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14185 assert_eq!(resumed.state.status, runner.state.status);
14186 assert!(
14187 resumed.state.parked,
14188 "still parked until `execute` actually walks the graph again"
14189 );
14190 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14191 }
14192
14193 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14195 let mut q = ask::Question::new(
14196 run.to_owned(),
14197 "implement".to_owned(),
14198 "impl-A".to_owned(),
14199 "Which storage backend should the cache use?".to_owned(),
14200 String::new(),
14201 vec!["SQLite".to_owned(), "Redis".to_owned()],
14202 );
14203 store.put(&mut q).unwrap();
14204 q
14205 }
14206
14207 #[test]
14208 fn a_failed_runs_open_question_is_abandoned() {
14209 ask_test_home();
14210 let store = ask::Questions::open();
14211 let mut runner = runner_at(RunStatus::Failed);
14212 let run = runner.state.id.clone();
14213 let q = ask_open_question(&store, &run);
14214
14215 runner.settle_questions();
14216
14217 let back = store.get(&q.id).unwrap();
14218 assert!(
14219 !back.status.open(),
14220 "the seat that asked died with the run; nobody is left to read an answer"
14221 );
14222 assert!(
14223 back.detail.contains(&run) && back.detail.contains("failed"),
14224 "the reason names what the run became, not just that it is gone: {}",
14225 back.detail
14226 );
14227 }
14228
14229 #[test]
14230 fn a_merged_runs_open_question_is_abandoned_too() {
14231 ask_test_home();
14232 let store = ask::Questions::open();
14233 for status in [RunStatus::Merged, RunStatus::Ready] {
14236 let mut runner = runner_at(status);
14237 let run = runner.state.id.clone();
14238 let q = ask_open_question(&store, &run);
14239
14240 runner.settle_questions();
14241
14242 let back = store.get(&q.id).unwrap();
14243 assert!(
14244 !back.status.open(),
14245 "{status:?} run's question must not outlive the run"
14246 );
14247 }
14248 }
14249
14250 #[test]
14251 fn a_still_resumable_runs_open_question_is_left_alone() {
14252 ask_test_home();
14253 let store = ask::Questions::open();
14254 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14260 let mut runner = runner_at(status);
14261 let run = runner.state.id.clone();
14262 let q = ask_open_question(&store, &run);
14263
14264 runner.settle_questions();
14265
14266 let back = store.get(&q.id).unwrap();
14267 assert!(
14268 back.status.open(),
14269 "{status:?} is still alive; the question must still be waiting"
14270 );
14271 }
14272 }
14273
14274 #[test]
14275 fn settle_questions_never_touches_an_already_answered_question() {
14276 ask_test_home();
14277 let store = ask::Questions::open();
14278 let mut runner = runner_at(RunStatus::Failed);
14279 let run = runner.state.id.clone();
14280 let mut q = ask_open_question(&store, &run);
14281 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14282 .unwrap();
14283 store.put(&mut q).unwrap();
14284
14285 runner.settle_questions();
14290 runner.settle_questions();
14291
14292 let back = store.get(&q.id).unwrap();
14293 assert_eq!(
14294 back.status,
14295 ask::QuestionStatus::Answered,
14296 "a real answer is a decision on record, never overwritten by a sweep"
14297 );
14298 }
14299
14300 #[tokio::test]
14311 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14312 crate::run::pin_test_home();
14313 let tmp = tempfile::tempdir().expect("tempdir");
14314 let repo = tmp.path().join("repo");
14315 std::fs::create_dir_all(&repo).unwrap();
14316 init_repo(&repo);
14317
14318 let mut config = Config::default();
14319 config.graph.worktree_root = Some(tmp.path().join("wt"));
14320
14321 let mut state = RunState::new(
14322 repo.clone(),
14323 "main".to_owned(),
14324 "deadbeef".to_owned(),
14325 "task".to_owned(),
14326 config,
14327 );
14328 let root = state.worktree_root();
14329 let wt_a = root.join("cand-A");
14330 let wt_b = root.join("cand-B");
14331 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
14332 .await
14333 .expect("worktree A");
14334 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
14335 .await
14336 .expect("worktree B");
14337
14338 state.candidates = vec![
14339 Candidate {
14340 index: 0,
14341 label: 'A',
14342 agent: "alpha".to_owned(),
14343 branch: "magi/x/A".to_owned(),
14344 worktree: wt_a.clone(),
14345 summary: String::new(),
14346 stat: String::new(),
14347 files: 0,
14348 commits: 0,
14349 empty: false,
14350 failed: None,
14351 verified_noop: None,
14352 duration_ms: 0,
14353 folded: false,
14354 },
14355 Candidate {
14356 index: 1,
14357 label: 'B',
14358 agent: "beta".to_owned(),
14359 branch: "magi/x/B".to_owned(),
14360 worktree: wt_b.clone(),
14361 summary: String::new(),
14362 stat: String::new(),
14363 files: 0,
14364 commits: 0,
14365 empty: false,
14366 failed: None,
14367 verified_noop: None,
14368 duration_ms: 0,
14369 folded: false,
14370 },
14371 ];
14372 state.tally = Some(Tally {
14373 first_choice: BTreeMap::from([('A', 1)]),
14374 borda: BTreeMap::new(),
14375 winner: 'A',
14376 rankings: 1,
14377 unanimous_initial: true,
14378 deliberated: false,
14379 changed_votes: 0,
14380 unanimous_final: true,
14381 tie_break: None,
14382 judges: 1,
14383 present: 1,
14384 quorum: 1,
14385 met_quorum: true,
14386 uncontested: None,
14387 });
14388 state.status = RunStatus::Ready;
14389
14390 fold_run(&mut state, false, &crate::run::home())
14391 .await
14392 .expect("fold_run");
14393
14394 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
14395 assert!(
14396 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14397 "the unmerged winner's branch survives"
14398 );
14399 assert!(
14400 !state.candidates[0].folded,
14401 "the winner is not marked folded"
14402 );
14403
14404 assert!(!wt_b.exists(), "the loser's worktree is removed");
14405 assert!(
14406 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
14407 "the loser's branch is removed"
14408 );
14409 assert!(state.candidates[1].folded, "the loser is marked folded");
14410 }
14411
14412 #[tokio::test]
14413 #[cfg(unix)]
14414 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
14415 use std::os::fd::AsRawFd as _;
14416 use std::time::Duration;
14417
14418 let tmp = tempfile::tempdir().unwrap();
14419 let repo = tmp.path().join("repo");
14420 std::fs::create_dir_all(&repo).unwrap();
14421 init_repo(&repo);
14422 let common = repo.join(".git");
14423 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
14424 let home = tmp.path().join("home");
14425 let mut config = Config::default();
14426 config.coordination = crate::config::test_coordination();
14427 config.graph.worktree_root = Some(tmp.path().join("wt"));
14428 config.graph.timeout_resource_wait = 1;
14429 let mut state = RunState::new(
14430 repo.clone(),
14431 "main".into(),
14432 "deadbeef".into(),
14433 "task".into(),
14434 config,
14435 );
14436 let root = state.worktree_root();
14437 let candidate = root.join("cand-A");
14438 let reviewer = root.join("review-1");
14439 let branch = state.branch_for('A');
14440 git::worktree_add_branch(&repo, &candidate, &branch, "main")
14441 .await
14442 .unwrap();
14443 git::worktree_add_detached(&repo, &reviewer, "main")
14444 .await
14445 .unwrap();
14446 git::acquire_worktree_config(&repo).await.unwrap();
14447 state.enabled_worktree_config = true;
14448 state.status = RunStatus::Ready;
14449 state.candidates.push(Candidate {
14450 index: 0,
14451 label: 'A',
14452 agent: "implement".into(),
14453 branch: branch.clone(),
14454 worktree: candidate.clone(),
14455 summary: String::new(),
14456 stat: String::new(),
14457 files: 0,
14458 commits: 0,
14459 empty: false,
14460 failed: None,
14461 verified_noop: None,
14462 duration_ms: 0,
14463 folded: false,
14464 });
14465 state.save_under(&home).unwrap();
14466 let config_before = std::fs::read(common.join("config")).unwrap();
14467 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
14468 .await
14469 .unwrap();
14470 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
14473 assert_eq!(
14474 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14475 0
14476 );
14477 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14478 assert!(
14479 tokio::time::timeout(Duration::from_millis(200), &mut fold)
14480 .await
14481 .is_err()
14482 );
14483 assert!(candidate.exists());
14484 assert!(reviewer.exists());
14485 assert!(git::branch_exists(&repo, &branch).await.unwrap());
14486 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14487 assert_eq!(
14488 git::git(&repo, &["worktree", "list", "--porcelain"])
14489 .await
14490 .unwrap(),
14491 worktrees_before
14492 );
14493 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
14494 .await
14495 .unwrap()
14496 .unwrap_err();
14497 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
14498 drop(fold);
14499 assert!(candidate.exists() && reviewer.exists());
14500 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
14501 assert_eq!(
14502 state.resource_block.as_ref().unwrap().kind,
14503 "repository coordination"
14504 );
14505 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
14506 drop(external);
14507 let mut fold = Box::pin(fold_run(&mut state, true, &home));
14508 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
14509 .await
14510 .unwrap()
14511 .unwrap();
14512 drop(fold);
14513 assert!(removed.contains(&branch));
14514 assert!(!root.exists());
14515 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
14516 assert!(state.candidates[0].folded);
14517 assert!(!state.enabled_worktree_config);
14518 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
14519 let enabled = git::git_raw(
14520 &repo,
14521 &["config", "--local", "--get", "extensions.worktreeConfig"],
14522 )
14523 .await
14524 .unwrap();
14525 assert!(!enabled.ok(), "the last config reference was released");
14526 }
14527
14528 #[tokio::test]
14531 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
14532 let tmp = tempfile::tempdir().expect("tempdir");
14533 let repo = tmp.path().join("repo");
14534 std::fs::create_dir_all(&repo).unwrap();
14535 init_repo(&repo);
14536 let home = tmp.path().join("home");
14537
14538 let mut config = Config::default();
14539 config.graph.worktree_root = Some(tmp.path().join("wt"));
14540 let mut state = RunState::new(
14541 repo.clone(),
14542 "main".to_owned(),
14543 "deadbeef".to_owned(),
14544 "task".to_owned(),
14545 config,
14546 );
14547 git::git(&repo, &["branch", "magi/x/A", "main"])
14549 .await
14550 .expect("branch");
14551 state.candidates = vec![Candidate {
14552 index: 0,
14553 label: 'A',
14554 agent: "alpha".to_owned(),
14555 branch: "magi/x/A".to_owned(),
14556 worktree: state.worktree_root().join("cand-A"),
14557 summary: String::new(),
14558 stat: String::new(),
14559 files: 0,
14560 commits: 0,
14561 empty: false,
14562 failed: None,
14563 verified_noop: None,
14564 duration_ms: 0,
14565 folded: true,
14566 }];
14567 state.released_to = Some("20260901-000000-new1".to_owned());
14568 state.released_branches = vec!["magi/x/A".to_owned()];
14569
14570 fold_run(&mut state, true, &home).await.expect("fold_run");
14571
14572 assert!(
14573 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
14574 "the handed-over branch survives a fold"
14575 );
14576 }
14577
14578 #[tokio::test]
14582 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
14583 let tmp = tempfile::tempdir().expect("tempdir");
14584 let repo = tmp.path().join("repo");
14585 std::fs::create_dir_all(&repo).unwrap();
14586 init_repo(&repo);
14587 let run = |args: &[&str]| {
14588 let out = std::process::Command::new("git")
14589 .quiet()
14590 .args(args)
14591 .current_dir(&repo)
14592 .output()
14593 .expect("spawn git");
14594 assert!(out.status.success(), "git {args:?}");
14595 };
14596 run(&["branch", "magi/x/A"]);
14597 run(&["checkout", "-q", "-b", "magi/x/B"]);
14598 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
14599 run(&["add", "-A"]);
14600 run(&["commit", "-q", "-m", "work"]);
14601 run(&["checkout", "-q", "main"]);
14602
14603 let probe = RunState::new(
14607 repo.clone(),
14608 "main".to_owned(),
14609 "deadbeef".to_owned(),
14610 "task".to_owned(),
14611 Config::default(),
14612 );
14613 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
14614 let bare = tmp.path().join("origin.git");
14615 let out = std::process::Command::new("git")
14616 .quiet()
14617 .args(["init", "-q", "--bare"])
14618 .arg(&bare)
14619 .output()
14620 .expect("spawn git");
14621 assert!(out.status.success(), "git init --bare");
14622 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
14623 run(&["push", "-q", "origin", "main"]);
14624
14625 let mut state = RunState::new(
14626 repo.clone(),
14627 "main".to_owned(),
14628 "deadbeef".to_owned(),
14629 "task".to_owned(),
14630 Config::default(),
14631 );
14632 state.seeds = vec![refs::Seed {
14633 token: "magi/27b2/A".to_owned(),
14634 kind: refs::SeedKind::Unresolved,
14635 sha: String::new(),
14636 branch: true,
14637 detail: "no branch or commit named magi/27b2/A".to_owned(),
14638 }];
14639
14640 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
14641 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
14642 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
14643 let detail = empty_candidate_detail(&state, "main");
14644 assert!(detail.starts_with("empty candidate"), "{detail}");
14645 assert!(detail.contains("magi/27b2/A"), "{detail}");
14646 }
14647
14648 #[tokio::test]
14657 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
14658 let tmp = tempfile::tempdir().expect("tempdir");
14659 let repo = tmp.path().join("repo");
14660 std::fs::create_dir_all(&repo).unwrap();
14661 init_repo(&repo);
14662
14663 let mut config = Config::default();
14664 config.merge.mode = MergeMode::Local;
14665
14666 let mut state = RunState::new(
14667 repo.clone(),
14668 "main".to_owned(),
14669 "deadbeef".to_owned(),
14670 "task".to_owned(),
14671 config,
14672 );
14673 state.candidates = vec![Candidate {
14674 index: 0,
14675 label: 'A',
14676 agent: "alpha".to_owned(),
14677 branch: "does-not-exist".to_owned(),
14678 worktree: repo.clone(),
14679 summary: String::new(),
14680 stat: String::new(),
14681 files: 0,
14682 commits: 0,
14683 empty: false,
14684 failed: None,
14685 verified_noop: None,
14686 duration_ms: 0,
14687 folded: false,
14688 }];
14689 state.tally = Some(Tally {
14690 first_choice: BTreeMap::from([('A', 1)]),
14691 borda: BTreeMap::new(),
14692 winner: 'A',
14693 rankings: 1,
14694 unanimous_initial: true,
14695 deliberated: false,
14696 changed_votes: 0,
14697 unanimous_final: true,
14698 tie_break: None,
14699 judges: 0,
14700 present: 0,
14701 quorum: 0,
14702 met_quorum: true,
14703 uncontested: Some("only candidate A produced a change".to_owned()),
14704 });
14705 state.reviews = vec![ReviewRound {
14706 round: 1,
14707 head: "deadbeef".to_owned(),
14708 verified_head: None,
14709 verified_at: None,
14710 reviews: Vec::new(),
14711 e2e: Vec::new(),
14712 fix: None,
14713 blocking: 0,
14714 answered: 0,
14715 expected: 0,
14716 clean: true,
14717 verify_retried: false,
14718 e2e_deferred: false,
14719 e2e_defer_reason: None,
14720 progressed: false,
14721 vote_split: false,
14722 reconsideration: Vec::new(),
14723 verdict: None,
14724 }];
14725 state.gate = vec![CommandOutcome {
14726 command: "test".to_owned(),
14727 code: Some(0),
14728 output_tail: String::new(),
14729 duration_ms: 0,
14730 resource_blocked: false,
14731 }];
14732 state.gate_ran = true;
14733 state.status = RunStatus::Ready;
14738 state.merge = Some(MergeOutcome {
14739 mode: MergeMode::Local,
14740 ok: false,
14741 detail: "already concluded".to_owned(),
14742 empty: false,
14743 });
14744
14745 let mut runner = Runner {
14746 state,
14747 roles: ResolvedRoles {
14748 implementers: Vec::new(),
14749 judges: Vec::new(),
14750 reviewers: Vec::new(),
14751 fixer: None,
14752 conductor: conductor(),
14753 implementer_roster: Vec::new(),
14754 judge_roster: Vec::new(),
14755 reviewer_roster: Vec::new(),
14756 },
14757 sem: Arc::new(Semaphore::new(1)),
14758 pause: Pause::new(),
14759 interrupt: Pause::new(),
14760 };
14761
14762 runner.merge().await.expect("merge");
14763
14764 assert_eq!(
14765 runner.state.status,
14766 RunStatus::Ready,
14767 "a concluded run's status must not change on reentry"
14768 );
14769 assert_eq!(
14770 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
14771 Some("already concluded"),
14772 "merge must not run again once the node already recorded an outcome"
14773 );
14774 }
14775
14776 #[tokio::test]
14785 async fn merge_refuses_a_gate_that_has_not_actually_run() {
14786 let tmp = tempfile::tempdir().expect("tempdir");
14787 let repo = tmp.path().join("repo");
14788 std::fs::create_dir_all(&repo).unwrap();
14789 init_repo(&repo);
14790
14791 let mut config = Config::default();
14792 config.merge.mode = MergeMode::Local;
14793
14794 let mut state = RunState::new(
14795 repo.clone(),
14796 "main".to_owned(),
14797 "deadbeef".to_owned(),
14798 "task".to_owned(),
14799 config,
14800 );
14801 state.candidates = vec![Candidate {
14802 index: 0,
14803 label: 'A',
14804 agent: "alpha".to_owned(),
14805 branch: "does-not-exist".to_owned(),
14806 worktree: repo.clone(),
14807 summary: String::new(),
14808 stat: String::new(),
14809 files: 0,
14810 commits: 0,
14811 empty: false,
14812 failed: None,
14813 verified_noop: None,
14814 duration_ms: 0,
14815 folded: false,
14816 }];
14817 state.tally = Some(Tally {
14818 first_choice: BTreeMap::from([('A', 1)]),
14819 borda: BTreeMap::new(),
14820 winner: 'A',
14821 rankings: 1,
14822 unanimous_initial: true,
14823 deliberated: false,
14824 changed_votes: 0,
14825 unanimous_final: true,
14826 tie_break: None,
14827 judges: 0,
14828 present: 0,
14829 quorum: 0,
14830 met_quorum: true,
14831 uncontested: Some("only candidate A produced a change".to_owned()),
14832 });
14833 state.reviews = vec![ReviewRound {
14834 round: 1,
14835 head: "deadbeef".to_owned(),
14836 verified_head: None,
14837 verified_at: None,
14838 reviews: Vec::new(),
14839 e2e: Vec::new(),
14840 fix: None,
14841 blocking: 0,
14842 answered: 0,
14843 expected: 0,
14844 clean: true,
14845 verify_retried: false,
14846 e2e_deferred: false,
14847 e2e_defer_reason: None,
14848 progressed: false,
14849 vote_split: false,
14850 reconsideration: Vec::new(),
14851 verdict: None,
14852 }];
14853 state.gate = Vec::new();
14855 state.gate_ran = false;
14856 state.status = RunStatus::Gating;
14857
14858 let mut runner = Runner {
14859 state,
14860 roles: ResolvedRoles {
14861 implementers: Vec::new(),
14862 judges: Vec::new(),
14863 reviewers: Vec::new(),
14864 fixer: None,
14865 conductor: conductor(),
14866 implementer_roster: Vec::new(),
14867 judge_roster: Vec::new(),
14868 reviewer_roster: Vec::new(),
14869 },
14870 sem: Arc::new(Semaphore::new(1)),
14871 pause: Pause::new(),
14872 interrupt: Pause::new(),
14873 };
14874
14875 runner.merge().await.expect("merge");
14876
14877 assert!(
14878 runner.state.merge.is_none(),
14879 "an empty gate must never be read as a passing one: {:?}",
14880 runner.state.merge
14881 );
14882 }
14883
14884 #[tokio::test]
14891 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
14892 let tmp = tempfile::tempdir().expect("tempdir");
14893 let repo = tmp.path().join("repo");
14894 std::fs::create_dir_all(&repo).unwrap();
14895 init_repo(&repo);
14896
14897 let config = Config::default();
14899
14900 let mut state = RunState::new(
14901 repo.clone(),
14902 "main".to_owned(),
14903 "deadbeef".to_owned(),
14904 "task".to_owned(),
14905 config,
14906 );
14907 state.candidates = vec![Candidate {
14908 index: 0,
14909 label: 'A',
14910 agent: "alpha".to_owned(),
14911 branch: "does-not-exist".to_owned(),
14912 worktree: repo.clone(),
14913 summary: String::new(),
14914 stat: String::new(),
14915 files: 0,
14916 commits: 0,
14917 empty: false,
14918 failed: None,
14919 verified_noop: None,
14920 duration_ms: 0,
14921 folded: false,
14922 }];
14923 state.tally = Some(Tally {
14924 first_choice: BTreeMap::from([('A', 1)]),
14925 borda: BTreeMap::new(),
14926 winner: 'A',
14927 rankings: 1,
14928 unanimous_initial: true,
14929 deliberated: false,
14930 changed_votes: 0,
14931 unanimous_final: true,
14932 tie_break: None,
14933 judges: 0,
14934 present: 0,
14935 quorum: 0,
14936 met_quorum: true,
14937 uncontested: Some("only candidate A produced a change".to_owned()),
14938 });
14939 state.reviews = vec![ReviewRound {
14940 round: 1,
14941 head: "deadbeef".to_owned(),
14942 verified_head: None,
14943 verified_at: None,
14944 reviews: Vec::new(),
14945 e2e: Vec::new(),
14946 fix: None,
14947 blocking: 0,
14948 answered: 0,
14949 expected: 0,
14950 clean: true,
14951 verify_retried: false,
14952 e2e_deferred: false,
14953 e2e_defer_reason: None,
14954 progressed: false,
14955 vote_split: false,
14956 reconsideration: Vec::new(),
14957 verdict: None,
14958 }];
14959
14960 let mut runner = Runner {
14961 state,
14962 roles: ResolvedRoles {
14963 implementers: Vec::new(),
14964 judges: Vec::new(),
14965 reviewers: Vec::new(),
14966 fixer: None,
14967 conductor: conductor(),
14968 implementer_roster: Vec::new(),
14969 judge_roster: Vec::new(),
14970 reviewer_roster: Vec::new(),
14971 },
14972 sem: Arc::new(Semaphore::new(1)),
14973 pause: Pause::new(),
14974 interrupt: Pause::new(),
14975 };
14976
14977 runner.gate().await.expect("gate");
14978 assert!(
14979 runner.state.gate_ran,
14980 "zero configured commands is still a real attempt, not an unrun gate"
14981 );
14982 assert!(runner.state.gate.is_empty());
14983 assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
14984 assert_ne!(
14985 runner.state.status,
14986 RunStatus::Blocked,
14987 "a gate with nothing to check must not read as failed"
14988 );
14989
14990 runner.merge().await.expect("merge");
14991 assert_eq!(
14992 runner.state.status,
14993 RunStatus::Ready,
14994 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
14995 );
14996 }
14997
14998 #[tokio::test]
15009 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15010 crate::run::pin_test_home();
15011 let home = crate::run::home();
15012
15013 let tmp = tempfile::tempdir().expect("tempdir");
15014 let repo = tmp.path().join("repo");
15015 std::fs::create_dir_all(&repo).unwrap();
15016 init_repo(&repo);
15017 let cache_dir = tmp.path().join("target");
15020
15021 let mut config = Config::default();
15022 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15023 config.graph.timeout_verify = Some(2);
15026 config.graph.timeout_resource_wait = 2;
15027
15028 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15029 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15030 .expect("no io error acquiring directly")
15031 {
15032 crate::cache::AcquireOutcome::Acquired(g) => g,
15033 crate::cache::AcquireOutcome::Busy(b) => {
15034 panic!("expected the direct acquire to win the lease first: {b:?}")
15035 }
15036 };
15037
15038 let mut state = RunState::new(
15039 repo.clone(),
15040 "main".to_owned(),
15041 "deadbeef".to_owned(),
15042 "task".to_owned(),
15043 config,
15044 );
15045 state.candidates = vec![Candidate {
15046 index: 0,
15047 label: 'A',
15048 agent: "alpha".to_owned(),
15049 branch: "does-not-exist".to_owned(),
15050 worktree: repo.clone(),
15051 summary: String::new(),
15052 stat: String::new(),
15053 files: 0,
15054 commits: 0,
15055 empty: false,
15056 failed: None,
15057 verified_noop: None,
15058 duration_ms: 0,
15059 folded: false,
15060 }];
15061 state.tally = Some(Tally {
15062 first_choice: BTreeMap::from([('A', 1)]),
15063 borda: BTreeMap::new(),
15064 winner: 'A',
15065 rankings: 1,
15066 unanimous_initial: true,
15067 deliberated: false,
15068 changed_votes: 0,
15069 unanimous_final: true,
15070 tie_break: None,
15071 judges: 0,
15072 present: 0,
15073 quorum: 0,
15074 met_quorum: true,
15075 uncontested: Some("only candidate A produced a change".to_owned()),
15076 });
15077 state.reviews = vec![ReviewRound {
15078 round: 1,
15079 head: "deadbeef".to_owned(),
15080 verified_head: None,
15081 verified_at: None,
15082 reviews: Vec::new(),
15083 e2e: Vec::new(),
15084 fix: None,
15085 blocking: 0,
15086 answered: 0,
15087 expected: 0,
15088 clean: true,
15089 verify_retried: false,
15090 e2e_deferred: false,
15091 e2e_defer_reason: None,
15092 progressed: false,
15093 vote_split: false,
15094 reconsideration: Vec::new(),
15095 verdict: None,
15096 }];
15097
15098 let mut runner = Runner {
15099 state,
15100 roles: ResolvedRoles {
15101 implementers: Vec::new(),
15102 judges: Vec::new(),
15103 reviewers: Vec::new(),
15104 fixer: None,
15105 conductor: conductor(),
15106 implementer_roster: Vec::new(),
15107 judge_roster: Vec::new(),
15108 reviewer_roster: Vec::new(),
15109 },
15110 sem: Arc::new(Semaphore::new(1)),
15111 pause: Pause::new(),
15112 interrupt: Pause::new(),
15113 };
15114
15115 let started = std::time::Instant::now();
15116 runner.gate().await.expect("gate");
15117 assert!(
15118 started.elapsed() < Duration::from_secs(1),
15119 "a gate with nothing to run must never wait on a lease it never needed"
15120 );
15121 assert!(
15122 runner.state.gate_ran,
15123 "zero commands is still a real, immediate attempt"
15124 );
15125 assert!(runner.state.gate.is_empty());
15126 assert_ne!(
15127 runner.state.status,
15128 RunStatus::Blocked,
15129 "must not read as resource-blocked on a lease it never asked for"
15130 );
15131 }
15132
15133 #[tokio::test]
15143 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15144 crate::run::pin_test_home();
15145
15146 let tmp = tempfile::tempdir().expect("tempdir");
15147 let repo = tmp.path().join("repo");
15148 std::fs::create_dir_all(&repo).unwrap();
15149 init_repo(&repo);
15150
15151 let mut config = Config::default();
15152 config.verify.gate = vec![
15153 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15154 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15155 .to_owned(),
15156 ];
15157
15158 let mut state = RunState::new(
15159 repo.clone(),
15160 "main".to_owned(),
15161 "deadbeef".to_owned(),
15162 "task".to_owned(),
15163 config,
15164 );
15165 let run_id = state.id.clone();
15166 state.candidates = vec![Candidate {
15167 index: 0,
15168 label: 'A',
15169 agent: "alpha".to_owned(),
15170 branch: "does-not-exist".to_owned(),
15171 worktree: repo.clone(),
15172 summary: String::new(),
15173 stat: String::new(),
15174 files: 0,
15175 commits: 0,
15176 empty: false,
15177 failed: None,
15178 verified_noop: None,
15179 duration_ms: 0,
15180 folded: false,
15181 }];
15182 state.tally = Some(Tally {
15183 first_choice: BTreeMap::from([('A', 1)]),
15184 borda: BTreeMap::new(),
15185 winner: 'A',
15186 rankings: 1,
15187 unanimous_initial: true,
15188 deliberated: false,
15189 changed_votes: 0,
15190 unanimous_final: true,
15191 tie_break: None,
15192 judges: 0,
15193 present: 0,
15194 quorum: 0,
15195 met_quorum: true,
15196 uncontested: Some("only candidate A produced a change".to_owned()),
15197 });
15198 state.reviews = vec![ReviewRound {
15199 round: 1,
15200 head: "deadbeef".to_owned(),
15201 verified_head: None,
15202 verified_at: None,
15203 reviews: Vec::new(),
15204 e2e: Vec::new(),
15205 fix: None,
15206 blocking: 0,
15207 answered: 0,
15208 expected: 0,
15209 clean: true,
15210 verify_retried: false,
15211 e2e_deferred: false,
15212 e2e_defer_reason: None,
15213 progressed: false,
15214 vote_split: false,
15215 reconsideration: Vec::new(),
15216 verdict: None,
15217 }];
15218
15219 let mut runner = Runner {
15220 state,
15221 roles: ResolvedRoles {
15222 implementers: Vec::new(),
15223 judges: Vec::new(),
15224 reviewers: Vec::new(),
15225 fixer: None,
15226 conductor: conductor(),
15227 implementer_roster: Vec::new(),
15228 judge_roster: Vec::new(),
15229 reviewer_roster: Vec::new(),
15230 },
15231 sem: Arc::new(Semaphore::new(1)),
15232 pause: Pause::new(),
15233 interrupt: Pause::new(),
15234 };
15235
15236 let started_marker = repo.join("started.marker");
15237 let release_marker = repo.join("release.marker");
15238 let poller = tokio::spawn(async move {
15239 for _ in 0..100 {
15244 if started_marker.exists()
15245 && let Ok(s) = crate::run::RunState::load(&run_id)
15246 && let Some(a) = s.active.get("gate")
15247 {
15248 std::fs::write(&release_marker, b"go").expect("release marker");
15249 return Some(a.clone());
15250 }
15251 tokio::time::sleep(Duration::from_millis(50)).await;
15252 }
15253 None
15254 });
15255
15256 runner.gate().await.expect("gate");
15257 let captured = poller.await.expect("poller task");
15258 let captured = captured.expect(
15259 "the poller never saw a `gate` task entry in run.json while the command was \
15260 still blocked on its own release marker",
15261 );
15262
15263 assert_eq!(captured.task.as_deref(), Some("gate"));
15264 assert_eq!(captured.node, "gate");
15265 assert_eq!(captured.index, Some(1));
15266 assert_eq!(captured.total, Some(1));
15267 assert!(
15268 captured
15269 .command
15270 .as_deref()
15271 .is_some_and(|c| c.contains("started.marker")),
15272 "{captured:?}"
15273 );
15274
15275 assert!(
15276 runner.state.active.is_empty(),
15277 "the entry must be cleared once the command actually finished: {:?}",
15278 runner.state.active
15279 );
15280 assert!(runner.state.gate_ran);
15281 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
15282 }
15283
15284 #[tokio::test]
15287 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
15288 use crate::verdict::{Finding, ReviewVote, Severity};
15289 crate::run::pin_test_home();
15290 let tmp = tempfile::tempdir().expect("tempdir");
15291 let repo = tmp.path().join("repo");
15292 std::fs::create_dir_all(&repo).unwrap();
15293 init_repo(&repo);
15294
15295 for (severity, vote, expect) in [
15296 (Severity::Major, ReviewVote::Reject, true),
15297 (Severity::Minor, ReviewVote::Reject, false),
15298 (Severity::Major, ReviewVote::Approve, false),
15299 ] {
15300 let mut round = review_round(false, 1, 1, 1, false, true);
15301 round.reviews = vec![ReviewRecord {
15302 reviewer: 1,
15303 agent: "alpha".to_owned(),
15304 summary: String::new(),
15305 findings: vec![Finding {
15306 id: "R1-1-1".to_owned(),
15307 severity,
15308 file: None,
15309 line: None,
15310 title: "t".to_owned(),
15311 detail: String::new(),
15312 }],
15313 vote: Some(vote),
15314 failed: None,
15315 duration_ms: 0,
15316 attempts: 0,
15317 }];
15318 let mut state = RunState::new(
15319 repo.clone(),
15320 "main".to_owned(),
15321 "deadbeef".to_owned(),
15322 "task".to_owned(),
15323 Config::default(),
15324 );
15325 state.reviews = vec![round];
15326 let mut runner = Runner {
15327 state,
15328 roles: ResolvedRoles {
15329 implementers: Vec::new(),
15330 judges: Vec::new(),
15331 reviewers: Vec::new(),
15332 fixer: None,
15333 conductor: conductor(),
15334 implementer_roster: Vec::new(),
15335 judge_roster: Vec::new(),
15336 reviewer_roster: Vec::new(),
15337 },
15338 sem: Arc::new(Semaphore::new(1)),
15339 pause: Pause::new(),
15340 interrupt: Pause::new(),
15341 };
15342 let shell = runner.state.config.shell();
15343 runner
15344 .stop_reviewing("round budget spent", &shell, &repo)
15345 .await
15346 .expect("stop_reviewing");
15347 assert_eq!(runner.state.status, RunStatus::Gating);
15348 assert_eq!(
15349 runner.state.contested_handoff.is_some(),
15350 expect,
15351 "{severity:?} + {vote:?}"
15352 );
15353 }
15354 }
15355
15356 #[tokio::test]
15369 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
15370 crate::run::pin_test_home();
15371 let home = crate::run::home();
15372
15373 let tmp = tempfile::tempdir().expect("tempdir");
15374 let repo = tmp.path().join("repo");
15375 std::fs::create_dir_all(&repo).unwrap();
15376 init_repo(&repo);
15377 let head = crate::git::rev_parse(&repo, "HEAD")
15378 .await
15379 .expect("rev-parse");
15380 let cache_dir = tmp.path().join("target");
15383
15384 let mut config = Config::default();
15385 config.verify.e2e = vec![format!(
15386 "CARGO_TARGET_DIR='{}' test -f README.md",
15387 cache_dir.display()
15388 )];
15389 config.graph.review_rounds = 1;
15390 config.graph.timeout_verify = Some(2);
15393 config.graph.timeout_resource_wait = 2;
15394
15395 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15396 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15397 .expect("no io error acquiring directly")
15398 {
15399 crate::cache::AcquireOutcome::Acquired(g) => g,
15400 crate::cache::AcquireOutcome::Busy(b) => {
15401 panic!("expected the direct acquire to win the lease first: {b:?}")
15402 }
15403 };
15404
15405 let mut state = RunState::new(
15406 repo.clone(),
15407 "main".to_owned(),
15408 head.clone(),
15409 "task".to_owned(),
15410 config,
15411 );
15412 state.candidates = vec![Candidate {
15413 index: 0,
15414 label: 'A',
15415 agent: "alpha".to_owned(),
15416 branch: "does-not-exist".to_owned(),
15417 worktree: repo.clone(),
15418 summary: String::new(),
15419 stat: String::new(),
15420 files: 0,
15421 commits: 0,
15422 empty: false,
15423 failed: None,
15424 verified_noop: None,
15425 duration_ms: 0,
15426 folded: false,
15427 }];
15428 state.tally = Some(Tally {
15429 first_choice: BTreeMap::from([('A', 1)]),
15430 borda: BTreeMap::new(),
15431 winner: 'A',
15432 rankings: 1,
15433 unanimous_initial: true,
15434 deliberated: false,
15435 changed_votes: 0,
15436 unanimous_final: true,
15437 tie_break: None,
15438 judges: 0,
15439 present: 0,
15440 quorum: 0,
15441 met_quorum: true,
15442 uncontested: Some("only candidate A produced a change".to_owned()),
15443 });
15444 state.reviews = vec![ReviewRound {
15448 round: 1,
15449 head: head.clone(),
15450 verified_head: None,
15451 verified_at: None,
15452 reviews: Vec::new(),
15453 e2e: Vec::new(),
15454 fix: None,
15455 blocking: 1,
15456 answered: 1,
15457 expected: 1,
15458 clean: false,
15459 verify_retried: false,
15460 e2e_deferred: true,
15461 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
15462 progressed: false,
15463 vote_split: false,
15464 reconsideration: Vec::new(),
15465 verdict: None,
15466 }];
15467
15468 let mut runner = Runner {
15469 state,
15470 roles: ResolvedRoles {
15471 implementers: Vec::new(),
15472 judges: Vec::new(),
15473 reviewers: Vec::new(),
15474 fixer: None,
15475 conductor: conductor(),
15476 implementer_roster: Vec::new(),
15477 judge_roster: Vec::new(),
15478 reviewer_roster: Vec::new(),
15479 },
15480 sem: Arc::new(Semaphore::new(1)),
15481 pause: Pause::new(),
15482 interrupt: Pause::new(),
15483 };
15484
15485 let shell = runner.state.config.shell();
15486 runner
15487 .stop_reviewing("round budget spent", &shell, &repo)
15488 .await
15489 .expect("stop_reviewing");
15490
15491 let last = runner.state.reviews.last().expect("round record");
15492 assert_eq!(
15493 last.e2e_status(),
15494 E2eStatus::ResourceBlocked,
15495 "the shared cache is still held; the attempt must read as blocked, not deferred or \
15496 failed: {last:?}"
15497 );
15498 assert_eq!(
15499 last.verified_head.as_deref(),
15500 Some(head.as_str()),
15501 "which commit this attempt targeted is known even though nothing finished checking \
15502 it"
15503 );
15504 let first_attempt_at = last
15505 .verified_at
15506 .expect("when this attempt ran is known too");
15507 assert_ne!(
15508 runner.state.status,
15509 RunStatus::Blocked,
15510 "contention is evidence about the machine, not the patch — it must not settle the \
15511 run as blocked: {:?}",
15512 runner.state.status
15513 );
15514 assert!(
15515 !runner
15516 .state
15517 .events
15518 .iter()
15519 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
15520 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
15521 runner.state.events
15522 );
15523
15524 runner
15529 .stop_reviewing("round budget spent", &shell, &repo)
15530 .await
15531 .expect("stop_reviewing retry");
15532 assert_eq!(
15533 runner.state.reviews.len(),
15534 1,
15535 "no new round was started: {:?}",
15536 runner.state.reviews
15537 );
15538 let last = runner.state.reviews.last().expect("round record");
15539 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
15540 assert!(
15541 last.verified_at.expect("still known") > first_attempt_at,
15542 "a second reentry must be a fresh attempt, not a stale copy of the first"
15543 );
15544 assert_ne!(runner.state.status, RunStatus::Blocked);
15545
15546 held.release();
15547 }
15548
15549 #[tokio::test]
15561 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
15562 crate::run::pin_test_home();
15563 let home = crate::run::home();
15564
15565 let tmp = tempfile::tempdir().expect("tempdir");
15566 let repo = tmp.path().join("repo");
15567 std::fs::create_dir_all(&repo).unwrap();
15568 init_repo(&repo);
15569 let head = crate::git::rev_parse(&repo, "HEAD")
15570 .await
15571 .expect("rev-parse");
15572 let cache_dir = tmp.path().join("target");
15573
15574 let mut config = Config::default();
15575 config.verify.e2e = vec![format!(
15576 "CARGO_TARGET_DIR='{}' test -f README.md",
15577 cache_dir.display()
15578 )];
15579 config.graph.review_rounds = 1;
15580 config.graph.timeout_verify = Some(2);
15581 config.graph.timeout_resource_wait = 2;
15582
15583 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15584 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15585 .expect("no io error acquiring directly")
15586 {
15587 crate::cache::AcquireOutcome::Acquired(g) => g,
15588 crate::cache::AcquireOutcome::Busy(b) => {
15589 panic!("expected the direct acquire to win the lease first: {b:?}")
15590 }
15591 };
15592
15593 let mut state = RunState::new(
15594 repo.clone(),
15595 "main".to_owned(),
15596 head.clone(),
15597 "task".to_owned(),
15598 config,
15599 );
15600 state.candidates = vec![Candidate {
15601 index: 0,
15602 label: 'A',
15603 agent: "alpha".to_owned(),
15604 branch: "does-not-exist".to_owned(),
15605 worktree: repo.clone(),
15606 summary: String::new(),
15607 stat: String::new(),
15608 files: 0,
15609 commits: 0,
15610 empty: false,
15611 failed: None,
15612 verified_noop: None,
15613 duration_ms: 0,
15614 folded: false,
15615 }];
15616 state.tally = Some(Tally {
15617 first_choice: BTreeMap::from([('A', 1)]),
15618 borda: BTreeMap::new(),
15619 winner: 'A',
15620 rankings: 1,
15621 unanimous_initial: true,
15622 deliberated: false,
15623 changed_votes: 0,
15624 unanimous_final: true,
15625 tie_break: None,
15626 judges: 0,
15627 present: 0,
15628 quorum: 0,
15629 met_quorum: true,
15630 uncontested: Some("only candidate A produced a change".to_owned()),
15631 });
15632 state.reviews = vec![ReviewRound {
15636 round: 1,
15637 head: head.clone(),
15638 verified_head: Some(head.clone()),
15639 verified_at: Some(jiff::Timestamp::now()),
15640 reviews: Vec::new(),
15641 e2e: vec![CommandOutcome {
15642 command: format!(
15643 "CARGO_TARGET_DIR='{}' test -f README.md",
15644 cache_dir.display()
15645 ),
15646 code: None,
15647 output_tail: "waiting for the shared build cache".to_owned(),
15648 duration_ms: 0,
15649 resource_blocked: true,
15650 }],
15651 fix: None,
15652 blocking: 1,
15653 answered: 1,
15654 expected: 1,
15655 clean: false,
15656 verify_retried: false,
15657 e2e_deferred: false,
15658 e2e_defer_reason: None,
15659 progressed: false,
15660 vote_split: false,
15661 reconsideration: Vec::new(),
15662 verdict: None,
15663 }];
15664
15665 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
15666 let mut runner = Runner {
15667 state,
15668 roles: ResolvedRoles {
15669 implementers: Vec::new(),
15670 judges: Vec::new(),
15671 reviewers: Vec::new(),
15672 fixer: None,
15673 conductor: conductor(),
15674 implementer_roster: Vec::new(),
15675 judge_roster: Vec::new(),
15676 reviewer_roster: Vec::new(),
15677 },
15678 sem: Arc::new(Semaphore::new(1)),
15679 pause: Pause::new(),
15680 interrupt: Pause::new(),
15681 };
15682
15683 runner.review_loop().await.expect("review_loop");
15688
15689 assert_eq!(
15690 runner.state.reviews.len(),
15691 1,
15692 "no new round was started on top of the unresolved one: {:?}",
15693 runner.state.reviews
15694 );
15695 let last = &runner.state.reviews[0];
15696 assert_eq!(
15697 last.e2e_status(),
15698 E2eStatus::ResourceBlocked,
15699 "still contended: {last:?}"
15700 );
15701 assert!(
15702 last.verified_at.expect("still known") > first_attempt_at,
15703 "review_loop must have actually retried the check, not left it exactly as found"
15704 );
15705 assert_ne!(
15706 runner.state.status,
15707 RunStatus::Blocked,
15708 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
15709 runner.state.status
15710 );
15711
15712 held.release();
15713 }
15714
15715 #[tokio::test]
15716 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
15717 crate::run::pin_test_home();
15718 let tmp = tempfile::tempdir().expect("tempdir");
15719 let repo = tmp.path().join("repo");
15720 std::fs::create_dir_all(&repo).unwrap();
15721 init_repo(&repo);
15722
15723 let mut config = Config::default();
15724 config.merge.mode = MergeMode::Pr;
15725 config.graph.land = true;
15726 config.graph.land_approval = false;
15727
15728 let mut state = RunState::new(
15729 repo.clone(),
15730 "main".to_owned(),
15731 "deadbeef".to_owned(),
15732 "task".to_owned(),
15733 config,
15734 );
15735 state.candidates = vec![Candidate {
15736 index: 0,
15737 label: 'A',
15738 agent: "alpha".to_owned(),
15739 branch: "does-not-exist".to_owned(),
15740 worktree: repo.clone(),
15741 summary: String::new(),
15742 stat: String::new(),
15743 files: 0,
15744 commits: 0,
15745 empty: false,
15746 failed: None,
15747 verified_noop: None,
15748 duration_ms: 0,
15749 folded: false,
15750 }];
15751 state.tally = Some(Tally {
15752 first_choice: BTreeMap::from([('A', 1)]),
15753 borda: BTreeMap::new(),
15754 winner: 'A',
15755 rankings: 1,
15756 unanimous_initial: true,
15757 deliberated: false,
15758 changed_votes: 0,
15759 unanimous_final: true,
15760 tie_break: None,
15761 judges: 0,
15762 present: 0,
15763 quorum: 0,
15764 met_quorum: true,
15765 uncontested: Some("only candidate A produced a change".to_owned()),
15766 });
15767 state.reviews = vec![ReviewRound {
15768 round: 1,
15769 head: "deadbeef".to_owned(),
15770 verified_head: None,
15771 verified_at: None,
15772 reviews: Vec::new(),
15773 e2e: Vec::new(),
15774 fix: None,
15775 blocking: 0,
15776 answered: 0,
15777 expected: 0,
15778 clean: true,
15779 verify_retried: false,
15780 e2e_deferred: false,
15781 e2e_defer_reason: None,
15782 progressed: false,
15783 vote_split: false,
15784 reconsideration: Vec::new(),
15785 verdict: None,
15786 }];
15787 state.gate = vec![CommandOutcome {
15788 command: "test".to_owned(),
15789 code: Some(0),
15790 output_tail: String::new(),
15791 duration_ms: 0,
15792 resource_blocked: false,
15793 }];
15794 state.gate_ran = true;
15795 state.status = RunStatus::Landing;
15799 state.merge = Some(MergeOutcome {
15800 mode: MergeMode::Pr,
15801 ok: true,
15802 detail: "https://example.invalid/x/y/pull/1".to_owned(),
15803 empty: false,
15804 });
15805
15806 ask_test_home();
15810 state.save().expect("save landing run");
15813 let store = ask::Questions::open();
15814 let q = ask_open_question(&store, &state.id);
15815
15816 let mut runner = Runner {
15817 state,
15818 roles: ResolvedRoles {
15819 implementers: Vec::new(),
15820 judges: Vec::new(),
15821 reviewers: Vec::new(),
15822 fixer: None,
15823 conductor: conductor(),
15824 implementer_roster: Vec::new(),
15825 judge_roster: Vec::new(),
15826 reviewer_roster: Vec::new(),
15827 },
15828 sem: Arc::new(Semaphore::new(1)),
15829 pause: Pause::new(),
15830 interrupt: Pause::new(),
15831 };
15832
15833 runner.execute().await.expect("execute");
15838
15839 assert_eq!(
15840 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15841 Some("https://example.invalid/x/y/pull/1"),
15842 "reentry must not push again or open a second pull request over the \
15843 one `land` is already watching"
15844 );
15845 assert_ne!(
15846 runner.state.status,
15847 RunStatus::Landing,
15848 "land could not actually reach the fake pull request, so it must \
15849 have given up rather than left the run silently parked forever"
15850 );
15851 assert_eq!(runner.state.status, RunStatus::Blocked);
15855 assert!(
15856 store.get(&q.id).unwrap().status.open(),
15857 "Blocked is still alive; settle_questions must have been a no-op here"
15858 );
15859 }
15860
15861 fn state_with_round(round: ReviewRound) -> RunState {
15862 let mut s = RunState::new(
15863 PathBuf::from("/repo"),
15864 "main".to_owned(),
15865 "abc1234".to_owned(),
15866 "add retries".to_owned(),
15867 Config::default(),
15868 );
15869 s.reviews = vec![round];
15870 s
15871 }
15872
15873 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
15874 crate::verdict::Finding {
15875 id: id.to_owned(),
15876 severity,
15877 file: None,
15878 line: None,
15879 title: title.to_owned(),
15880 detail: String::new(),
15881 }
15882 }
15883
15884 #[test]
15885 fn pr_body_names_open_findings_and_declined_ones() {
15886 let round = ReviewRound {
15887 round: 2,
15888 head: "deadbee".to_owned(),
15889 verified_head: None,
15890 verified_at: None,
15891 reviews: vec![ReviewRecord {
15892 attempts: 0,
15893 reviewer: 1,
15894 agent: "alpha".to_owned(),
15895 summary: String::new(),
15896 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
15897 vote: None,
15898 failed: None,
15899 duration_ms: 0,
15900 }],
15901 e2e: vec![CommandOutcome {
15902 command: "cargo test".to_owned(),
15903 code: Some(0),
15904 output_tail: String::new(),
15905 duration_ms: 0,
15906 resource_blocked: false,
15907 }],
15908 verify_retried: false,
15909 e2e_deferred: false,
15910 e2e_defer_reason: None,
15911 fix: Some(FixRecord {
15912 agent: "alpha".to_owned(),
15913 addressed: Vec::new(),
15914 rejected: vec![crate::verdict::Rejection {
15915 id: "R1-1-1".to_owned(),
15916 why: "not reachable from any caller".to_owned(),
15917 }],
15918 notes: String::new(),
15919 committed: true,
15920 failed: None,
15921 duration_ms: 0,
15922 continuation: None,
15923 }),
15924 blocking: 0,
15925 answered: 1,
15926 expected: 1,
15927 clean: false,
15928 progressed: true,
15929 vote_split: false,
15930 reconsideration: Vec::new(),
15931 verdict: None,
15932 };
15933 let state = state_with_round(round);
15934 let body = pr_message(&state, 'A').body;
15935
15936 assert!(body.contains("add retries"), "the task must still be there");
15937 assert!(body.contains("R2-1-1"), "{body}");
15938 assert!(!body.contains("unused import"), "{body}");
15939 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
15940 assert!(
15941 !body.contains("not reachable from any caller"),
15942 "the reason it was declined: {body}"
15943 );
15944 }
15945
15946 #[test]
15947 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
15948 let round = ReviewRound {
15949 round: 1,
15950 head: "deadbee".to_owned(),
15951 verified_head: None,
15952 verified_at: None,
15953 reviews: vec![ReviewRecord {
15954 attempts: 0,
15955 reviewer: 1,
15956 agent: "alpha".to_owned(),
15957 summary: String::new(),
15958 findings: Vec::new(),
15959 vote: None,
15960 failed: None,
15961 duration_ms: 0,
15962 }],
15963 e2e: Vec::new(),
15964 verify_retried: false,
15965 e2e_deferred: false,
15966 e2e_defer_reason: None,
15967 fix: None,
15968 blocking: 0,
15969 answered: 1,
15970 expected: 1,
15971 clean: true,
15972 progressed: false,
15973 vote_split: false,
15974 reconsideration: Vec::new(),
15975 verdict: None,
15976 };
15977 let state = state_with_round(round);
15978 let body = pr_message(&state, 'A').body;
15979 assert!(!body.contains("Open review findings"), "{body}");
15980 assert!(!body.contains("Declined"), "{body}");
15981 }
15982
15983 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
15984 let mut state = RunState::new(
15985 PathBuf::from("/repo"),
15986 "main".to_owned(),
15987 "abc1234".to_owned(),
15988 instruction.to_owned(),
15989 Config::default(),
15990 );
15991 state.candidates.push(Candidate {
15992 index: 0,
15993 label: 'A',
15994 agent: "alpha".to_owned(),
15995 branch: "magi/x/A".to_owned(),
15996 worktree: PathBuf::from("/wt"),
15997 summary: summary.to_owned(),
15998 stat: String::new(),
15999 files: 1,
16000 commits: 1,
16001 empty: false,
16002 failed: None,
16003 verified_noop: None,
16004 folded: false,
16005 duration_ms: 0,
16006 });
16007 state
16008 }
16009
16010 fn review_state(subjects: &[&str]) -> RunState {
16011 let mut state = state_with_summary(
16012 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16013 "",
16014 );
16015 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16016 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16017 state
16018 }
16019
16020 #[tokio::test]
16023 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16024 ask_test_home();
16025 let tmp = tempfile::tempdir().unwrap();
16026 let repo = tmp.path().join("repo");
16027 std::fs::create_dir_all(&repo).unwrap();
16028 init_repo(&repo);
16029 let origin = tmp.path().join("origin.git");
16030 let g = |dir: &Path, args: &[&str]| {
16031 let out = std::process::Command::new("git")
16032 .args(args)
16033 .current_dir(dir)
16034 .quiet()
16035 .output()
16036 .expect("spawn git");
16037 assert!(
16038 out.status.success(),
16039 "git {args:?}: {}",
16040 String::from_utf8_lossy(&out.stderr)
16041 );
16042 };
16043 g(
16044 tmp.path(),
16045 &[
16046 "clone",
16047 "--bare",
16048 "-q",
16049 repo.to_str().unwrap(),
16050 origin.to_str().unwrap(),
16051 ],
16052 );
16053 g(
16054 &repo,
16055 &["remote", "add", "origin", origin.to_str().unwrap()],
16056 );
16057 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16058
16059 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16061 g(&repo, &["add", "-A"]);
16062 g(
16063 &repo,
16064 &["commit", "-q", "-m", "chore(deps): update a crate"],
16065 );
16066 g(&repo, &["push", "-q", "origin", "main"]);
16067 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16068 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16069 g(&repo, &["add", "-A"]);
16070 g(
16071 &repo,
16072 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16073 );
16074 g(&repo, &["checkout", "-q", "main"]);
16075
16076 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16077 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16078 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16080 assert_eq!(fallback, c1);
16081
16082 let mut state = review_state(&[
16083 "chore(deps): update a crate",
16084 "fix(daemon): apply a chosen action",
16085 ]);
16086 state.repo = repo.clone();
16087 state.base_branch = "main".to_owned();
16088 state.base_commit = c1;
16089 refresh_reviewed_commits(&mut state, "feat/own").await;
16090 assert_eq!(
16091 state.reviewed_commits,
16092 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16093 );
16094 assert_eq!(
16095 review_title(&state).as_deref(),
16096 Some("fix(daemon): apply a chosen action")
16097 );
16098 assert_eq!(
16099 leaked_subjects(&state, "feat/own").await,
16100 Some(vec!["chore(deps): update a crate".to_owned()])
16101 );
16102 state.config.merge.remote = "nowhere".to_owned();
16105 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16106 }
16107
16108 struct SummaryFixture {
16112 _tmp: tempfile::TempDir,
16113 repo: PathBuf,
16114 reply: PathBuf,
16115 runner: Runner,
16116 }
16117
16118 fn summary_fixture() -> SummaryFixture {
16119 ask_test_home();
16120 let tmp = tempfile::tempdir().unwrap();
16121 let repo = tmp.path().join("repo");
16122 std::fs::create_dir_all(&repo).unwrap();
16123 init_repo(&repo);
16124 let origin = tmp.path().join("origin.git");
16125 let g = |args: &[&str]| {
16126 let out = std::process::Command::new("git")
16127 .args(args)
16128 .current_dir(&repo)
16129 .quiet()
16130 .output()
16131 .expect("spawn git");
16132 assert!(
16133 out.status.success(),
16134 "git {args:?}: {}",
16135 String::from_utf8_lossy(&out.stderr)
16136 );
16137 };
16138 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16139 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16140 g(&["push", "-q", "origin", "main"]);
16141 g(&["checkout", "-q", "-b", "feat/own"]);
16142 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16143 g(&["add", "-A"]);
16144 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16145 let reply = tmp.path().join("reply.txt");
16146 std::fs::write(
16147 &reply,
16148 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16149 )
16150 .unwrap();
16151 let mut runner = runner_at(RunStatus::Gating);
16152 let mut state = review_state(&["feat(own): add the own file"]);
16153 state.repo = repo.clone();
16154 state.base_branch = "main".to_owned();
16155 state.candidates[0].branch = "feat/own".to_owned();
16156 state.candidates[0].worktree = repo.clone();
16157 let mut author = spec("alpha");
16158 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16159 author
16160 .env
16161 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16162 state.config.agents = vec![author];
16163 runner.state = state;
16164 SummaryFixture {
16165 _tmp: tmp,
16166 repo,
16167 reply,
16168 runner,
16169 }
16170 }
16171
16172 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16173 let winner = f.runner.state.candidates[0].clone();
16174 f.runner
16175 .guarded_pr_message(&winner, None, true)
16176 .await
16177 .unwrap()
16178 .expect("an English description never parks")
16179 }
16180
16181 fn clean_round_with_minor_findings() -> ReviewRound {
16182 let mut round = passing_round(2, None);
16183 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16184 round.reviews = vec![ReviewRecord {
16185 attempts: 0,
16186 reviewer: 1,
16187 agent: "alpha".into(),
16188 summary: "内部レビュー".into(),
16189 findings: vec![
16190 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16191 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16192 ],
16193 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16194 failed: None,
16195 duration_ms: 0,
16196 }];
16197 round
16198 }
16199
16200 #[tokio::test]
16201 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16202 for review_only in [false, true] {
16203 let mut f = summary_fixture();
16204 if !review_only {
16205 f.runner.state.candidates[0].agent = "alpha".into();
16206 }
16207 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16208 assert!(f.runner.state.reviews[0].clean);
16209 assert!(f.runner.state.reviews[0].fix.is_none());
16210 assert!(f.runner.state.open_findings().is_empty());
16211 assert_eq!(
16212 crate::pr_outcome::required_finding_ids(&f.runner.state),
16213 ["R2-1-1", "R2-1-2"]
16214 );
16215 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16216 let evidence = crate::pr_outcome::summary_evidence(
16217 &f.runner.state,
16218 crate::pr_outcome::FinalHead::At(&head),
16219 );
16220 assert!(
16221 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
16222 );
16223 let omitted = posted(&mut f).await;
16225 assert!(omitted.body.contains("No description"));
16226 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
16227 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
16228 assert!(f.runner.state.change_summary.is_none());
16229 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
16230 let described = posted(&mut f).await;
16231 assert!(described.body.contains("improve documentation for callers"));
16232 assert!(described.body.contains("remaining edge case"));
16233 assert!(described.body.contains("## Open review findings"));
16234 assert!(!described.body.contains("未修正"));
16235 }
16236 }
16237
16238 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
16239 let mut f = summary_fixture();
16240 if !review_only {
16241 f.runner.state.candidates[0].agent = "alpha".into();
16242 }
16243 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16244 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
16245 fixed
16246 .fix
16247 .as_mut()
16248 .unwrap()
16249 .rejected
16250 .push(crate::verdict::Rejection {
16251 id: "R1-1-1".into(),
16252 why: "呼び出されないコードのため辞退".into(),
16253 });
16254 let mut clean = clean_round_with_minor_findings();
16255 clean.verified_head = Some(head.clone());
16256 f.runner.state.reviews = vec![fixed, clean];
16257 ok_gate(&mut f.runner.state, Some(&head));
16258 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
16259 f.runner.state.config.agents.push(scripted("beta", reply));
16260 f.runner.state.config.roles.language_judge =
16261 Some(crate::config::AgentChoice::One("beta".into()));
16262 f
16263 }
16264
16265 #[tokio::test]
16266 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
16267 for review_only in [false, true] {
16268 let mut f = rewrite_fixture_with_findings(review_only,
16269 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
16270 let m = posted(&mut f).await;
16271 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16272 let expected =
16273 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
16274 assert!(m.body.contains(&expected));
16275 assert!(m.body.contains("## Review rounds"));
16276 assert!(
16277 m.body
16278 .contains("Gate (`verify.gate`): passed on the final commit")
16279 );
16280 assert!(
16281 m.body
16282 .contains("Tests (`verify.e2e`): passed on the final commit")
16283 );
16284 let c = f.runner.state.change_summary.as_ref().unwrap();
16285 assert!(
16286 m.body
16287 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
16288 );
16289 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16290 assert!(crate::pr_outcome::covers_required_findings(
16291 &f.runner.state,
16292 &m.body
16293 ));
16294 assert!(m.body.contains("documentation remains incomplete"));
16295 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
16296 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
16297 assert!(f.runner.state.github_text.is_none());
16298 }
16299 }
16300
16301 #[tokio::test]
16302 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
16303 for review_only in [false, true] {
16304 for reply in [
16305 "TITLE: fix: public description\nA short English description.",
16306 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
16307 ] {
16308 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
16309 let winner = f.runner.state.candidates[0].clone();
16310 let posted = f
16311 .runner
16312 .guarded_pr_message(&winner, None, true)
16313 .await
16314 .unwrap();
16315 assert!(posted.is_none());
16316 assert!(f.runner.state.parked);
16317 let withheld = f.runner.state.github_text.as_ref().unwrap();
16318 assert!(
16319 withheld.rewrite.is_none(),
16320 "an incomplete rewrite must not be selectable"
16321 );
16322 let q = ask::Questions::open()
16323 .get(withheld.question.as_ref().unwrap())
16324 .unwrap();
16325 assert!(
16326 !q.choices
16327 .contains(&crate::github_text::USE_REWRITE.to_owned())
16328 );
16329 assert!(
16330 q.detail.contains("R1-1-1")
16331 && q.detail.contains("R2-1-1")
16332 && q.detail.contains("R2-1-2")
16333 );
16334 }
16335 }
16336 }
16337
16338 #[tokio::test]
16339 async fn japanese_reports_stay_local_in_both_public_run_modes() {
16340 for review_only in [false, true] {
16341 let mut f = summary_fixture();
16342 if !review_only {
16343 f.runner.state.candidates[0].agent = "alpha".into();
16344 }
16345 f.runner.state.instruction = "内部タスクの説明".into();
16346 f.runner.state.candidates[0].summary = "内部の候補報告".into();
16347 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16348 f.runner.state.reviews = vec![
16349 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
16350 round_with_notes(2, Some("二回目の内部修正報告です。")),
16351 passing_round(3, Some(&head)),
16352 ];
16353 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
16354 ok_gate(&mut f.runner.state, Some(&head));
16355 let captured = f.repo.join("public-prompt.txt");
16356 let author = &mut f.runner.state.config.agents[0];
16357 author.command = vec![
16358 "sh".into(),
16359 "-c".into(),
16360 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16361 ];
16362 author.env.insert(
16363 "PROMPT_FILE".into(),
16364 captured.to_string_lossy().into_owned(),
16365 );
16366 let m = posted(&mut f).await;
16367 assert!(m.body.contains("Adds `own.txt`"));
16368 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
16369 assert!(m.body.contains("R1-1-1"));
16370 assert!(
16371 m.body
16372 .contains("Gate (`verify.gate`): passed on the final commit")
16373 );
16374 assert!(
16375 m.body
16376 .contains("Tests (`verify.e2e`): passed on the final commit")
16377 );
16378 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16379 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
16380 let prompt = std::fs::read_to_string(captured).unwrap();
16381 assert!(prompt.contains(&format!(
16382 "git diff {} {head}",
16383 f.runner.state.change_summary.as_ref().unwrap().base
16384 )));
16385 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
16386 assert_eq!(
16387 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
16388 "二回目の内部修正報告です。"
16389 );
16390 }
16391 }
16392
16393 #[tokio::test]
16394 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
16395 for review_only in [false, true] {
16396 let mut f = summary_fixture();
16397 if !review_only {
16398 f.runner.state.candidates[0].agent = "alpha".into();
16399 }
16400 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
16401 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
16402 std::fs::write(&f.reply, "no usable description").unwrap();
16403 let failed = posted(&mut f).await;
16404 assert!(
16405 failed
16406 .body
16407 .contains("No description of what this change does could be generated")
16408 );
16409 assert!(failed.body.is_ascii());
16410 assert!(f.runner.state.github_text.is_none());
16411 assert!(f.runner.state.change_summary.is_none());
16412 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16413 let evidence = crate::pr_outcome::summary_evidence(
16414 &f.runner.state,
16415 crate::pr_outcome::FinalHead::At(&head),
16416 );
16417 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
16418 assert!(
16419 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
16420 .exists()
16421 );
16422 f.runner.state =
16423 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
16424 std::fs::write(
16425 &f.reply,
16426 "TITLE: feat(own): add the own file\nAdds the committed own file.",
16427 )
16428 .unwrap();
16429 let retried = posted(&mut f).await;
16430 assert!(retried.body.contains("Adds the committed own file."));
16431 assert_eq!(
16432 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
16433 "日本語の修正報告"
16434 );
16435 }
16436 }
16437
16438 #[tokio::test]
16439 async fn same_head_verification_and_findings_invalidate_public_prose() {
16440 let mut f = summary_fixture();
16441 posted(&mut f).await;
16442 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16443 let original = f
16444 .runner
16445 .state
16446 .change_summary
16447 .as_ref()
16448 .unwrap()
16449 .evidence
16450 .clone();
16451 ok_gate(&mut f.runner.state, Some(&head));
16452 std::fs::write(
16453 &f.reply,
16454 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
16455 )
16456 .unwrap();
16457 let verified = posted(&mut f).await;
16458 assert!(verified.body.contains("now verified"));
16459 assert_ne!(
16460 f.runner.state.change_summary.as_ref().unwrap().evidence,
16461 original
16462 );
16463 let mut round = round_with_notes(1, Some("内部報告"));
16464 round.reviews.push(ReviewRecord {
16465 attempts: 0,
16466 reviewer: 1,
16467 agent: "alpha".into(),
16468 summary: "内部レビュー".into(),
16469 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
16470 vote: None,
16471 failed: None,
16472 duration_ms: 0,
16473 });
16474 round
16475 .fix
16476 .as_mut()
16477 .unwrap()
16478 .rejected
16479 .push(crate::verdict::Rejection {
16480 id: "R1-1-1".into(),
16481 why: "呼び出し元から到達できない".into(),
16482 });
16483 f.runner.state.reviews = vec![round];
16484 let omitted = posted(&mut f).await;
16486 assert!(omitted.body.contains("No description"));
16487 assert!(omitted.body.contains("R1-1-1"));
16488 assert!(!omitted.body.contains("到達"));
16489 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
16490 let described = posted(&mut f).await;
16491 assert!(described.body.contains("unreachable from callers"));
16492 assert!(described.body.contains("R1-1-1"));
16493 assert!(!described.body.contains("到達"));
16494 let evidence = f
16495 .runner
16496 .state
16497 .change_summary
16498 .as_ref()
16499 .unwrap()
16500 .evidence
16501 .clone();
16502 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
16503 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
16504 assert!(posted(&mut f).await.body.contains("rationale updated"));
16505 assert_ne!(
16506 f.runner.state.change_summary.as_ref().unwrap().evidence,
16507 evidence
16508 );
16509 f.runner
16511 .state
16512 .change_summary
16513 .as_mut()
16514 .unwrap()
16515 .evidence
16516 .clear();
16517 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
16518 assert!(posted(&mut f).await.body.contains("current specification"));
16519 }
16520
16521 #[tokio::test]
16522 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
16523 let mut f = summary_fixture();
16524 let m = posted(&mut f).await;
16525 assert_eq!(m.title, "feat(own): add the own file");
16526 assert!(
16527 m.body
16528 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
16529 "{}",
16530 m.body
16531 );
16532 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
16533 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
16534 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
16535 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16536 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
16537 let kept = f.runner.state.change_summary.as_ref().unwrap();
16538 assert_eq!(
16539 (kept.head.as_str(), kept.base.as_str()),
16540 (head.as_str(), base.as_str())
16541 );
16542 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
16543 }
16544
16545 #[tokio::test]
16546 async fn review_only_summary_receives_the_configured_github_language() {
16547 let mut f = summary_fixture();
16548 f.runner.state.config.graph.github_language = "ja".into();
16549 let captured = f.repo.join("summary-prompt.txt");
16550 let author = &mut f.runner.state.config.agents[0];
16551 author.command = vec![
16552 "sh".into(),
16553 "-c".into(),
16554 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
16555 ];
16556 author.env.insert(
16557 "PROMPT_FILE".into(),
16558 captured.to_string_lossy().into_owned(),
16559 );
16560 std::fs::write(
16561 &f.reply,
16562 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
16563 )
16564 .unwrap();
16565 let m = posted(&mut f).await;
16566 let prompt = std::fs::read_to_string(captured).unwrap();
16567 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
16568 assert!(!prompt.contains("in English"));
16569 assert_eq!(m.title, "feat(own): ファイルを追加");
16570 assert!(m.body.contains("own.txt を追加します。"));
16571 }
16572
16573 #[tokio::test]
16574 async fn review_fixes_come_after_the_whole_change_summary() {
16575 let mut f = summary_fixture();
16576 f.runner.state.reviews = vec![
16577 round_with_notes(1, Some("handled the PATH case")),
16578 round_with_notes(2, None),
16579 ];
16580 let m = posted(&mut f).await;
16581 let summary = m.body.find("## Summary").expect("summary");
16582 let fixes = m.body.find("## Review rounds").expect("review rounds");
16583 assert!(summary < fixes, "{}", m.body);
16584 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
16585 assert!(m.body.contains("## Final verification"), "{}", m.body);
16586 }
16587
16588 #[tokio::test]
16589 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
16590 let mut f = summary_fixture();
16591 let first = posted(&mut f).await;
16592 let first_head = f.runner.state.change_summary.clone().unwrap().head;
16593 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
16595 let again = posted(&mut f).await;
16596 assert_eq!((again.title, again.body), (first.title, first.body));
16597
16598 let g = |args: &[&str]| {
16599 let out = std::process::Command::new("git")
16600 .args(args)
16601 .current_dir(&f.repo)
16602 .quiet()
16603 .output()
16604 .expect("spawn git");
16605 assert!(out.status.success(), "git {args:?}");
16606 };
16607 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
16608 g(&["add", "-A"]);
16609 g(&["commit", "-q", "-m", "fix(own): handle more"]);
16610 let second = posted(&mut f).await;
16611 let kept = f.runner.state.change_summary.clone().unwrap();
16612 assert_ne!(kept.head, first_head);
16613 assert!(second.body.contains("Not read yet."), "{}", second.body);
16614 assert!(!second.body.contains("so the first summary is read"));
16615 assert_eq!(second.title, "feat(own): other");
16616 }
16617
16618 #[tokio::test]
16619 async fn an_unavailable_summary_is_stated_and_never_invented() {
16620 let mut f = summary_fixture();
16621 std::fs::write(&f.reply, "no title line here").unwrap();
16622 let m = posted(&mut f).await;
16623 assert!(
16624 m.body
16625 .contains("No description of what this change does could be generated"),
16626 "{}",
16627 m.body
16628 );
16629 assert!(m.body.contains("internal reports remain in local records"));
16630 assert!(f.runner.state.change_summary.is_none());
16631 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
16632 let winner = f.runner.state.candidates[0].clone();
16634 let preview = f
16635 .runner
16636 .guarded_pr_message(&winner, None, false)
16637 .await
16638 .unwrap()
16639 .unwrap();
16640 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
16641 }
16642
16643 #[test]
16644 fn pr_message_review_single_commit_uses_its_subject() {
16645 let state = review_state(&["feat(nats): per-role user"]);
16646 let m = pr_message(&state, 'A');
16647 assert_eq!(m.title, "feat(nats): per-role user");
16648 assert!(!m.body.contains("Review the work already"), "{}", m.body);
16649 assert!(m.body.contains("## Commits under review"), "{}", m.body);
16650 }
16651
16652 #[test]
16653 fn pr_message_review_multi_commit_takes_the_oldest() {
16654 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
16655 let m = pr_message(&state, 'A');
16656 assert_eq!(m.title, "feat: the change");
16657 for s in ["feat: the change", "fix: typo", "fix: again"] {
16658 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
16659 }
16660 }
16661
16662 #[test]
16663 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
16664 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
16665 let mut state = review_state(&[first, "fix: later fixup"]);
16666 state.candidates[0].branch = "機能/ログイン".to_owned();
16667 let m = pr_message(&state, 'A');
16668 assert!(
16669 m.title.starts_with("chore: land candidate A of run"),
16670 "{}",
16671 m.title
16672 );
16673 }
16674 }
16675
16676 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
16677 BranchFacts {
16678 commits: commits
16679 .iter()
16680 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
16681 .collect(),
16682 stat: stat.to_owned(),
16683 }
16684 }
16685
16686 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
16687 let mut r = review_round(true, 0, 1, 1, true, true);
16688 r.round = round;
16689 r.fix = notes.map(|n| FixRecord {
16690 agent: "fixer".to_owned(),
16691 addressed: Vec::new(),
16692 rejected: Vec::new(),
16693 notes: n.to_owned(),
16694 committed: true,
16695 failed: None,
16696 duration_ms: 0,
16697 continuation: None,
16698 });
16699 r
16700 }
16701
16702 #[test]
16703 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
16704 let state = review_state(&["ignored"]);
16705 let f = facts(
16706 &[
16707 (
16708 "fix(login): resolve PATH on macOS",
16709 "Login shells skip rc files.",
16710 ),
16711 ("fix: address review", ""),
16712 ],
16713 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
16714 );
16715 let m = pr_message_with(&state, 'A', Some(&f));
16716 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
16717 assert!(
16718 m.body
16719 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
16720 "{}",
16721 m.body
16722 );
16723 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
16724 assert!(m.body.contains("## Diff stat"), "{}", m.body);
16725 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
16726 for banned in [
16727 "Review the work already",
16728 "no task statement",
16729 "Original task",
16730 ] {
16731 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
16732 }
16733 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
16734 }
16735
16736 #[test]
16737 fn pr_message_review_truncates_a_huge_first_commit_body() {
16738 let state = review_state(&["ignored"]);
16739 let f = facts(
16740 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
16741 "s",
16742 );
16743 let m = pr_message_with(&state, 'A', Some(&f));
16744 assert!(m.body.len() < 30_000, "{}", m.body.len());
16745 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
16746 assert!(
16747 m.body.contains("- ... 1 more commit(s)"),
16748 "{}",
16749 m.body.len()
16750 );
16751 assert!(m.body.ends_with("magi:candidate-a\n"));
16752 }
16753
16754 #[test]
16755 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
16756 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
16757 assert_eq!(m.title, "feat: x");
16758 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
16759 assert!(!m.body.contains("Diff stat"), "{}", m.body);
16760 assert!(!m.body.contains("no task statement"), "{}", m.body);
16761 }
16762
16763 #[test]
16764 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
16765 let mut state = review_state(&["日本語の件名"]);
16766 state.candidates[0].branch = "fix/macos-login-path".to_owned();
16767 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
16768 state.candidates[0].branch = "機能/ログイン".to_owned();
16769 assert!(
16770 pr_message(&state, 'A')
16771 .title
16772 .starts_with("chore: land candidate A")
16773 );
16774 }
16775
16776 #[test]
16777 fn pr_message_review_fixes_survive_a_clean_final_round() {
16778 let mut state = review_state(&["feat: x"]);
16779 state.reviews = vec![
16780 round_with_notes(1, Some("handled the PATH case")),
16781 round_with_notes(2, None),
16782 ];
16783 let body = pr_message(&state, 'A').body;
16784 assert!(
16785 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
16786 "{body}"
16787 );
16788 assert!(body.contains("### Round 1"), "{body}");
16789
16790 state.reviews = vec![
16791 round_with_notes(1, Some("first fix")),
16792 round_with_notes(2, Some("")),
16793 round_with_notes(3, Some("second fix")),
16794 round_with_notes(4, None),
16795 ];
16796 let body = pr_message(&state, 'A').body;
16797 assert!(
16798 body.contains("### Round 1") && !body.contains("first fix"),
16799 "{body}"
16800 );
16801 assert!(
16802 body.contains("### Round 3") && !body.contains("second fix"),
16803 "{body}"
16804 );
16805 assert!(body.contains("### Round 2"), "{body}");
16806 }
16807
16808 const FINAL: &str = "abcdef1234567890";
16809
16810 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
16811 let mut r = round_with_notes(round, None);
16812 r.verified_head = verified.map(str::to_owned);
16813 r
16814 }
16815
16816 fn ok_gate(state: &mut RunState, head: Option<&str>) {
16817 state.gate = vec![CommandOutcome {
16818 command: "gate".to_owned(),
16819 code: Some(0),
16820 output_tail: String::new(),
16821 duration_ms: 0,
16822 resource_blocked: false,
16823 }];
16824 state.gate_ran = true;
16825 state.gate_head = head.map(str::to_owned);
16826 }
16827
16828 fn body_at(state: &RunState) -> String {
16829 pr_message_raw(
16830 state,
16831 'A',
16832 None,
16833 crate::pr_outcome::FinalHead::At(FINAL),
16834 ChangeNote::NotAsked,
16835 )
16836 .body
16837 }
16838
16839 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
16840
16841 #[test]
16842 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
16843 let mut state = review_state(&["feat: x"]);
16844 state.reviews = vec![
16845 round_with_notes(1, Some(SANDBOXED)),
16846 passing_round(2, Some(FINAL)),
16847 ];
16848 ok_gate(&mut state, Some(FINAL));
16849 let body = body_at(&state);
16850 assert!(!body.contains("Handled the PATH case."), "{body}");
16851 assert!(!body.contains("could not commit"), "{body}");
16852 assert!(!body.contains("could not run"), "{body}");
16853 assert!(
16854 body.contains("The fix is committed on the branch"),
16855 "{body}"
16856 );
16857 assert!(!body.contains("sandbox"), "{body}");
16858 assert!(
16859 body.contains("Tests (`verify.e2e`): passed on the final commit"),
16860 "{body}"
16861 );
16862 assert!(
16863 body.contains("Gate (`verify.gate`): passed on the final commit"),
16864 "{body}"
16865 );
16866 assert!(
16868 state.reviews[0]
16869 .fix
16870 .as_ref()
16871 .unwrap()
16872 .notes
16873 .contains("could not commit")
16874 );
16875 }
16876
16877 #[test]
16878 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
16879 let mut state = review_state(&["feat: x"]);
16880 let mut r1 = round_with_notes(1, Some(SANDBOXED));
16881 r1.fix.as_mut().unwrap().committed = false;
16882 let mut r2 = passing_round(2, Some(FINAL));
16883 r2.e2e[0].code = Some(1);
16884 state.reviews = vec![r1, r2];
16885 let body = body_at(&state);
16886 assert!(!body.contains("could not commit"), "{body}");
16887 assert!(!body.contains("could not run"), "{body}");
16888 assert!(body.contains("This round produced no commit"), "{body}");
16889 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
16890 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
16891 assert!(!body.contains("passed"), "{body}");
16892 }
16893
16894 #[test]
16895 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
16896 let mut state = review_state(&["feat: x"]);
16897 state.reviews = vec![
16898 round_with_notes(1, Some(SANDBOXED)),
16899 passing_round(2, Some("0123456789abcdef")),
16900 ];
16901 ok_gate(&mut state, Some("0123456789abcdef"));
16902 let body = body_at(&state);
16903 assert!(!body.contains("HTTP tests could not run"), "{body}");
16904 assert!(!body.contains("could not commit"), "{body}");
16906 assert!(
16907 body.contains("passed on an earlier commit (`0123456`)"),
16908 "{body}"
16909 );
16910 assert!(!body.contains("passed on the final commit"), "{body}");
16911
16912 ok_gate(&mut state, None);
16914 let body = body_at(&state);
16915 assert!(
16916 body.contains("passed on a commit that was not recorded"),
16917 "{body}"
16918 );
16919 }
16920
16921 #[test]
16922 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
16923 let state = state_with_summary(
16924 "add retries to the client",
16925 "TITLE: feat: retries\n\nDid it.",
16926 );
16927 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
16928 assert_eq!(m.title, "feat: retries");
16929 assert!(
16930 m.body.contains("<summary>Original task</summary>"),
16931 "{}",
16932 m.body
16933 );
16934 assert!(!m.body.contains("Commits under review"), "{}", m.body);
16935 assert!(
16936 m.body
16937 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
16938 "{}",
16939 m.body
16940 );
16941 }
16942
16943 #[test]
16944 fn pr_message_review_bounds_a_long_english_subject() {
16945 let long = format!("feat: {}", "word ".repeat(100));
16946 let m = pr_message(&review_state(&[&long]), 'A');
16947 assert!(m.title.starts_with("feat: word"), "{}", m.title);
16948 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
16949 }
16950
16951 #[test]
16952 fn pr_message_implementation_run_is_unchanged_by_review_support() {
16953 let state = state_with_summary("add retries\n\ndetails", "- did some things");
16954 let m = pr_message(&state, 'A');
16955 assert_eq!(m.title, "add retries");
16956 assert!(m.body.contains("<summary>Original task</summary>"));
16957 assert!(!m.body.contains("Commits under review"));
16958 assert_eq!(landing_subject_source(&state), state.instruction);
16959 }
16960
16961 #[test]
16962 fn review_run_squash_subject_is_the_change_not_the_prompt() {
16963 let state = review_state(&["feat: the change", "fix: typo"]);
16964 let source = landing_subject_source(&state);
16965 assert_eq!(land::merge_subject("", &source), "feat: the change");
16966 assert_eq!(
16967 land::merge_subject("magi: candidate A (uncommitted work)", &source),
16968 "feat: the change"
16969 );
16970 assert_eq!(
16972 land::merge_subject("feat: renamed by hand", &source),
16973 "feat: renamed by hand"
16974 );
16975 let blank = review_state(&["日本語"]);
16976 assert!(
16977 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
16978 );
16979 }
16980
16981 #[test]
16982 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
16983 let state = review_state(&["feat: the change"]);
16984 let source = landing_subject_source(&state);
16985 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
16986 assert_eq!(
16987 land::merge_subject(landing_title(&state, old), &source),
16988 "feat: the change"
16989 );
16990 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
16991 let task = state_with_summary("add retries", "");
16992 assert_eq!(landing_title(&task, old), old);
16993 }
16994
16995 #[test]
16996 fn pr_message_describes_the_change_not_the_task() {
16997 let state = state_with_summary(
16998 "今回やってほしいこと: results projector を直す",
16999 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17000 );
17001 let m = pr_message(&state, 'A');
17002 assert_eq!(m.title, "fix(web): batch the runs list reads");
17003 assert!(
17004 m.body.starts_with("## Summary\n\n- reads run.json once"),
17005 "{}",
17006 m.body
17007 );
17008 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17009 let task_at = m.body.find("今回やってほしいこと").unwrap();
17010 let details_at = m.body.find("<details>").unwrap();
17011 assert!(
17012 details_at < task_at,
17013 "the task lives inside <details>: {}",
17014 m.body
17015 );
17016 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17017 assert!(m.body.contains("magi:candidate-a"));
17018 }
17019
17020 #[test]
17021 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17022 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17023 let m = pr_message(&state, 'A');
17024 assert_eq!(m.title, "add retries");
17025 assert!(
17026 m.body.contains("## Summary\n\n- did some things"),
17027 "{}",
17028 m.body
17029 );
17030
17031 let none = RunState::new(
17032 PathBuf::from("/repo"),
17033 "main".to_owned(),
17034 "abc1234".to_owned(),
17035 "add retries".to_owned(),
17036 Config::default(),
17037 );
17038 let m = pr_message(&none, 'A');
17039 assert_eq!(m.title, "add retries");
17040 assert!(!m.body.contains("## Summary"), "{}", m.body);
17041 }
17042
17043 #[test]
17044 fn pr_message_refuses_the_candidate_commit_subject() {
17045 for bad in [
17046 "TITLE: magi: candidate A (uncommitted work)",
17047 "TITLE: chore: stuff (uncommitted work)",
17048 "TITLE: ",
17049 ] {
17050 let state = state_with_summary("add retries", bad);
17051 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17052 }
17053 }
17054
17055 #[test]
17056 fn pr_message_bounds_a_very_long_task_and_title() {
17057 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17058 let state = state_with_summary(&long, "- nothing");
17059 let m = pr_message(&state, 'A');
17060 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17061 assert!(!m.title.contains('\n'));
17062
17063 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17064 let m = pr_message(&state, 'A');
17065 assert!(m.title.starts_with("feat: "));
17066 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17067 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17068 }
17069
17070 fn long_title_of(instruction: &str) -> String {
17071 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17072 }
17073
17074 #[test]
17075 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17076 let first = "Make the landing path keep a readable title for long tasks";
17077 let line = format!(
17078 "{first}. {}",
17079 "And then keep going with more words ".repeat(20)
17080 );
17081 let t = long_title_of(&line);
17082 assert_eq!(t, first);
17083 assert!(!t.starts_with("chore: land"));
17084 }
17085
17086 #[test]
17087 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17088 let line = "word ".repeat(200);
17089 let t = long_title_of(&line);
17090 assert!(t.ends_with("word..."), "{t}");
17091 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17092 }
17093
17094 #[test]
17095 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17096 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17097 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17098 }
17099 }
17100
17101 #[test]
17102 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17103 let mut state = state_with_summary("再試行を追加する", "- nothing");
17104 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17105 state.config.graph.github_language = "ja".into();
17106 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17107 }
17108
17109 #[test]
17110 fn pr_message_title_limit_is_exact() {
17111 let at = "a".repeat(PR_TITLE_MAX);
17112 assert_eq!(long_title_of(&at), at);
17113 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17114 assert!(over.ends_with("..."), "{over}");
17115 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17116 }
17117
17118 #[test]
17119 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17120 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17121 let t = long_title_of(&line);
17122 assert!(t.ends_with("..."), "{t}");
17123 assert!(t.is_ascii(), "{t}");
17124 }
17125
17126 #[test]
17127 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17128 let line = format!(
17129 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17130 "plus more filler words ".repeat(20)
17131 );
17132 let t = long_title_of(&line);
17133 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17134 }
17135
17136 #[test]
17137 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17138 let line = format!("\n\n# {}", "title words ".repeat(40));
17139 let state = state_with_summary(&line, "- nothing");
17140 let m = pr_message(&state, 'A');
17141 assert!(m.title.starts_with("title words"), "{}", m.title);
17142 assert_eq!(
17143 land::merge_subject(&m.title, &landing_subject_source(&state)),
17144 m.title
17145 );
17146 assert_eq!(
17148 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17149 "feat: renamed by hand"
17150 );
17151 }
17152
17153 #[tokio::test]
17154 async fn github_text_rewrite_is_bounded_and_falls_back() {
17155 crate::run::pin_test_home();
17156 for (reply, accepted) in [
17157 (
17158 "TITLE: fix: retries\nAdd retries for failed requests.",
17159 true,
17160 ),
17161 (
17162 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17163 false,
17164 ),
17165 (
17166 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17167 true,
17168 ),
17169 (
17170 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17171 true,
17172 ),
17173 (
17174 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17175 true,
17176 ),
17177 (
17178 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17179 false,
17180 ),
17181 ("TITLE: fix: retries\n", false),
17182 ] {
17183 let dir = tempfile::tempdir().unwrap();
17184 let mut runner = runner_at(RunStatus::Gating);
17185 runner.state = state_with_summary(
17186 "add retries",
17187 "TITLE: fix: retries\n日本語の説明を書きます。",
17188 );
17189 runner.state.repo = dir.path().to_owned();
17190 runner.state.candidates[0].worktree = dir.path().to_owned();
17191 let mut author = spec("alpha");
17192 author.command = vec![
17193 "sh".into(),
17194 "-c".into(),
17195 "printf '%s' \"$REWRITE_REPLY\"".into(),
17196 ];
17197 author.env.insert("REWRITE_REPLY".into(), reply.into());
17198 runner.state.config.agents = vec![author];
17199 seed_public_summary(&mut runner);
17200 let winner = runner.state.candidates[0].clone();
17201 let first = runner
17202 .guarded_pr_message(&winner, None, true)
17203 .await
17204 .unwrap();
17205 let message = if accepted {
17206 first.expect("a passing rewrite never asks")
17207 } else {
17208 assert!(first.is_none());
17210 assert!(runner.state.parked);
17211 let store = ask::Questions::open();
17212 let asked: Vec<_> = store
17213 .list()
17214 .into_iter()
17215 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17216 .collect();
17217 assert_eq!(asked.len(), 1);
17218 assert!(!asked[0].detail.contains("secret"));
17219 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
17220 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
17221 assert_eq!(
17222 asked[0]
17223 .choices
17224 .contains(&crate::github_text::USE_REWRITE.to_owned()),
17225 rewrite.is_some()
17226 );
17227 store
17230 .update(&asked[0].id, |q| {
17231 q.abandon("test");
17232 Ok(())
17233 })
17234 .unwrap();
17235 let again = runner
17236 .guarded_pr_message(&winner, None, true)
17237 .await
17238 .unwrap()
17239 .expect("an abandoned question falls back");
17240 let n = store
17241 .list()
17242 .into_iter()
17243 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17244 .count();
17245 assert_eq!(n, 1, "asked once per rejected text");
17246 again
17247 };
17248 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
17249 assert!(!message.body.contains("secret"));
17250 assert!(!message.body.contains("prefix"));
17251 assert!(!message.title.contains("secret"));
17252 assert_eq!(
17253 message.body.contains("Add retries for failed requests."),
17254 accepted
17255 );
17256 assert_eq!(
17257 runner.state.seats["github-text-rewrite"].turns, 1,
17258 "only one rewrite invocation"
17259 );
17260 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
17261 if accepted {
17262 assert!(runner.state.github_text.is_none());
17263 assert!(!runner.state.parked);
17264 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
17265 assert!(message.body.contains("## Final verification"));
17266 assert!(message.body.contains("Evidence: final diff"));
17267 assert!(
17268 message
17269 .body
17270 .contains(&format!("magi:run/{}", runner.state.id))
17271 );
17272 }
17273 if !accepted {
17274 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17275 assert!(
17276 message
17277 .body
17278 .contains(&format!("magi:run/{}", runner.state.id))
17279 );
17280 }
17281 }
17282 }
17283
17284 fn scripted(id: &str, reply: &str) -> AgentSpec {
17285 let mut a = spec(id);
17286 a.command = vec![
17287 "sh".into(),
17288 "-c".into(),
17289 "printf '%s' \"$REWRITE_REPLY\"".into(),
17290 ];
17291 a.env.insert("REWRITE_REPLY".into(), reply.into());
17292 a
17293 }
17294
17295 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
17296 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
17297
17298 fn seed_public_summary(runner: &mut Runner) {
17300 let repo = &runner.state.repo;
17301 let g = |args: &[&str]| {
17302 let out = std::process::Command::new("git")
17303 .args(args)
17304 .current_dir(repo)
17305 .quiet()
17306 .output()
17307 .unwrap();
17308 assert!(out.status.success(), "git {args:?}");
17309 String::from_utf8_lossy(&out.stdout).trim().to_owned()
17310 };
17311 g(&["init", "-q", "-b", "main"]);
17312 g(&[
17313 "-c",
17314 "user.name=Test",
17315 "-c",
17316 "user.email=test@example.invalid",
17317 "commit",
17318 "-q",
17319 "--allow-empty",
17320 "-m",
17321 "feat: fixture",
17322 ]);
17323 let head = g(&["rev-parse", "HEAD"]);
17324 g(&["update-ref", "refs/remotes/origin/main", &head]);
17325 let evidence = crate::pr_outcome::summary_evidence(
17326 &runner.state,
17327 crate::pr_outcome::FinalHead::At(&head),
17328 );
17329 let generated = &runner.state.candidates[0].summary;
17330 runner.state.change_summary = Some(crate::run::ChangeSummary {
17331 base: head.clone(),
17332 head,
17333 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
17334 title: summary_title(generated).unwrap(),
17335 body: summary_without_title(generated),
17336 });
17337 }
17338
17339 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
17340 let mut runner = runner_at(RunStatus::Gating);
17341 runner.state = state_with_summary("add retries", JA_SUMMARY);
17342 if review_only {
17343 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
17344 }
17345 runner.state.repo = dir.to_owned();
17346 runner.state.candidates[0].worktree = dir.to_owned();
17347 runner.state.config.agents = agents;
17348 seed_public_summary(&mut runner);
17349 runner
17350 }
17351
17352 fn gate_questions(runner: &Runner) -> usize {
17353 ask::Questions::open()
17354 .list()
17355 .into_iter()
17356 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17357 .count()
17358 }
17359
17360 #[tokio::test]
17361 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
17362 crate::run::pin_test_home();
17363 let dir = tempfile::tempdir().unwrap();
17364 let mut runner = rewrite_runner(
17365 dir.path(),
17366 true,
17367 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17368 );
17369 runner.state.config.roles.language_judge =
17370 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
17371 let winner = runner.state.candidates[0].clone();
17372 let msg = runner
17373 .guarded_pr_message(&winner, None, true)
17374 .await
17375 .unwrap()
17376 .expect("an English rewrite posts without asking");
17377 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
17378 assert!(msg.body.contains("<details>kept</details>"));
17379 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
17380 assert_eq!(gate_questions(&runner), 0);
17381 assert!(runner.state.github_text.is_none());
17382 assert!(!runner.state.seats.contains_key("impl-A"));
17383 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17384 }
17385
17386 #[tokio::test]
17387 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
17388 crate::run::pin_test_home();
17389 let dir = tempfile::tempdir().unwrap();
17390 let mut runner = rewrite_runner(
17391 dir.path(),
17392 false,
17393 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
17394 );
17395 runner.state.config.roles.language_judge =
17396 Some(crate::config::AgentChoice::One("beta".into()));
17397 runner.state.seats.insert(
17398 "impl-A".into(),
17399 SeatState::new("impl-A", "alpha", runner.state.seed),
17400 );
17401 let winner = runner.state.candidates[0].clone();
17402 let msg = runner
17403 .guarded_pr_message(&winner, None, true)
17404 .await
17405 .unwrap()
17406 .expect("rewritten");
17407 assert!(msg.body.contains("<details>kept</details>"));
17408 assert_eq!(runner.state.seats["impl-A"].turns, 0);
17409 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17410 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
17411 }
17412
17413 #[tokio::test]
17414 async fn rewrite_reuses_a_resumable_implementer_session() {
17415 crate::run::pin_test_home();
17416 let dir = tempfile::tempdir().unwrap();
17417 let mut runner = rewrite_runner(
17418 dir.path(),
17419 false,
17420 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
17421 );
17422 runner.state.config.roles.language_judge =
17423 Some(crate::config::AgentChoice::One("beta".into()));
17424 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
17425 seat.turns = 1;
17426 runner.state.seats.insert("impl-A".into(), seat);
17427 let winner = runner.state.candidates[0].clone();
17428 runner
17429 .guarded_pr_message(&winner, None, true)
17430 .await
17431 .unwrap()
17432 .expect("rewritten");
17433 assert_eq!(runner.state.seats["impl-A"].turns, 2);
17434 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
17435 }
17436
17437 #[tokio::test]
17438 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
17439 crate::run::pin_test_home();
17440 let dir = tempfile::tempdir().unwrap();
17441 let mut broken = spec("bad");
17442 broken.command = vec!["false".into()];
17443 let mut runner = rewrite_runner(
17444 dir.path(),
17445 true,
17446 vec![
17447 broken,
17448 scripted("noformat", "Add retries."),
17449 scripted("good", EN_REWRITE),
17450 ],
17451 );
17452 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
17453 "bad".into(),
17454 "noformat".into(),
17455 "good".into(),
17456 ]));
17457 let winner = runner.state.candidates[0].clone();
17458 let msg = runner
17459 .guarded_pr_message(&winner, None, true)
17460 .await
17461 .unwrap()
17462 .expect("the third agent rewrites");
17463 assert!(msg.body.contains("`src/net.rs`"));
17464 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
17465 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
17466 }
17467
17468 #[tokio::test]
17469 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
17470 crate::run::pin_test_home();
17471 for (agents, role, saved) in [
17473 (
17474 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
17475 Some("beta"),
17476 true,
17477 ),
17478 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
17479 (Vec::new(), None, false),
17480 ] {
17481 let dir = tempfile::tempdir().unwrap();
17482 let mut runner = rewrite_runner(dir.path(), true, agents);
17483 runner.state.config.roles.language_judge =
17484 role.map(|r| crate::config::AgentChoice::One(r.into()));
17485 let winner = runner.state.candidates[0].clone();
17486 let first = runner
17487 .guarded_pr_message(&winner, None, true)
17488 .await
17489 .unwrap();
17490 assert!(first.is_none(), "withheld text parks the run");
17491 assert!(runner.state.parked);
17492 assert_eq!(gate_questions(&runner), 1);
17493 let g = runner.state.github_text.as_ref().unwrap();
17494 assert_eq!(g.rewrite.is_some(), saved);
17495 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
17496 }
17497 }
17498
17499 #[tokio::test]
17500 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
17501 crate::run::pin_test_home();
17502 let dir = tempfile::tempdir().unwrap();
17503 let mut runner = runner_at(RunStatus::Gating);
17504 runner.state =
17505 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
17506 runner.state.repo = dir.path().to_owned();
17507 runner.state.candidates[0].worktree = dir.path().to_owned();
17508 let mut author = spec("alpha");
17509 author.command = vec![
17510 "sh".into(),
17511 "-c".into(),
17512 "printf '%s' \"$REWRITE_REPLY\"".into(),
17513 ];
17514 author.env.insert(
17515 "REWRITE_REPLY".into(),
17516 format!(
17517 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
17518 "English sentence. ".repeat(600)
17519 ),
17520 );
17521 runner.state.config.agents = vec![author];
17522 seed_public_summary(&mut runner);
17523 let winner = runner.state.candidates[0].clone();
17524 assert!(
17525 runner
17526 .guarded_pr_message(&winner, None, true)
17527 .await
17528 .unwrap()
17529 .is_none()
17530 );
17531 let record = runner.state.github_text.clone().unwrap();
17532 let rewrite = record.rewrite.clone().unwrap();
17533 assert_eq!(rewrite.failed, ["body-language"]);
17534 assert!(
17535 rewrite
17536 .body
17537 .ends_with(&format!("magi:run/{}", runner.state.id))
17538 );
17539 let store = ask::Questions::open();
17540 let q = store.get(record.question.as_ref().unwrap()).unwrap();
17541 assert!(
17542 q.choices
17543 .contains(&crate::github_text::USE_REWRITE.to_owned())
17544 );
17545 for label in [
17546 "ORIGINAL",
17547 "FALLBACK",
17548 "REWRITE",
17549 "Rules still failed: body-language",
17550 "truncated",
17551 ] {
17552 assert!(q.detail.contains(label), "{label}");
17553 }
17554 let artifact = std::fs::read_to_string(crate::run::artifact_path(
17555 &runner.state,
17556 &crate::github_text::artifact_name(&record.fingerprint),
17557 ))
17558 .unwrap();
17559 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
17560 runner.state =
17562 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
17563 store
17564 .update(&q.id, |q| {
17565 q.say("まだ日本語")?;
17566 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
17567 })
17568 .unwrap();
17569 assert!(
17570 runner
17571 .guarded_pr_message(&winner, None, true)
17572 .await
17573 .unwrap()
17574 .is_none()
17575 );
17576 let retry = runner.state.github_text.as_ref().unwrap();
17577 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
17578 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
17579 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
17580 assert!(
17581 q.posting_comparison
17582 .as_ref()
17583 .unwrap()
17584 .matches_detail(&q.detail)
17585 );
17586 store
17587 .update(&q.id, |q| {
17588 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17589 })
17590 .unwrap();
17591 let message = runner
17592 .guarded_pr_message(&winner, None, true)
17593 .await
17594 .unwrap()
17595 .unwrap();
17596 assert_eq!(message.title, rewrite.title);
17597 assert_eq!(message.body, rewrite.body);
17598 assert_eq!(
17599 runner.state.github_text.as_ref().unwrap().choice,
17600 Some(GithubTextChoice::Rewrite)
17601 );
17602 let again = runner
17603 .guarded_pr_message(&winner, None, true)
17604 .await
17605 .unwrap()
17606 .unwrap();
17607 assert_eq!(again.title, message.title);
17608 assert_eq!(again.body, message.body);
17609 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
17610 assert!(
17611 !runner
17612 .state
17613 .events
17614 .iter()
17615 .any(|e| e.message.contains(&rewrite.title)
17616 || e.message.contains("English sentence."))
17617 );
17618 }
17619
17620 #[tokio::test]
17621 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
17622 crate::run::pin_test_home();
17623 let dir = tempfile::tempdir().unwrap();
17624 let mut runner = runner_at(RunStatus::Gating);
17625 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
17626 runner.state.repo = dir.path().to_owned();
17627 runner.state.candidates[0].worktree = dir.path().to_owned();
17628 seed_public_summary(&mut runner);
17629 let winner = runner.state.candidates[0].clone();
17630 assert!(
17631 runner
17632 .guarded_pr_message(&winner, None, true)
17633 .await
17634 .unwrap()
17635 .is_none()
17636 );
17637 let record = runner.state.github_text.as_ref().unwrap();
17638 assert!(record.rewrite.is_none());
17639 let mut old = serde_json::to_value(record).unwrap();
17641 old.as_object_mut().unwrap().remove("rewrite");
17642 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
17643 assert!(old.rewrite.is_none());
17644 let id = record.question.clone().unwrap();
17645 ask::Questions::open()
17646 .update(&id, |q| {
17647 q.choices.push(crate::github_text::USE_REWRITE.into());
17648 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
17649 })
17650 .unwrap();
17651 let message = runner
17652 .guarded_pr_message(&winner, None, true)
17653 .await
17654 .unwrap()
17655 .unwrap();
17656 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
17657 assert_eq!(
17658 runner.state.github_text.as_ref().unwrap().choice,
17659 Some(GithubTextChoice::Fallback)
17660 );
17661 }
17662
17663 #[tokio::test]
17664 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
17665 crate::run::pin_test_home();
17666 let dir = tempfile::tempdir().unwrap();
17667 let mut runner = runner_at(RunStatus::Gating);
17668 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
17669 runner.state.repo = dir.path().to_owned();
17670 runner.state.candidates[0].worktree = dir.path().to_owned();
17671 seed_public_summary(&mut runner);
17672 let winner = runner.state.candidates[0].clone();
17673 assert!(
17674 runner
17675 .guarded_pr_message(&winner, None, true)
17676 .await
17677 .unwrap()
17678 .is_none()
17679 );
17680 let store = ask::Questions::open();
17681 let find = |run: &str| {
17682 store
17683 .list()
17684 .into_iter()
17685 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
17686 .unwrap()
17687 };
17688 let run = runner.state.id.clone();
17689 let q = find(&run);
17690 let fp = runner
17691 .state
17692 .github_text
17693 .as_ref()
17694 .unwrap()
17695 .fingerprint
17696 .clone();
17697 let name = crate::github_text::artifact_name(&fp);
17698 let path = crate::run::artifact_path(&runner.state, &name);
17699 assert!(q.detail.contains(&path.display().to_string()));
17700 assert!(q.detail.contains("title-language: title"));
17701 let saved = std::fs::read_to_string(&path).expect("artifact written");
17702 assert!(saved.contains("日本語のタイトル"));
17703 assert!(q.cwd.is_none());
17704 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
17705 let say = |id: &str, text: &str, choice: &str| {
17707 store
17708 .update(id, |q| {
17709 q.thread.push(ask::Turn {
17710 who: ask::Who::Operator,
17711 body: text.to_owned(),
17712 at: jiff::Timestamp::now(),
17713 note: None,
17714 });
17715 q.answer(ask::Answer::Choice(choice.to_owned()))
17716 })
17717 .unwrap();
17718 };
17719 say(&q.id, "まだ日本語", "use my text");
17720 assert!(
17721 runner
17722 .guarded_pr_message(&winner, None, true)
17723 .await
17724 .unwrap()
17725 .is_none()
17726 );
17727 let second = find(&run);
17728 assert_ne!(second.id, q.id);
17729 assert!(second.detail.contains("did not pass"));
17730 assert!(second.detail.contains("日本語のタイトル"));
17733 assert!(!second.detail.contains("まだ日本語"));
17734 assert!(second.detail.contains(&path.display().to_string()));
17735 say(&second.id, "fix: retry failed requests", "use my text");
17736 let message = runner
17737 .guarded_pr_message(&winner, None, true)
17738 .await
17739 .unwrap()
17740 .expect("a vetted replacement is posted");
17741 assert_eq!(message.title, "fix: retry failed requests");
17742 assert_eq!(
17743 runner
17744 .state
17745 .github_text
17746 .as_ref()
17747 .unwrap()
17748 .chosen_title
17749 .as_deref(),
17750 Some("fix: retry failed requests")
17751 );
17752 assert!(runner.state.github_text.as_ref().unwrap().resolved);
17753 let again = runner
17755 .guarded_pr_message(&winner, None, true)
17756 .await
17757 .unwrap()
17758 .unwrap();
17759 assert_eq!(again.title, message.title);
17760 assert_eq!(
17761 store
17762 .list()
17763 .into_iter()
17764 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
17765 .count(),
17766 2
17767 );
17768 }
17769
17770 #[test]
17771 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
17772 let mut state = state_with_summary(
17776 "add retries",
17777 "TITLE: fix(web): batch reads\n- reads run.json once",
17778 );
17779 state.config.graph.language = "ja".to_owned();
17780 let m = pr_message(&state, 'A');
17781 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
17782 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
17783
17784 let task = "今回やってほしいこと: results projector を直す";
17787 let mut state = state_with_summary(task, "- no title line");
17788 state.config.graph.language = "ja".to_owned();
17789 let m = pr_message(&state, 'A');
17790 assert_eq!(
17791 m.title,
17792 format!("chore: land candidate A of run {}", state.id)
17793 );
17794 assert!(
17795 m.body.contains(&format!(
17796 "<summary>Original task</summary>\n\n{task}\n\n</details>"
17797 )),
17798 "{}",
17799 m.body
17800 );
17801 }
17802
17803 #[test]
17804 fn pr_message_scrubs_home_paths_and_addresses() {
17805 let state = state_with_summary(
17806 "fix it in /Users/someone/src/x",
17807 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
17808 );
17809 let m = pr_message(&state, 'A');
17810 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
17811 assert!(!m.body.contains(leak), "{}", m.body);
17812 }
17813 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
17814 }
17815
17816 #[test]
17817 fn pr_message_survives_a_task_that_closes_details() {
17818 let state = state_with_summary("a </details> b", "TITLE: fix: x");
17819 let m = pr_message(&state, 'A');
17820 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
17821 }
17822
17823 #[test]
17824 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
17825 let cmd = manual_merge_command(
17826 MergeStyle::Squash,
17827 Path::new("/repo"),
17828 "b",
17829 "fix: \"quoted\" $(x) `y`\n\nbody",
17830 );
17831 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
17832 }
17833
17834 #[test]
17835 fn manual_merge_command_matches_the_configured_style() {
17836 let repo = Path::new("/repo");
17837 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
17838
17839 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
17840 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
17841
17842 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
17843 assert_eq!(
17844 squash,
17845 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
17846 \"Merge magi run 0832 (candidate A)\""
17847 );
17848
17849 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
17850 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
17851 }
17852
17853 #[test]
17854 fn a_nudge_gets_a_quarter_of_the_budget() {
17855 assert_eq!(retry_budget(secs(1200), true), secs(300));
17857 assert_eq!(retry_budget(secs(3600), true), secs(900));
17858 }
17859
17860 #[test]
17861 fn a_resent_prompt_keeps_the_whole_budget() {
17862 assert_eq!(retry_budget(secs(1200), false), secs(1200));
17865 assert_eq!(retry_budget(secs(60), false), secs(60));
17866 }
17867
17868 #[test]
17869 fn the_floor_never_exceeds_the_original_budget() {
17870 assert_eq!(retry_budget(secs(60), true), secs(60));
17874 assert_eq!(retry_budget(secs(480), true), secs(120));
17875 assert_eq!(retry_budget(secs(0), true), secs(0));
17876 }
17877
17878 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
17879 agent::CommandEvidence {
17880 id: "item1".to_owned(),
17881 description: "cargo test".to_owned(),
17882 exit_code,
17883 result_summary: String::new(),
17884 source: "codex".to_owned(),
17885 }
17886 }
17887
17888 #[test]
17889 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
17890 assert!(!has_unconfirmed_command(&[]));
17894 }
17895
17896 #[test]
17897 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
17898 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
17902 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
17903 assert!(!has_unconfirmed_command(&[
17904 evidence(Some(0)),
17905 evidence(Some(101))
17906 ]));
17907 }
17908
17909 #[test]
17910 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
17911 assert!(has_unconfirmed_command(&[
17912 evidence(Some(0)),
17913 evidence(None)
17914 ]));
17915 }
17916
17917 #[test]
17918 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
17919 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17920 assert_eq!(
17921 verified_noop_claim(true, &[], text).as_deref(),
17922 Some("already fixed by b32cfc4, on main.")
17923 );
17924 }
17925
17926 #[test]
17927 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
17928 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17931 assert!(verified_noop_claim(false, &[], text).is_none());
17932 }
17933
17934 #[test]
17935 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
17936 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
17937 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
17938 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
17940 }
17941
17942 #[test]
17943 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
17944 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
17945 }
17946
17947 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
17950 runner.state.candidates = shape
17951 .iter()
17952 .enumerate()
17953 .map(|(i, &(empty, verified))| Candidate {
17954 index: i,
17955 label: (b'A' + i as u8) as char,
17956 agent: "sonnet".to_owned(),
17957 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
17958 worktree: PathBuf::from(format!("/wt/{i}")),
17959 summary: String::new(),
17960 stat: String::new(),
17961 files: 0,
17962 commits: 0,
17963 empty,
17964 failed: None,
17965 verified_noop: verified.map(str::to_owned),
17966 duration_ms: 0,
17967 folded: false,
17968 })
17969 .collect();
17970 }
17971
17972 #[test]
17973 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
17974 ask_test_home();
17975 let mut runner = runner_at(RunStatus::Implementing);
17976 set_candidates(
17977 &mut runner,
17978 &[
17979 (true, Some("already on main at b32cfc4")),
17980 (true, Some("same fix, see the existing test")),
17981 ],
17982 );
17983
17984 runner
17985 .after_implement()
17986 .expect("a verified no-op is not an error");
17987
17988 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
17989 }
17990
17991 #[test]
17992 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
17993 ask_test_home();
17994 let mut runner = runner_at(RunStatus::Implementing);
17995 set_candidates(
17999 &mut runner,
18000 &[(true, Some("already on main at b32cfc4")), (true, None)],
18001 );
18002
18003 let err = runner
18004 .after_implement()
18005 .expect_err("an unverified empty candidate must still fail the run");
18006
18007 assert!(
18008 err.to_string().contains("no candidate produced a change"),
18009 "{err}"
18010 );
18011 assert_eq!(runner.state.status, RunStatus::Failed);
18012 }
18013
18014 #[test]
18015 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18016 ask_test_home();
18017 let mut runner = runner_at(RunStatus::Implementing);
18018 set_candidates(&mut runner, &[(true, None), (true, None)]);
18019
18020 let err = runner
18021 .after_implement()
18022 .expect_err("no candidate declared anything; this is an ordinary failure");
18023
18024 assert!(
18025 err.to_string().contains("no candidate produced a change"),
18026 "{err}"
18027 );
18028 assert_eq!(runner.state.status, RunStatus::Failed);
18029 }
18030}
18031
18032#[derive(Debug)]
18035struct GraftRestoreFailed(String);
18036
18037impl std::fmt::Display for GraftRestoreFailed {
18038 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18039 write!(
18040 f,
18041 "could not restore the winner to its pre-graft commit: {}",
18042 self.0
18043 )
18044 }
18045}
18046
18047impl std::error::Error for GraftRestoreFailed {}
18048
18049#[cfg(test)]
18053mod resource_wait_tests {
18054 use super::*;
18055 use std::collections::BTreeMap;
18056
18057 fn sh(id: &str, script: &str) -> AgentSpec {
18058 AgentSpec {
18059 id: id.to_owned(),
18060 kind: crate::config::AgentKind::Command,
18061 model: None,
18062 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18063 extra_args: Vec::new(),
18064 env: BTreeMap::new(),
18065 prompt_delivery: None,
18066 }
18067 }
18068
18069 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18070 SeatJob {
18071 seat: SeatState::new("impl-A", &spec.id, 1),
18072 spec,
18073 cwd: dir.to_path_buf(),
18074 prompt: "do it".to_owned(),
18075 timeout,
18076 allow_write: true,
18077 sessions: false,
18078 artifacts: dir.join("art"),
18079 stem: "impl-A".to_owned(),
18080 handover: None,
18081 }
18082 }
18083
18084 fn state(resource_wait_secs: u64) -> RunState {
18085 let mut config = Config::default();
18086 config.graph.timeout_resource_wait = resource_wait_secs;
18087 RunState::new(
18088 PathBuf::from("/repo"),
18089 "main".to_owned(),
18090 "abc1234".to_owned(),
18091 "task".to_owned(),
18092 config,
18093 )
18094 }
18095
18096 fn hold(cache: &Path) -> crate::cache::Guard {
18097 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18098 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18099 crate::cache::AcquireOutcome::Acquired(g) => g,
18100 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18101 }
18102 }
18103
18104 async fn run(
18105 jobs: Vec<SeatJob>,
18106 sem: Arc<Semaphore>,
18107 cache: Option<&Path>,
18108 st: &mut RunState,
18109 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18110 let prompts = Prompts::default();
18111 let ctx = WaveCtx {
18112 run: "r",
18113 node: "implement",
18114 prompts: &prompts,
18115 cache,
18116 round: None,
18117 carry_seats: false,
18118 pauses: None,
18119 };
18120 wave(jobs, sem, &ctx, st, 0).await
18121 }
18122
18123 #[tokio::test]
18124 async fn verification_cache_wait_preserves_the_command_budget() {
18125 crate::run::pin_test_home();
18126 let tmp = tempfile::tempdir().unwrap();
18127 let cache = tmp.path().join("target");
18128 let guard = hold(&cache);
18129 let release = tokio::spawn(async move {
18130 tokio::time::sleep(Duration::from_millis(100)).await;
18131 guard.release();
18132 });
18133 let mut st = state(10);
18134 let budget = Duration::from_millis(50);
18135 let wait = repository_wait(&st, None);
18136 let (outcomes, _) = with_cache_lease(
18137 &mut st,
18138 Some(&cache),
18139 "gate",
18140 "gate",
18141 tmp.path(),
18142 "abc1234",
18143 budget,
18144 "test gate",
18145 &wait,
18146 |_, available| async move {
18147 assert_eq!(
18148 available, budget,
18149 "contention must not shorten verification"
18150 );
18151 (Vec::new(), false, Vec::new())
18152 },
18153 )
18154 .await
18155 .unwrap();
18156 release.await.unwrap();
18157 assert!(outcomes.is_empty(), "{outcomes:?}");
18158 assert!(
18159 st.events
18160 .iter()
18161 .any(|event| event.message.contains("waiting for"))
18162 );
18163 assert!(st.resource_wait.is_none());
18164 hold(&cache).release();
18166 }
18167
18168 #[tokio::test]
18169 async fn verification_cache_wait_expires_without_starting_commands() {
18170 crate::run::pin_test_home();
18171 let tmp = tempfile::tempdir().unwrap();
18172 let cache = tmp.path().join("target");
18173 let _held = hold(&cache);
18174 let mut st = state(0);
18175 let wait = repository_wait(&st, None);
18176 let (outcomes, _) = with_cache_lease(
18177 &mut st,
18178 Some(&cache),
18179 "gate",
18180 "gate",
18181 tmp.path(),
18182 "abc1234",
18183 Duration::from_secs(60),
18184 "test gate",
18185 &wait,
18186 |_, _| async { panic!("commands must not start without the cache lease") },
18187 )
18188 .await
18189 .unwrap();
18190 assert_eq!(outcomes.len(), 1);
18191 assert!(outcomes[0].resource_blocked);
18192 assert!(st.resource_wait.is_none());
18193 }
18194
18195 #[tokio::test]
18196 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18197 crate::run::pin_test_home();
18198 let tmp = tempfile::tempdir().unwrap();
18199 let cache = tmp.path().join("target-a");
18200 let guard = hold(&cache);
18201 tokio::spawn(async move {
18203 tokio::time::sleep(Duration::from_millis(1500)).await;
18204 guard.release();
18205 });
18206 let mut st = state(60);
18207 let results = run(
18208 vec![job(
18209 sh("alpha", "echo ok"),
18210 tmp.path(),
18211 Duration::from_secs(1),
18212 )],
18213 Arc::new(Semaphore::new(1)),
18214 Some(&cache),
18215 &mut st,
18216 )
18217 .await;
18218 assert!(
18219 matches!(results[0].2, AgentOutcome::Ok(_)),
18220 "the wait must not be charged to the agent"
18221 );
18222 assert!(st.resource_block.is_none());
18223 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18224 assert!(st.resource_wait.is_none() && st.active.is_empty());
18225 }
18226
18227 #[tokio::test]
18228 #[cfg(unix)]
18229 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
18230 use std::os::fd::AsRawFd as _;
18231 crate::run::pin_test_home();
18232 let tmp = tempfile::tempdir().unwrap();
18233 crate::git::git(tmp.path(), &["init"]).await.unwrap();
18234 let common = tmp.path().join(".git");
18235 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18236 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
18237 assert_eq!(
18238 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
18239 0
18240 );
18241 let mut st = state(1);
18242 st.config.coordination = crate::config::test_coordination();
18243 let mut reader = job(
18244 sh("reader", "echo must-not-run"),
18245 tmp.path(),
18246 Duration::from_secs(30),
18247 );
18248 reader.allow_write = false;
18249 let results = run(
18250 vec![reader.clone()],
18251 Arc::new(Semaphore::new(1)),
18252 None,
18253 &mut st,
18254 )
18255 .await;
18256 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18257 assert_eq!(results[0].1.turns, 0);
18258 assert!(FailClass::of(&results[0].2).is_none());
18259 assert_eq!(
18260 st.resource_block.as_ref().unwrap().kind,
18261 "repository coordination"
18262 );
18263 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18264 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18265 drop(external);
18266 let editor = crate::repo_lock::SeatLease::for_checkout(
18267 tmp.path(),
18268 true,
18269 crate::config::test_coordination().as_ref(),
18270 )
18271 .await
18272 .unwrap();
18273 let pause = Pause::new();
18274 let interrupt = Pause::new();
18275 let prompts = Prompts::default();
18276 let ctx = WaveCtx {
18277 run: "r",
18278 node: "review",
18279 prompts: &prompts,
18280 cache: None,
18281 round: None,
18282 carry_seats: false,
18283 pauses: Some((&pause, &interrupt)),
18284 };
18285 let (results, ()) = tokio::join!(
18286 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
18287 async {
18288 tokio::time::sleep(Duration::from_millis(50)).await;
18289 pause.park();
18290 }
18291 );
18292 assert!(
18293 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
18294 );
18295 assert_eq!(results[0].1.turns, 0);
18296 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
18297 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
18298 drop(editor);
18299 }
18300
18301 #[tokio::test]
18302 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
18303 crate::run::pin_test_home();
18304 let tmp = tempfile::tempdir().unwrap();
18305 let cache = tmp.path().join("target-b");
18306 let _held = hold(&cache);
18307 let mut st = state(1);
18308 let results = run(
18309 vec![job(
18310 sh("alpha", "echo ok"),
18311 tmp.path(),
18312 Duration::from_secs(1),
18313 )],
18314 Arc::new(Semaphore::new(1)),
18315 Some(&cache),
18316 &mut st,
18317 )
18318 .await;
18319 let (_, seat, out) = &results[0];
18320 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
18321 assert!(wave_blocked(&results));
18322 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
18323 assert_eq!(seat.turns, 0, "nothing was asked");
18324 assert!(
18325 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
18326 "no agent was spawned"
18327 );
18328 assert!(FailClass::of(out).is_none(), "not a failure class");
18329 let block = st.resource_block.as_ref().expect("recorded for the resume");
18330 assert_eq!(block.kind, "shared build cache");
18331 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
18332 assert!(st.active.is_empty());
18333 }
18334
18335 #[tokio::test]
18336 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
18337 crate::run::pin_test_home();
18338 let tmp = tempfile::tempdir().unwrap();
18339 let mut st = state(1);
18340 let results = run(
18341 vec![job(
18342 sh("alpha", "echo ok"),
18343 tmp.path(),
18344 Duration::from_secs(1),
18345 )],
18346 Arc::new(Semaphore::new(0)),
18347 None,
18348 &mut st,
18349 )
18350 .await;
18351 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
18352 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
18353 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
18354 }
18355
18356 #[tokio::test]
18357 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
18358 crate::run::pin_test_home();
18359 let tmp = tempfile::tempdir().unwrap();
18360 let mut st = state(60);
18361 let results = run(
18362 vec![job(
18363 sh("alpha", "sleep 30"),
18364 tmp.path(),
18365 Duration::from_secs(1),
18366 )],
18367 Arc::new(Semaphore::new(1)),
18368 None,
18369 &mut st,
18370 )
18371 .await;
18372 let out = &results[0].2;
18373 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
18374 let class = FailClass::of(out).expect("a failure");
18375 assert_eq!(class, FailClass::Timeout);
18376 assert!(should_hand_over(None, &class), "existing handover policy");
18377 assert!(st.resource_block.is_none());
18378 }
18379}