1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 for id in &takeover.earlier {
1033 if let Ok(old) = RunState::load_under(id, &takeover.home)
1034 && old.repo == state.repo
1035 && (old.pr_head.as_deref() == Some(branch)
1036 || old.winner().is_some_and(|w| w.branch == branch))
1037 {
1038 for hold in old.branch_owner_holds {
1039 if !state.branch_owner_holds.contains(&hold) {
1040 state.event(
1041 "approval-policy",
1042 format!(
1043 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044 hold.control.key(),
1045 hold.file,
1046 hold.head
1047 ),
1048 );
1049 state.branch_owner_holds.push(hold);
1050 }
1051 }
1052 }
1053 }
1054 let reviewed_head = git::rev_parse(repo, branch).await?;
1055 state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057 let worktree = state.worktree_root().join("under-review");
1060 if let Some(parent) = worktree.parent() {
1061 tokio::fs::create_dir_all(parent).await.ok();
1062 }
1063 let path = worktree.to_string_lossy().to_string();
1064 git::git(repo, &["worktree", "add", &path, branch])
1065 .await
1066 .with_context(|| {
1067 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068 })?;
1069
1070 let setup = match ensure_setup_config(&mut state, repo).await {
1071 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072 Err(e) => Err(e),
1073 };
1074 if let Err(e) = setup {
1075 git::worktree_remove_clean(repo, &worktree).await.ok();
1076 return Err(e);
1077 }
1078 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079 .await
1080 .unwrap_or(0);
1081 if commits == 0 {
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084 }
1085 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086 .await
1087 .map(|f| f.len())
1088 .unwrap_or(0);
1089 if files == 0
1090 && let (Ok(head_tree), Ok(base_tree)) = (
1091 git::tree_of(&worktree, "HEAD").await,
1092 git::tree_of(&worktree, &base_commit).await,
1093 )
1094 && head_tree == base_tree
1095 {
1096 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097 git::worktree_remove_clean(repo, &worktree).await.ok();
1098 bail!(
1099 "`{branch}` at {} has a tree identical to base {}; this usually means \
1100 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101 against `{}/{branch}`) rather than an empty change",
1102 short(&head),
1103 short(&base_commit),
1104 state.config.merge.remote
1105 );
1106 }
1107 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108 .await
1109 .unwrap_or_default();
1110
1111 state.candidates.push(Candidate {
1112 index: 0,
1113 label: 'A',
1114 agent: EXISTING_BRANCH.to_owned(),
1117 branch: branch.to_owned(),
1118 worktree,
1119 summary: String::new(),
1120 stat,
1121 files,
1122 commits,
1123 empty: false,
1124 failed: None,
1125 verified_noop: None,
1126 duration_ms: 0,
1127 folded: false,
1128 });
1129 state.tally = Some(Tally {
1130 first_choice: BTreeMap::from([('A', 0)]),
1131 borda: BTreeMap::new(),
1132 winner: 'A',
1133 rankings: 0,
1134 unanimous_initial: false,
1135 deliberated: false,
1136 changed_votes: 0,
1137 unanimous_final: false,
1138 tie_break: None,
1139 judges: 0,
1143 present: 0,
1144 quorum: 0,
1145 met_quorum: true,
1146 uncontested: Some("review-only run: nothing competed".to_owned()),
1147 });
1148 state.status = RunStatus::Reviewing;
1149 state.prep_complete = Some(true);
1150 state.event(
1151 "start",
1152 format!(
1153 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154 state.id
1155 ),
1156 );
1157 state.save()?;
1158 Ok(Self {
1159 state,
1160 roles,
1161 sem: Arc::new(Semaphore::new(max_parallel)),
1162 pause: Pause::new(),
1163 interrupt: Pause::new(),
1164 })
1165 }
1166
1167 pub fn resume(id: &str) -> Result<Self> {
1169 let state = RunState::load(id)?;
1170 crate::pr_head::ensure_owner(&state)?;
1171 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172 bail!(
1173 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174 );
1175 }
1176 if let Some(to) = &state.released_to {
1177 bail!(
1178 "run {} cannot be resumed: its worktree was released to run {}",
1179 state.short(),
1180 crate::run::short_of(to)
1181 );
1182 }
1183 let roles = state.config.resolve_roles()?;
1184 if let Some(id) = &state.config.graph.selected_implementer
1185 && state.candidates.iter().any(|c| c.agent != *id)
1186 {
1187 bail!(
1188 "replacement implementer pin does not match the saved candidates; task must stay held"
1189 );
1190 }
1191 let max_parallel = state.config.graph.max_parallel.max(1);
1192 Ok(Self {
1193 state,
1194 roles,
1195 sem: Arc::new(Semaphore::new(max_parallel)),
1196 pause: Pause::new(),
1197 interrupt: Pause::new(),
1198 })
1199 }
1200
1201 pub async fn execute(&mut self) -> Result<()> {
1208 if let Some(selection) = &self.state.pr_recovery
1209 && (selection.source_proof.is_some() || selection.source_run.is_some())
1210 {
1211 Box::pin(selection.verify_source(&self.state.repo)).await?;
1212 }
1213 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216 || self.state.land_approval.is_some()
1217 || self.state.merge_decision.is_some()
1218 || !self.state.merge_repairs.is_empty()
1219 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220 {
1221 let claim = FixClaim::acquire(&self.state.dir())?;
1222 let disk = RunState::load(&self.state.id)?;
1223 let checkpoint = |s: &RunState| {
1224 (
1225 s.merge_repairs.len(),
1226 s.merge_repairs
1227 .last()
1228 .map(|r| (r.question.clone(), r.phase)),
1229 s.land_approval
1230 .as_ref()
1231 .map(|a| (a.question.clone(), a.head.clone())),
1232 )
1233 };
1234 if checkpoint(&disk) != checkpoint(&self.state) {
1235 bail!(
1236 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237 );
1238 }
1239 Some(claim)
1240 } else {
1241 None
1242 };
1243 let mut execution = self.claim_driver().await?;
1244 let runner = &mut *execution.runner;
1245 let wait = runner.repository_wait();
1246 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249 let result = match result {
1250 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251 record_repository_block(&mut runner.state, &e);
1252 runner.park_here()?;
1253 Ok(())
1254 }
1255 other => other,
1256 };
1257 execution.driver.finish(&mut runner.state).await?;
1258 let ended = if result.is_err() {
1259 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260 } else {
1261 crate::notices::run_ended(&runner.state)
1262 };
1263 if let Some(notice) = ended {
1264 crate::notices::raise(notice);
1265 }
1266 result
1267 }
1268
1269 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271 }
1272
1273 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274 let wait = self.repository_wait();
1275 coordinate_operation(&mut self.state, cwd, &wait).await
1276 }
1277
1278 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280 let owner = match crate::pr_head::owner_lock_wait(
1281 &crate::run::home(),
1282 &self.state.id,
1283 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1284 )
1285 .await
1286 {
1287 Ok(owner) => owner,
1288 Err(error) => {
1289 let detail = format!("worktree maintenance or ownership update: {error:#}");
1290 self.state.resource_block = Some(crate::run::ResourceBlock {
1294 node: "start".into(),
1295 kind: "worktree maintenance".into(),
1296 detail: detail.clone(),
1297 at: Timestamp::now(),
1298 });
1299 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1300 }
1301 };
1302 let disk = RunState::load(&self.state.id)?;
1303 crate::pr_head::ensure_owner(&disk)?;
1304 if disk.liveness(false) == crate::run::Liveness::Live {
1305 bail!("run already has a live driver; wait for its execution to end");
1306 }
1307 if let Some(to) = &disk.released_to {
1308 bail!(
1309 "run {} cannot continue: its worktree was released to run {}",
1310 self.state.short(),
1311 crate::run::short_of(to)
1312 );
1313 }
1314 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1315 let mut execution = DriverExecution {
1316 runner: self,
1317 driver,
1318 };
1319 let _owner = owner;
1322 let state = &mut execution.runner.state;
1323 state.parked = false;
1328 state.resource_wait = None;
1329 state.resource_block = None;
1330 state.clear_active();
1337 let pid = std::process::id();
1352 state.driver_pid = Some(pid);
1353 state.driver_started_at = crate::proc::process_started_at(pid);
1354 state.driver_exited = false;
1355 state.begin_pass();
1356 execution.driver.bind(state);
1357 state.save()?;
1358 execution.driver.record(state)?;
1359 Ok(execution)
1360 }
1361
1362 async fn execute_graph(&mut self) -> Result<()> {
1363 self.state.configure_recovery_resume()?;
1364 self.persist_candidates().await?;
1367 if self
1368 .state
1369 .merge_decision
1370 .as_ref()
1371 .is_some_and(|decision| !decision.closed)
1372 && self.state.status != RunStatus::Landing
1373 {
1374 if crate::merge_decision::resumable_hold(&self.state) {
1375 self.state.merge = Some(MergeOutcome {
1376 mode: MergeMode::Pr,
1377 ok: false,
1378 detail: self
1379 .state
1380 .merge_decision
1381 .as_ref()
1382 .unwrap()
1383 .target
1384 .pr
1385 .clone(),
1386 empty: false,
1387 });
1388 self.run_land().await?;
1389 self.settle_questions();
1390 return Ok(());
1391 }
1392 bail!(
1393 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1394 );
1395 }
1396 if self.state.config.graph.selected_implementer.is_some()
1397 && self.state.status == RunStatus::Blocked
1398 && self.state.tally.is_none()
1399 && self.state.candidate_preservation.is_empty()
1400 && self.state.interrupted_implementations.is_empty()
1401 {
1402 bail!(
1403 "selected implementation is held; explicitly release the owning task to retry it"
1404 );
1405 }
1406
1407 if self.state.merge_repairs.last().is_some_and(|r| {
1408 matches!(
1409 r.phase,
1410 crate::run::MergeRepairPhase::Requested
1411 | crate::run::MergeRepairPhase::Fixing
1412 | crate::run::MergeRepairPhase::Reviewing
1413 )
1414 }) {
1415 if self.repair_merge_review().await? {
1416 self.run_land().await?;
1417 }
1418 self.settle_questions();
1419 return Ok(());
1420 }
1421 if self.state.pr_publication_pending {
1424 self.merge().await?;
1425 if self.resource_halt()? {
1426 return Ok(());
1427 }
1428 self.state.save()?;
1429 self.settle_questions();
1430 return Ok(());
1431 }
1432 if self.state.status == RunStatus::Stalled {
1445 if self.recover_stall().await? {
1446 self.finish_after_tally().await?;
1447 } else {
1448 self.state.save()?;
1450 }
1451 return Ok(());
1452 }
1453 if self.state.status == RunStatus::Landing
1463 || crate::handover::landing_checkpoint(&self.state)
1464 {
1465 self.state.land_sync_pending = true;
1466 self.state.save()?;
1467 self.run_land().await?;
1468 self.settle_questions();
1473 return Ok(());
1474 }
1475 if self.state.github_text.is_some()
1480 && self.state.merge.is_none()
1481 && self.state.status == RunStatus::Gating
1482 {
1483 self.merge().await?;
1484 if self.resource_halt()? {
1485 return Ok(());
1486 }
1487 self.state.save()?;
1488 return Ok(());
1489 }
1490 if let Err(e) = self.prep().await {
1491 if e.is::<crate::repo_lock::ResourceBlocked>() {
1492 return Err(e);
1493 }
1494 self.state.status = RunStatus::Blocked;
1495 self.state
1496 .prep_failure
1497 .get_or_insert_with(|| format!("{e:#}"));
1498 self.state
1499 .event("prep", format!("preparation stopped: {e:#}"));
1500 self.state.save()?;
1501 return Err(e);
1502 }
1503 if self.park_here()? {
1504 return Ok(());
1505 }
1506 self.advise().await?;
1507 if self.park_here()? {
1508 return Ok(());
1509 }
1510 self.implement().await?;
1511 if self.park_here()? {
1512 return Ok(());
1513 }
1514 if self.state.status == RunStatus::VerifiedNoop {
1518 return Ok(());
1519 }
1520 self.judge().await?;
1521 if self.park_here()? {
1522 return Ok(());
1523 }
1524 self.deliberate().await?;
1525 if self.park_here()? {
1526 return Ok(());
1527 }
1528 self.vote().await?;
1529 if self.park_here()? {
1530 return Ok(());
1531 }
1532 self.tally()?;
1533 if self.state.status == RunStatus::Stalled {
1538 self.state.save()?;
1542 return Ok(());
1543 }
1544 self.finish_after_tally().await?;
1545 Ok(())
1546 }
1547
1548 fn stop_for_resource(&mut self) -> Result<()> {
1553 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1555 {
1556 self.state.status = RunStatus::Blocked;
1557 }
1558 self.state.save()
1559 }
1560
1561 fn resource_halt(&mut self) -> Result<bool> {
1563 if self.state.resource_block.is_none() {
1564 return Ok(false);
1565 }
1566 self.stop_for_resource()?;
1567 Ok(true)
1568 }
1569
1570 fn park_here(&mut self) -> Result<bool> {
1577 let parking = self.pause.parked() || self.interrupt.parked();
1578 if !parking && self.state.resource_block.is_some() {
1579 self.stop_for_resource()?;
1580 return Ok(true);
1581 }
1582 if !self.pause.parked() && !self.interrupt.parked() {
1587 return Ok(false);
1588 }
1589 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1590 let why = match cause.as_ref().map(ParkCause::label) {
1591 Some(reason) => format!(
1592 "parked after `{}` ({reason}) — resume to carry on from here",
1593 self.state.status.as_str()
1594 ),
1595 None => format!(
1596 "parked after `{}` — resume to carry on from here",
1597 self.state.status.as_str()
1598 ),
1599 };
1600 self.state.event("park", why);
1601 self.state
1602 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1603 self.state.parked = true;
1604 self.state.save()?;
1605 Ok(true)
1606 }
1607
1608 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1610 if self.state.prep_failure.is_none() {
1611 return Ok(());
1612 }
1613 let failed_at = self
1614 .state
1615 .events
1616 .iter()
1617 .rev()
1618 .find(|event| event.node == "prep")
1619 .map_or(self.state.updated_at, |event| event.at);
1620 anyhow::ensure!(
1621 approval_at > failed_at,
1622 "preparation retry requires a new explicit release after the failure"
1623 );
1624 self.state.prep_failure = None;
1625 self.state.prep_complete = Some(false);
1626 self.state
1627 .event("prep-retry", "operator approved retry of preparation");
1628 Ok(())
1629 }
1630
1631 pub fn on_pause(&mut self, pause: Pause) {
1633 self.pause = pause;
1634 }
1635
1636 pub fn watch_interrupt(&mut self, pause: Pause) {
1642 self.interrupt = pause;
1643 }
1644
1645 fn settle_questions(&mut self) {
1665 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1666 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1667 }
1668 }
1669
1670 async fn finish_after_tally(&mut self) -> Result<()> {
1673 self.graft().await?;
1676 if self.resource_halt()? {
1677 return Ok(());
1678 }
1679 self.fold_losers().await?;
1680 self.sync_to_base().await?;
1685 if self.resource_halt()? {
1686 return Ok(());
1687 }
1688 if self.state.status == RunStatus::AlreadyInBase {
1689 return Ok(());
1690 }
1691 self.review_loop().await?;
1692 if self.resource_halt()? {
1693 return Ok(());
1694 }
1695 self.sync_to_base().await?;
1696 if self.resource_halt()? {
1697 return Ok(());
1698 }
1699 if self.state.status == RunStatus::AlreadyInBase {
1700 return Ok(());
1701 }
1702 self.gate().await?;
1703 self.merge().await?;
1704 if self.resource_halt()? {
1705 return Ok(());
1706 }
1707 self.state.save()?;
1708 Ok(())
1709 }
1710
1711 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1718 let result = match ensure_setup_config(&mut self.state, repo).await {
1719 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1720 Err(e) => Err(e),
1721 };
1722 if let Err(e) = result {
1723 git::worktree_remove_clean(repo, wt).await.ok();
1724 if in_prep {
1725 self.rollback_prep(repo).await;
1726 }
1727 self.state.status = RunStatus::Blocked;
1728 self.state.event(
1729 "setup",
1730 format!("worktree setup failed in {}: {e:#}", wt.display()),
1731 );
1732 self.state.save()?;
1733 return Err(e);
1734 }
1735 Ok(())
1736 }
1737
1738 async fn rollback_prep(&mut self, repo: &Path) {
1740 let root = self.state.worktree_root();
1741 for c in std::mem::take(&mut self.state.candidates) {
1742 if matches!(
1743 git::worktree_remove_clean(repo, &c.worktree).await,
1744 Ok(true)
1745 ) {
1746 git::branch_delete(repo, &c.branch).await.ok();
1747 } else {
1748 self.state.candidates.push(c);
1749 }
1750 }
1751 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1752 for j in 1..=seats {
1753 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1754 .await
1755 .ok();
1756 }
1757 for k in 1..=self.state.config.graph.advisors {
1758 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1759 .await
1760 .ok();
1761 }
1762 }
1763
1764 async fn prep(&mut self) -> Result<()> {
1765 if let Some(reason) = &self.state.prep_failure {
1766 bail!(
1767 "candidate preparation previously failed: {reason}; correct the source before retrying"
1768 );
1769 }
1770 if self
1771 .state
1772 .prep_complete
1773 .unwrap_or(!self.state.candidates.is_empty())
1774 {
1775 return Ok(());
1776 }
1777 let repo = self.state.repo.clone();
1778 let _coordination = self.coordinate(&repo).await?;
1781 let base = self.state.base_commit.clone();
1782 let plan = self.state.sources.plan(&repo, &base).await?;
1783 self.rollback_prep(&repo).await;
1784 if !self.state.candidates.is_empty() {
1785 bail!(
1786 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1787 );
1788 }
1789 self.state.prep_complete = Some(false);
1790 self.state.status = RunStatus::Prep;
1791 self.state.save()?;
1792 let repo = self.state.repo.clone();
1793 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1794 let root = self.state.worktree_root();
1795 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1796
1797 let hooks_dir = self.state.dir().join("hooks");
1800 if self.state.config.blind.commit_msg_hook {
1801 std::fs::create_dir_all(&hooks_dir)
1802 .with_context(|| format!("create {}", hooks_dir.display()))?;
1803 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1804 let path = hooks_dir.join("commit-msg");
1805 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1806 make_executable(&path)?;
1807 git::acquire_worktree_config(&repo).await?;
1815 self.state.enabled_worktree_config = true;
1816 }
1817
1818 ensure_setup_config(&mut self.state, &repo).await?;
1821
1822 for (index, (spec, label)) in self
1823 .roles
1824 .implementers
1825 .clone()
1826 .into_iter()
1827 .zip(labels)
1828 .enumerate()
1829 {
1830 let branch = self.state.branch_for(label);
1831 let worktree = root.join(format!("cand-{label}"));
1832 self.state.event(
1836 "prep-intent",
1837 serde_json::json!({
1838 "branch": branch,
1839 "worktree": worktree,
1840 "start": start,
1841 "picks": plan.picks,
1842 })
1843 .to_string(),
1844 );
1845 self.state.save()?;
1846 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1847 if self.state.config.blind.commit_msg_hook {
1848 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1849 }
1850 git::local_exclude(&worktree, "/.magi/").await?;
1851 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1852 git::worktree_remove_clean(&repo, &worktree).await.ok();
1855 git::branch_delete(&repo, &branch).await.ok();
1856 self.rollback_prep(&repo).await;
1857 self.state.status = RunStatus::Blocked;
1858 self.state
1859 .event("prep", format!("worktree setup failed: {e:#}"));
1860 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1861 self.state.save()?;
1862 return Err(e);
1863 }
1864 self.state.candidates.push(Candidate {
1865 index,
1866 label,
1867 agent: spec.id.clone(),
1868 branch,
1869 worktree,
1870 summary: String::new(),
1871 stat: String::new(),
1872 files: 0,
1873 commits: 0,
1874 empty: false,
1875 failed: None,
1876 verified_noop: None,
1877 duration_ms: 0,
1878 folded: false,
1879 });
1880 self.state.save()?;
1881 for pick in &plan.picks {
1882 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1883 {
1884 self.state.status = RunStatus::Blocked;
1885 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1886 self.state
1887 .event("prep", format!("cannot apply referenced commit: {e}"));
1888 self.state.save()?;
1889 return Err(e);
1890 }
1891 }
1892 }
1893
1894 for j in 1..=self.roles.judges.len() {
1895 let wt = root.join(format!("judge-{j}"));
1896 if !wt.exists() {
1897 git::worktree_add_detached(&repo, &wt, &base).await?;
1898 self.setup_seat_worktree(&repo, &wt, true).await?;
1899 }
1900 }
1901
1902 if self.state.config.graph.advise {
1910 for k in 1..=self.state.config.graph.advisors {
1911 let wt = root.join(format!("advisor-{k}"));
1912 if !wt.exists() {
1913 git::worktree_add_detached(&repo, &wt, &base).await?;
1914 self.setup_seat_worktree(&repo, &wt, true).await?;
1915 }
1916 }
1917 }
1918
1919 let authors: Vec<&str> = self
1924 .roles
1925 .implementers
1926 .iter()
1927 .map(|a| a.id.as_str())
1928 .collect();
1929 let overlap: Vec<String> = self
1930 .roles
1931 .judges
1932 .iter()
1933 .enumerate()
1934 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1935 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1936 .collect();
1937 if !overlap.is_empty() {
1938 let note = format!(
1939 "{} also authored a candidate; blind, but the panel is less \
1940 independent than {} distinct agents would be",
1941 overlap.join(", "),
1942 self.roles.judges.len()
1943 );
1944 self.state.event("prep", note);
1945 }
1946
1947 self.state.event(
1948 "prep",
1949 format!(
1950 "{} candidates, {} judges, base {} ({})",
1951 self.state.candidates.len(),
1952 self.roles.judges.len(),
1953 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1954 self.state.base_branch
1955 ),
1956 );
1957 self.state.prep_complete = Some(true);
1958 self.state.status = RunStatus::Implementing;
1959 self.state.save()?;
1960 Ok(())
1961 }
1962
1963 async fn advise(&mut self) -> Result<()> {
1996 let implement_untouched = self
1997 .state
1998 .candidates
1999 .iter()
2000 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2001 if !self.state.config.graph.advise || self.state.advise_attempted {
2002 return Ok(());
2003 }
2004 if !implement_untouched {
2005 self.state.event(
2006 "advise",
2007 "skipping the design-deliberation stage: at least one \
2008 candidate already shows implementation progress, so this \
2009 run is past the point the stage exists to run before"
2010 .to_owned(),
2011 );
2012 self.state.advise_attempted = true;
2013 self.state.save()?;
2014 return Ok(());
2015 }
2016 let run_id = self.state.id.clone();
2017 let prompts = self.state.config.prompts.clone();
2018 let instruction = self.state.instruction.clone();
2019 let language = self.state.config.graph.language.clone();
2020 let root = self.state.worktree_root();
2021 let n = self.state.config.graph.advisors;
2022 let where_recorded = self.state.dir().join("run.json");
2023
2024 let seats = match self.state.config.advisors() {
2025 Ok(seats) if !seats.is_empty() => seats,
2026 Ok(_) => {
2027 self.state.event(
2028 "advise",
2029 format!(
2030 "[graph] advisors is 0; skipping the design-deliberation \
2031 stage and continuing without a synthesis brief (see {})",
2032 where_recorded.display()
2033 ),
2034 );
2035 self.state.advise_attempted = true;
2036 self.state.save()?;
2037 return Ok(());
2038 }
2039 Err(e) => {
2040 self.state.event(
2041 "advise",
2042 format!(
2043 "could not resolve advisor seats ({e:#}); continuing \
2044 without a design-deliberation brief (see {})",
2045 where_recorded.display()
2046 ),
2047 );
2048 self.state.advise_attempted = true;
2049 self.state.save()?;
2050 return Ok(());
2051 }
2052 };
2053
2054 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2055 let artifacts = agent::artifacts_dir(&self.state.dir());
2056 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2057
2058 let mut jobs = Vec::new();
2059 for (i, spec) in seats.iter().cloned().enumerate() {
2060 let seat_key = format!("advisor-{}", i + 1);
2061 let seat = self.seat(&seat_key, &spec.id);
2062 jobs.push(SeatJob {
2063 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2064 spec,
2065 seat,
2066 cwd: worktrees[i % worktrees.len()].clone(),
2067 timeout,
2068 allow_write: false,
2069 sessions: false,
2070 artifacts: artifacts.clone(),
2071 stem: seat_key,
2072 handover: None,
2073 });
2074 }
2075
2076 self.state.event(
2077 "advise",
2078 format!(
2079 "{} advisor seat(s) sketching a design in parallel",
2080 jobs.len()
2081 ),
2082 );
2083 let mut quota_losses = Vec::new();
2084 let cache = self.state.config.cache_dir();
2085 let ctx = WaveCtx {
2086 carry_seats: false,
2087 pauses: Some((&self.pause, &self.interrupt)),
2088 run: &run_id,
2089 node: "advise",
2090 prompts: &prompts,
2091 cache: cache.as_deref(),
2092 round: None,
2093 };
2094 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2095 let results = ask_json_wave::<Proposal>(
2096 jobs,
2097 Arc::clone(&self.sem),
2098 self.state.config.graph.retries,
2099 &advisor_roster,
2100 &ctx,
2101 &mut quota_losses,
2102 &mut self.state,
2103 &|p: &Proposal| p.validate(),
2104 )
2105 .await;
2106 if self.state.resource_block.is_some() {
2107 return self.stop_for_resource();
2108 }
2109 self.state.quota.extend(quota_losses);
2110
2111 let mut records = Vec::with_capacity(results.len());
2112 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2113 let agent_id = seat.agent.clone();
2114 self.state.seats.insert(seat.key.clone(), seat);
2115 match res {
2116 Ok((proposal, out)) => {
2117 self.state
2118 .event("advise", format!("advisor-{} proposed a design", i + 1));
2119 records.push(advise::AdvisorRecord::proposed(
2120 i + 1,
2121 agent_id,
2122 proposal,
2123 out.duration_ms,
2124 ));
2125 }
2126 Err(e) => {
2127 self.state.event(
2128 "advise",
2129 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2130 );
2131 records.push(advise::AdvisorRecord::failed(
2132 i + 1,
2133 agent_id,
2134 e.to_string(),
2135 ));
2136 }
2137 }
2138 }
2139
2140 let mut advice = advise::Advice {
2141 records,
2142 synthesis: None,
2143 };
2144 if advice.proposals().is_empty() {
2145 self.state.event(
2146 "advise",
2147 "no advisor produced a usable proposal; continuing without a \
2148 synthesis brief"
2149 .to_owned(),
2150 );
2151 } else {
2152 match self
2153 .synthesize_brief(
2154 &advice,
2155 &instruction,
2156 &language,
2157 &worktrees[0],
2158 &artifacts,
2159 &run_id,
2160 &prompts,
2161 cache.as_deref(),
2162 )
2163 .await
2164 {
2165 Ok(Some(text)) => {
2166 self.state.event(
2167 "advise",
2168 "synthesized a design brief for the implementer".to_owned(),
2169 );
2170 advice.synthesis = Some(text);
2171 }
2172 Ok(None) => {
2173 self.state.event(
2174 "advise",
2175 "the synthesis seat produced nothing usable; continuing \
2176 without a design brief"
2177 .to_owned(),
2178 );
2179 }
2180 Err(e) => {
2181 if e.is::<crate::repo_lock::ResourceBlocked>() {
2182 return Err(e);
2183 }
2184 self.state.event(
2185 "advise",
2186 format!("could not synthesize a design brief: {e:#}"),
2187 );
2188 }
2189 }
2190 }
2191 advise::apply_reflection(&mut advice);
2192
2193 self.state.advice = Some(advice);
2194 self.state.advise_attempted = true;
2195 self.state.save()?;
2196 Ok(())
2197 }
2198
2199 #[allow(clippy::too_many_arguments)]
2211 async fn synthesize_brief(
2212 &mut self,
2213 advice: &advise::Advice,
2214 instruction: &str,
2215 language: &str,
2216 cwd: &Path,
2217 artifacts: &Path,
2218 run_id: &str,
2219 prompts: &Prompts,
2220 cache: Option<&Path>,
2221 ) -> Result<Option<String>> {
2222 let chain = agent::pick_chain(
2223 &self.state.config.agents,
2224 self.state.config.roles.synthesizer.as_ref(),
2225 &agent::installed,
2226 "synthesizer",
2227 )?;
2228 let proposals = advice.proposals();
2229 let mut prompt = prompt::with_overlay(
2230 prompt::synthesize_brief(instruction, &proposals, language),
2231 prompts.overlay("advise"),
2232 );
2233 if cache.is_some() {
2234 prompt.push('\n');
2239 prompt.push_str(&prompt::build_cache_note("advise", false));
2240 }
2241 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2242 let mut last = None;
2247 for (n, spec) in chain.iter().enumerate() {
2248 if n > 0 {
2249 self.state
2250 .event("advise", format!("synthesis falling back to {}", spec.id));
2251 }
2252 let mut seat = self.seat("advise-synthesis", &spec.id);
2253 let wait = self.repository_wait();
2254 let outcome = wait
2255 .scope(agent::invoke(
2256 spec,
2257 &mut seat,
2258 &Invocation {
2259 scope: crate::agent::InvocationScope::Repository,
2260 coordination: self.state.config.coordination.as_ref(),
2261 fetched_base: None,
2262 cwd,
2263 prompt: &prompt,
2264 timeout,
2265 allow_write: false,
2266 unsandboxed: false,
2267 sessions: false,
2268 artifacts,
2269 stem: &if n == 0 {
2270 "advise-synthesis".to_owned()
2271 } else {
2272 format!("advise-synthesis-{}", spec.id)
2273 },
2274 run: run_id,
2275 node: "advise",
2276 cache_dir: None,
2277 attachments: &[],
2278 writable: &[],
2279 },
2280 ))
2281 .await;
2282 if outcome.is_ok() {
2283 self.state.seats.insert(seat.key.clone(), seat);
2284 }
2285 let advance = agent::chain_advances(&outcome);
2286 last = Some(outcome);
2287 if !advance {
2288 break;
2289 }
2290 }
2291 let out = last.expect("a chain holds at least one agent")?;
2294 if !out.usable() {
2295 return Ok(None);
2296 }
2297 let text =
2298 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2299 Ok((!text.trim().is_empty()).then_some(text))
2300 }
2301
2302 async fn implement(&mut self) -> Result<()> {
2305 self.persist_candidates().await?;
2306 let run_id = self.state.id.clone();
2311 let prompts = self.state.config.prompts.clone();
2312 let todo: Vec<usize> = self
2313 .state
2314 .candidates
2315 .iter()
2316 .enumerate()
2317 .filter(|(_, c)| {
2318 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2319 && c.failed.is_none()
2320 && !c.empty
2321 })
2322 .map(|(i, _)| i)
2323 .collect();
2324 if todo.is_empty() {
2325 return self.after_implement();
2326 }
2327 self.state.status = RunStatus::Implementing;
2328
2329 let language = self.state.config.graph.language.clone();
2330 let github_language = self.state.config.graph.github_language.clone();
2331 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2332 let sessions = self.state.config.graph.sessions;
2333 let artifacts = agent::artifacts_dir(&self.state.dir());
2334 let brief = self
2338 .state
2339 .advice
2340 .as_ref()
2341 .and_then(|a| a.synthesis.as_deref())
2342 .map(str::to_owned);
2343 let attachments = self.state.attachments.clone();
2344
2345 let mut jobs = Vec::new();
2346 for &i in &todo {
2347 let (index, label, worktree) = {
2348 let c = &self.state.candidates[i];
2349 (c.index, c.label, c.worktree.clone())
2350 };
2351 let spec = self.roles.implementers[index].clone();
2352 let seat_key = format!("impl-{label}");
2353 let seat = self.seat(&seat_key, &spec.id);
2354 let instruction = seeded_instruction(&self.state);
2355 jobs.push(SeatJob {
2356 spec,
2357 seat,
2358 prompt: prompt::implement_in(
2359 &instruction,
2360 &worktree.to_string_lossy(),
2361 &language,
2362 &github_language,
2363 brief.as_deref(),
2364 &attachments,
2365 ),
2366 cwd: worktree,
2367 timeout,
2368 allow_write: true,
2369 sessions,
2370 artifacts: artifacts.clone(),
2371 stem: format!("impl-{label}"),
2372 handover: None,
2373 });
2374 }
2375
2376 self.state.event(
2377 "implement",
2378 format!("{} candidates in parallel", jobs.len()),
2379 );
2380 let mut sent = jobs.clone();
2386 let cache = self.state.config.cache_dir();
2387 let ctx = WaveCtx {
2388 carry_seats: false,
2389 pauses: Some((&self.pause, &self.interrupt)),
2390 run: &run_id,
2391 node: "implement",
2392 prompts: &prompts,
2393 cache: cache.as_deref(),
2394 round: None,
2395 };
2396 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2397 if !wave_blocked(&results) {
2398 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2399 .await;
2400 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2401 .await?;
2402 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2403 .await;
2404 }
2405 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2406
2407 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2408 && results
2409 .iter()
2410 .any(|(_, _, out)| FailClass::of(out).is_some());
2411 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2412 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2413 if self
2417 .state
2418 .candidate_preservation
2419 .contains_key(&self.state.candidates[i].label)
2420 {
2421 self.state.seats.insert(seat.key.clone(), seat);
2422 }
2423 continue;
2424 }
2425 let seat_key = seat.key.clone();
2426 let agent = seat.agent.clone();
2435 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2436 self.state.seats.insert(seat.key.clone(), seat);
2437 let label = self.state.candidates[i].label;
2438 self.state.interrupted_implementations.remove(&label);
2439
2440 let (summary, duration, failed, verified_claim) = match out {
2441 AgentOutcome::Ok(o) => {
2442 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2443 let failed = (!o.usable()).then(|| {
2444 if o.timed_out {
2445 "agent timed out".to_owned()
2446 } else {
2447 format!(
2448 "agent exited with {}",
2449 crate::notices::exit_status(o.exit_code)
2450 )
2451 }
2452 });
2453 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2454 (text, o.duration_ms, failed, verified_claim)
2455 }
2456 AgentOutcome::Dropped(o) => {
2462 let why = o
2463 .dropped
2464 .as_ref()
2465 .map(|d| d.why.as_str())
2466 .unwrap_or("the CLI ended the stream without delivering its answer");
2467 (
2468 String::new(),
2469 o.duration_ms,
2470 Some(format!("the CLI dropped the stream ({why})")),
2471 None,
2472 )
2473 }
2474 AgentOutcome::Quota(o) => {
2475 self.state.quota.push(QuotaLoss {
2476 seat: seat_key,
2477 node: "implement".to_owned(),
2478 at: Timestamp::now(),
2479 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2480 });
2481 (
2482 String::new(),
2483 o.duration_ms,
2484 Some("rate limited (quota); produced no change".to_owned()),
2485 None,
2486 )
2487 }
2488 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2489 (String::new(), 0, Some(e), None)
2490 }
2491 };
2492
2493 let c = &mut self.state.candidates[i];
2496 if !exhausted_the_fallback_chain {
2497 c.agent = agent;
2498 }
2499 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2500 c.duration_ms = duration;
2501 c.failed = failed;
2502 c.verified_noop = verified_claim;
2503 self.state
2504 .candidate_preservation
2505 .insert(label, "awaiting persistence".to_owned());
2506 }
2507 self.state.save()?;
2508 self.persist_candidates().await?;
2509
2510 if resource_blocked {
2511 return self.stop_for_resource();
2512 }
2513 if selected_failed {
2514 self.state.status = RunStatus::Blocked;
2515 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2516 self.state.save()?;
2517 bail!(
2518 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2519 );
2520 }
2521 self.after_implement()
2522 }
2523
2524 async fn persist_candidates(&mut self) -> Result<()> {
2526 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2527 for label in labels {
2528 let i = self
2529 .state
2530 .candidates
2531 .iter()
2532 .position(|c| c.label == label)
2533 .context("pending candidate is missing")?;
2534 let c = self.state.candidates[i].clone();
2535 let interrupted = self.state.interrupted_implementations.contains(&label);
2536 let result = async {
2537 let wait = crate::repo_lock::WaitPolicy::new(
2538 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2539 || false,
2540 );
2541 let _coordination =
2542 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2543 if let Err(e) = git::snapshot_uncommitted(
2546 &c.worktree,
2547 &self
2548 .state
2549 .dir()
2550 .join("artifacts")
2551 .join(format!("candidate-{label}-recovery")),
2552 )
2553 .await
2554 {
2555 self.state.event(
2556 "preserve",
2557 format!("candidate {label} snapshot incomplete: {e:#}"),
2558 );
2559 }
2560 let rescued = git::rescue_commit(
2561 &c.worktree,
2562 &format!("magi: candidate {label} (uncommitted work)"),
2563 )
2564 .await?;
2565 self.state.note_withheld("implement", &rescued.withheld);
2566 let base = &self.state.base_commit;
2567 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2568 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2569 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2570 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2571 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2572 if c.empty && commits > 0 && !patch.trim().is_empty()
2573 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2574 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2575 || self.state.tally.is_some())
2576 {
2577 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2581 }
2582 let c = &mut self.state.candidates[i];
2583 c.commits = commits;
2584 c.stat = stat;
2585 c.files = files;
2586 let no_changes = commits == 0 || patch.trim().is_empty();
2587 c.empty = no_changes && !interrupted;
2588 if interrupted && no_changes {
2589 c.failed = None;
2590 c.verified_noop = None;
2591 }
2592 if !c.empty {
2593 c.failed = None;
2594 c.verified_noop = None;
2595 }
2596 self.state.event(
2597 "preserve",
2598 format!("candidate {label}: {files} files, {commits} commits"),
2599 );
2600 Ok::<_, anyhow::Error>(())
2601 }
2602 .await;
2603 match result {
2604 Ok(()) => {
2605 self.state.candidate_preservation.remove(&label);
2606 if !interrupted || self.state.candidates[i].files > 0 {
2607 self.state.interrupted_implementations.remove(&label);
2608 }
2609 }
2610 Err(e) => {
2611 let reason = format!("candidate {label} preservation failed: {e:#}");
2612 self.state.event("preserve", &reason);
2613 self.state.candidate_preservation.insert(label, reason);
2614 }
2615 }
2616 if self.state.candidate_preservation.is_empty()
2617 && self.state.status == RunStatus::Blocked
2618 && self.state.candidates.iter().all(|c| c.failed.is_none())
2619 {
2620 self.state.status = RunStatus::Implementing;
2624 self.state.event(
2625 "preserve",
2626 "candidate persistence recovered; implementation can continue",
2627 );
2628 }
2629 self.state.save()?;
2630 }
2631 if !self.state.candidate_preservation.is_empty() {
2632 self.state.status = RunStatus::Blocked;
2633 self.state.save()?;
2634 bail!(
2635 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2636 );
2637 }
2638 Ok(())
2639 }
2640
2641 async fn resume_undelivered(
2669 &mut self,
2670 results: &mut [(usize, SeatState, AgentOutcome)],
2671 sent: &[SeatJob],
2672 prompts: &Prompts,
2673 run_id: &str,
2674 ) {
2675 for (wi, seat, out) in results.iter_mut() {
2676 let Some(dropped) = (match &*out {
2677 AgentOutcome::Dropped(o) => o.dropped.clone(),
2678 _ => None,
2679 }) else {
2680 continue;
2681 };
2682 let Some(job) = sent.get(*wi) else { continue };
2683 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2685 self.state.event(
2686 "implement",
2687 format!(
2688 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2689 work is in the tree",
2690 seat.key, dropped.output_tokens, dropped.why
2691 ),
2692 );
2693 continue;
2694 }
2695 if !has_context(&job.spec, seat, job.sessions) {
2703 self.state.event(
2704 "implement",
2705 format!(
2706 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2707 is no session left to resume",
2708 seat.key, dropped.output_tokens, dropped.why
2709 ),
2710 );
2711 continue;
2712 }
2713 self.state.event(
2714 "implement",
2715 format!(
2716 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2717 conversation",
2718 seat.key, dropped.output_tokens, dropped.why
2719 ),
2720 );
2721 let mut retry = job.clone();
2722 retry.seat = seat.clone();
2723 retry.prompt = prompt::resume_after_drop(&dropped.why);
2724 retry.timeout = retry_budget(job.timeout, true);
2725 retry.stem = format!("{}-resume", job.stem);
2726 let cache = self.state.config.cache_dir();
2727 let ctx = WaveCtx {
2728 carry_seats: false,
2729 pauses: Some((&self.pause, &self.interrupt)),
2730 run: run_id,
2731 node: "implement",
2732 prompts,
2733 cache: cache.as_deref(),
2734 round: None,
2735 };
2736 let (resumed_seat, resumed) =
2737 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2738 *seat = resumed_seat;
2739 *out = resumed;
2740 }
2741 }
2742
2743 async fn resume_seat_handovers(
2808 &mut self,
2809 results: &mut [(usize, SeatState, AgentOutcome)],
2810 sent: &mut [SeatJob],
2811 prompts: &Prompts,
2812 run_id: &str,
2813 ) -> Result<()> {
2814 let instruction = seeded_instruction(&self.state);
2815 let language = self.state.config.graph.language.clone();
2816 let github_language = self.state.config.graph.github_language.clone();
2817 let brief = self
2818 .state
2819 .advice
2820 .as_ref()
2821 .and_then(|a| a.synthesis.as_deref())
2822 .map(str::to_owned);
2823 let attachments = self.state.attachments.clone();
2824 for (wi, seat, out) in results.iter_mut() {
2825 let others: BTreeSet<String> = sent
2828 .iter()
2829 .enumerate()
2830 .filter(|(j, _)| j != wi)
2831 .map(|(_, j)| j.spec.id.clone())
2832 .collect();
2833 let Some(job) = sent.get_mut(*wi) else {
2834 continue;
2835 };
2836 let start = self
2841 .roles
2842 .implementer_roster
2843 .iter()
2844 .position(|s| s.id == job.spec.id)
2845 .unwrap_or(0);
2846 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2847 let mut fallback_attempt = 0usize;
2848 let mut prev: Option<FailClass> = None;
2849 while let Some(cur) = FailClass::of(&*out) {
2850 if !should_hand_over(prev.as_ref(), &cur) {
2851 break;
2852 }
2853 let Some(next) =
2854 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2855 .cloned()
2856 else {
2857 break;
2858 };
2859 tried.insert(next.id.clone());
2860 fallback_attempt += 1;
2861
2862 let coordination = self.coordinate(&job.cwd).await?;
2863 if let Ok(r) = git::rescue_commit(
2864 &job.cwd,
2865 &format!(
2866 "magi: candidate {} (uncommitted work before {} fallback)",
2867 seat.key,
2868 if cur == FailClass::Quota {
2869 "quota"
2870 } else {
2871 "handover"
2872 }
2873 ),
2874 )
2875 .await
2876 {
2877 self.state.note_withheld("implement", &r.withheld);
2878 }
2879 drop(coordination);
2880
2881 record_handover(
2882 &mut self.state,
2883 "implement",
2884 &seat.key,
2885 &seat.agent,
2886 &next.id,
2887 &cur,
2888 &fail_reason(&*out),
2889 );
2890 prev = Some(cur.clone());
2891
2892 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2893 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2894 job.spec = next.clone();
2902 let mut retry = job.clone();
2903 retry.seat = new_seat;
2904 retry.prompt = prompt::implement_in(
2905 &instruction,
2906 &job.cwd.to_string_lossy(),
2907 &language,
2908 &github_language,
2909 brief.as_deref(),
2910 &attachments,
2911 );
2912 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2913 let cache = self.state.config.cache_dir();
2914 let ctx = WaveCtx {
2915 carry_seats: false,
2916 pauses: Some((&self.pause, &self.interrupt)),
2917 run: run_id,
2918 node: "implement",
2919 prompts,
2920 cache: cache.as_deref(),
2921 round: None,
2922 };
2923 let (fallback_seat, fallback_out) = run_one(
2924 retry,
2925 Arc::clone(&self.sem),
2926 &ctx,
2927 &mut self.state,
2928 fallback_attempt,
2929 )
2930 .await;
2931 *seat = fallback_seat;
2932 *out = fallback_out;
2933 }
2934 }
2935 Ok(())
2936 }
2937
2938 async fn resume_unconfirmed_commands(
2962 &mut self,
2963 results: &mut [(usize, SeatState, AgentOutcome)],
2964 sent: &[SeatJob],
2965 prompts: &Prompts,
2966 run_id: &str,
2967 ) {
2968 for (wi, seat, out) in results.iter_mut() {
2969 let AgentOutcome::Ok(o) = &*out else {
2970 continue;
2971 };
2972 if !has_unconfirmed_command(&o.commands) {
2973 continue;
2974 }
2975 let Some(job) = sent.get(*wi) else { continue };
2976 if !has_context(&job.spec, seat, job.sessions) {
2977 self.state.event(
2978 "implement",
2979 format!(
2980 "{}: the reply named a command whose own CLI never confirmed the exit \
2981 status of, but there is no session left to resume",
2982 seat.key
2983 ),
2984 );
2985 continue;
2986 }
2987 self.state.event(
2988 "implement",
2989 format!(
2990 "{}: the reply named a command whose own CLI never confirmed the exit \
2991 status of; resuming the conversation",
2992 seat.key
2993 ),
2994 );
2995 let mut retry = job.clone();
2996 retry.seat = seat.clone();
2997 retry.prompt = prompt::resume_incomplete(
2998 "a command in your last reply had no confirmed exit status",
2999 );
3000 retry.timeout = retry_budget(job.timeout, true);
3001 retry.stem = format!("{}-confirm", job.stem);
3002 let cache = self.state.config.cache_dir();
3003 let ctx = WaveCtx {
3004 carry_seats: false,
3005 pauses: Some((&self.pause, &self.interrupt)),
3006 run: run_id,
3007 node: "implement",
3008 prompts,
3009 cache: cache.as_deref(),
3010 round: None,
3011 };
3012 let (resumed_seat, resumed) =
3013 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3014 *seat = resumed_seat;
3015 *out = resumed;
3016 }
3017 }
3018
3019 async fn continue_fix_report(
3040 &mut self,
3041 mut seat: SeatState,
3042 parse_err: String,
3043 job: &SeatJob,
3044 prompts: &Prompts,
3045 run_id: &str,
3046 round: usize,
3047 ) -> (
3048 SeatState,
3049 Option<FixReport>,
3050 Option<String>,
3051 ContinuationRecord,
3052 ) {
3053 let mut last_err = parse_err;
3054 let mut cumulative_wait_ms = 0u64;
3055 let mut attempts = 0usize;
3056 loop {
3057 if !has_context(&job.spec, &seat, job.sessions) {
3058 self.state.event(
3059 "fix",
3060 format!(
3061 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3062 session left to resume into"
3063 ),
3064 );
3065 let outcome = if attempts == 0 {
3066 ContinuationOutcome::NoSession
3067 } else {
3068 ContinuationOutcome::Exhausted
3069 };
3070 return (
3071 seat,
3072 None,
3073 Some(format!("unparsable fix report: {last_err}")),
3074 ContinuationRecord {
3075 attempts,
3076 cumulative_wait_ms,
3077 outcome,
3078 },
3079 );
3080 }
3081 if attempts >= MAX_FIX_CONTINUATIONS {
3082 self.state.event(
3083 "fix",
3084 format!(
3085 "round {round}: fixer's reply still had no adoption report after \
3086 {attempts} continuation(s) ({last_err}); giving up"
3087 ),
3088 );
3089 return (
3090 seat,
3091 None,
3092 Some(format!(
3093 "unparsable fix report after {attempts} continuation(s): {last_err}"
3094 )),
3095 ContinuationRecord {
3096 attempts,
3097 cumulative_wait_ms,
3098 outcome: ContinuationOutcome::Exhausted,
3099 },
3100 );
3101 }
3102 attempts += 1;
3103 self.state.event(
3104 "fix",
3105 format!(
3106 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3107 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3108 ),
3109 );
3110 let mut retry = job.clone();
3111 retry.seat = seat.clone();
3112 retry.prompt = prompt::resume_incomplete(&last_err);
3113 retry.timeout = retry_budget(job.timeout, true);
3114 retry.stem = format!("{}-continue{attempts}", job.stem);
3115 let cache = self.state.config.cache_dir();
3116 let ctx = WaveCtx {
3117 carry_seats: false,
3118 pauses: Some((&self.pause, &self.interrupt)),
3119 run: run_id,
3120 node: "fix",
3121 prompts,
3122 cache: cache.as_deref(),
3123 round: Some(round),
3124 };
3125 let (resumed_seat, resumed_out) = run_one(
3126 retry,
3127 Arc::clone(&self.sem),
3128 &ctx,
3129 &mut self.state,
3130 attempts,
3131 )
3132 .await;
3133 seat = resumed_seat;
3134 match resumed_out {
3135 AgentOutcome::Ok(o) => {
3136 cumulative_wait_ms += o.duration_ms;
3137 match verdict::extract_json::<FixReport>(&o.text) {
3138 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3139 self.state.event(
3140 "fix",
3141 format!(
3142 "round {round}: fixer's adoption report recovered after \
3143 {attempts} continuation(s)"
3144 ),
3145 );
3146 return (
3147 seat,
3148 Some(report),
3149 None,
3150 ContinuationRecord {
3151 attempts,
3152 cumulative_wait_ms,
3153 outcome: ContinuationOutcome::Resumed,
3154 },
3155 );
3156 }
3157 Ok(_) => {
3165 last_err = "the reply parsed, but it reported a command whose own CLI \
3166 never confirmed an exit status"
3167 .to_owned();
3168 }
3169 Err(e) => last_err = e.to_string(),
3170 }
3171 }
3172 AgentOutcome::Quota(o) => {
3173 cumulative_wait_ms += o.duration_ms;
3174 self.state.quota.push(QuotaLoss {
3175 seat: seat.key.clone(),
3176 node: "fix".to_owned(),
3177 at: Timestamp::now(),
3178 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3179 });
3180 self.state.event(
3181 "fix",
3182 format!(
3183 "round {round}: continuation rate limited (quota); not retrying now"
3184 ),
3185 );
3186 return (
3187 seat,
3188 None,
3189 Some("rate limited (quota) while recovering the fix report".to_owned()),
3190 ContinuationRecord {
3191 attempts,
3192 cumulative_wait_ms,
3193 outcome: ContinuationOutcome::QuotaLost,
3194 },
3195 );
3196 }
3197 AgentOutcome::Dropped(o) => {
3198 cumulative_wait_ms += o.duration_ms;
3199 let why = o
3200 .dropped
3201 .as_ref()
3202 .map(|d| d.why.as_str())
3203 .unwrap_or("the CLI ended the stream without delivering its answer");
3204 last_err = format!("the CLI dropped the stream ({why})");
3205 }
3206 AgentOutcome::Failed(e) => last_err = e,
3207 AgentOutcome::ResourceBlocked(e) => {
3210 return (
3211 seat,
3212 None,
3213 Some(format!("the continuation was not started: {e}")),
3214 ContinuationRecord {
3215 attempts: attempts.saturating_sub(1),
3216 cumulative_wait_ms,
3217 outcome: ContinuationOutcome::NoSession,
3218 },
3219 );
3220 }
3221 }
3222 }
3223 }
3224
3225 fn after_implement(&mut self) -> Result<()> {
3226 if !self.state.candidate_preservation.is_empty() {
3227 bail!("candidate preservation incomplete; judging is unsafe");
3228 }
3229 if self.state.leaks.is_empty() {
3231 let cfg = self.state.config.blind.clone();
3232 let mut leaks = Vec::new();
3233 for c in &self.state.candidates {
3234 let Some(patch) =
3235 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3236 else {
3237 continue;
3238 };
3239 leaks.extend(blind::scan(
3240 &format!("candidate {} patch", c.label),
3241 &patch,
3242 &cfg.vendor_tokens,
3243 ));
3244 }
3245 if !leaks.is_empty() {
3246 let summary = leaks
3247 .iter()
3248 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3249 .collect::<Vec<_>>()
3250 .join(", ");
3251 match cfg.on_leak {
3252 LeakPolicy::Fail => {
3253 self.state.status = RunStatus::Failed;
3254 self.state
3255 .event("blind", format!("vendor text in a patch: {summary}"));
3256 self.state.leaks = leaks;
3257 self.state.save()?;
3258 self.settle_questions();
3259 bail!(
3260 "blind.on_leak = \"fail\" and vendor text reached a \
3261 judged patch: {summary}"
3262 );
3263 }
3264 LeakPolicy::Redact => self.state.event(
3265 "blind",
3266 format!("redacting vendor text for judging: {summary}"),
3267 ),
3268 LeakPolicy::Warn => self.state.event(
3269 "blind",
3270 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3271 ),
3272 }
3273 self.state.leaks = leaks;
3274 }
3275 }
3276
3277 if self.state.viable().is_empty() {
3278 if self.state.all_candidates_verified_noop() {
3279 self.state.status = RunStatus::VerifiedNoop;
3290 self.state.save()?;
3291 self.settle_questions();
3292 return Ok(());
3293 }
3294 self.state.status = RunStatus::Failed;
3295 self.state.save()?;
3296 self.settle_questions();
3297 bail!("no candidate produced a change; nothing to judge");
3298 }
3299 self.state.status = RunStatus::Judging;
3300 self.state.save()?;
3301 Ok(())
3302 }
3303
3304 async fn judge(&mut self) -> Result<()> {
3307 let run_id = self.state.id.clone();
3312 let prompts = self.state.config.prompts.clone();
3313 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3314 return Ok(());
3315 }
3316 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3317 if viable.len() == 1 {
3318 self.state.judge_skipped = true;
3325 self.state.event(
3326 "judge",
3327 format!(
3328 "only candidate {} produced a change; judging skipped",
3329 viable[0].label
3330 ),
3331 );
3332 self.state.save()?;
3333 return Ok(());
3334 }
3335 self.state.status = RunStatus::Judging;
3336
3337 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3338 let language = self.state.config.graph.language.clone();
3339 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3340 let sessions = self.state.config.graph.sessions;
3341 let artifacts = agent::artifacts_dir(&self.state.dir());
3342 let root = self.state.worktree_root();
3343 let base_short = short(&self.state.base_commit);
3344
3345 let mut jobs = Vec::new();
3346 let mut orders = Vec::new();
3347 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3348 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3349 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3350 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3351 let seat_key = format!("judge-{}", j + 1);
3352 let seat = self.seat(&seat_key, &spec.id);
3353 jobs.push(SeatJob {
3354 prompt: prompt::judge(
3355 &self.state.instruction,
3356 &views,
3357 self.roles.judges.len(),
3358 &base_short,
3359 &language,
3360 ),
3361 spec,
3362 seat,
3363 cwd: root.join(format!("judge-{}", j + 1)),
3364 timeout,
3365 allow_write: false,
3366 sessions,
3367 artifacts: artifacts.clone(),
3368 stem: format!("judge-{}", j + 1),
3369 handover: None,
3370 });
3371 }
3372
3373 self.state.event(
3374 "judge",
3375 format!(
3376 "{} judges ranking {} candidates blind",
3377 jobs.len(),
3378 viable.len()
3379 ),
3380 );
3381 let labels_for_check = labels.clone();
3382 let mut quota_losses = Vec::new();
3383 let cache = self.state.config.cache_dir();
3384 let ctx = WaveCtx {
3385 carry_seats: false,
3386 pauses: Some((&self.pause, &self.interrupt)),
3387 run: &run_id,
3388 node: "judge",
3389 prompts: &prompts,
3390 cache: cache.as_deref(),
3391 round: None,
3392 };
3393 let results = ask_json_wave::<Ranking>(
3394 jobs,
3395 Arc::clone(&self.sem),
3396 self.state.config.graph.retries,
3397 &self.roles.judge_roster,
3398 &ctx,
3399 &mut quota_losses,
3400 &mut self.state,
3401 &move |r: &Ranking| r.validate(&labels_for_check),
3402 )
3403 .await;
3404 if self.state.resource_block.is_some() {
3405 return self.stop_for_resource();
3406 }
3407 self.state.quota.extend(quota_losses);
3408
3409 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3410 let agent_id = seat.agent.clone();
3411 self.state.seats.insert(seat.key.clone(), seat);
3412 let mut record = Judgement {
3413 judge: j + 1,
3414 seat: format!("judge-{}", j + 1),
3415 agent: agent_id,
3416 ranking: Vec::new(),
3417 reasons: BTreeMap::new(),
3418 confidence: None,
3419 strengths: BTreeMap::new(),
3420 order: orders[j].clone(),
3421 failed: None,
3422 duration_ms: 0,
3423 };
3424 match res {
3425 Ok((ranking, out)) => {
3426 record.ranking = ranking.normalized();
3427 record.reasons = ranking.reasons;
3428 record.strengths = ranking.strengths;
3429 record.confidence = ranking.confidence;
3430 record.duration_ms = out.duration_ms;
3431 self.state.event(
3432 "judge",
3433 format!(
3434 "judge {} ranked {}",
3435 j + 1,
3436 record.ranking.iter().collect::<String>()
3437 ),
3438 );
3439 }
3440 Err(e) => {
3441 record.failed = Some(e.to_string());
3442 self.state
3443 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3444 }
3445 }
3446 self.state.judgements.push(record);
3447 self.state.save()?;
3448 }
3449 Ok(())
3450 }
3451
3452 async fn deliberate(&mut self) -> Result<()> {
3455 let run_id = self.state.id.clone();
3460 let prompts = self.state.config.prompts.clone();
3461 if !self.state.deliberation.is_empty() {
3462 return Ok(());
3463 }
3464 let tops: Vec<char> = self
3465 .state
3466 .judgements
3467 .iter()
3468 .filter_map(|j| j.ranking.first().copied())
3469 .collect();
3470 let rounds = self.state.config.graph.deliberate_rounds;
3471 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3472 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3473 self.state.event(
3474 "deliberate",
3475 format!("judges agreed on {} outright; no deliberation", tops[0]),
3476 );
3477 }
3478 self.state.status = RunStatus::Voting;
3479 self.state.save()?;
3480 return Ok(());
3481 }
3482
3483 self.state.status = RunStatus::Deliberating;
3484 self.state.event(
3485 "deliberate",
3486 format!(
3487 "split: first choices were {} — opening {rounds} round(s)",
3488 tops.iter().collect::<String>()
3489 ),
3490 );
3491
3492 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3493 let language = self.state.config.graph.language.clone();
3494 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3495 let sessions = self.state.config.graph.sessions;
3496 let artifacts = agent::artifacts_dir(&self.state.dir());
3497 let root = self.state.worktree_root();
3498 let base_short = short(&self.state.base_commit);
3499
3500 for round in 1..=rounds {
3504 let mut turns: Vec<DeliberationTurn> = Vec::new();
3505 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3506 if self.state.judgements[j].failed.is_some() {
3507 continue;
3508 }
3509 let seat_key = format!("judge-{}", j + 1);
3510 let spec = self.occupant(&seat_key, spec);
3511 let mut seat = self.seat(&seat_key, &spec.id);
3512 let transcript = self.transcript(&turns, j);
3513 let build = |context: Option<&str>| {
3514 prompt::deliberate(
3515 &self.state.instruction,
3516 context,
3517 &transcript,
3518 round,
3519 rounds,
3520 &language,
3521 )
3522 };
3523 let block = self.candidate_block(&viable, &base_short);
3524 let full = build(Some(&block));
3525 let text = if has_context(&spec, &seat, sessions) {
3526 build(None)
3527 } else {
3528 full.clone()
3529 };
3530 let job = SeatJob {
3531 spec,
3532 seat: seat.clone(),
3533 prompt: text,
3534 cwd: root.join(format!("judge-{}", j + 1)),
3535 timeout,
3536 allow_write: false,
3537 sessions,
3538 artifacts: artifacts.clone(),
3539 stem: format!("delib-{round}-judge-{}", j + 1),
3540 handover: Some(full),
3541 };
3542 let cache = self.state.config.cache_dir();
3543 let ctx = WaveCtx {
3544 carry_seats: false,
3545 pauses: Some((&self.pause, &self.interrupt)),
3546 run: &run_id,
3547 node: "deliberate",
3548 prompts: &prompts,
3549 cache: cache.as_deref(),
3550 round: None,
3551 };
3552 let mut losses = Vec::new();
3556 let mut results = ask_wave_with::<String>(
3557 vec![job],
3558 Arc::clone(&self.sem),
3559 0,
3560 &self.roles.judge_roster,
3561 &ctx,
3562 &mut losses,
3563 &mut self.state,
3564 &|text: &str| {
3565 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3566 },
3567 )
3568 .await;
3569 self.state.quota.extend(losses);
3570 let (updated, res, _) = results.pop().expect("one job in, one result out");
3571 seat = updated;
3572 let agent_id = seat.agent.clone();
3573 self.state.seats.insert(seat.key.clone(), seat);
3574 let body = match res {
3575 Ok((body, _)) => body,
3576 Err(e) => {
3579 self.state
3580 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3581 continue;
3582 }
3583 };
3584 let tentative = verdict::extract_json::<Position>(&body)
3585 .ok()
3586 .and_then(|p| p.tentative)
3587 .and_then(|s| s.trim().chars().next())
3588 .map(|c| c.to_ascii_uppercase());
3589 self.state.event(
3590 "deliberate",
3591 format!(
3592 "round {round}: judge {} now favours {}",
3593 j + 1,
3594 tentative.map_or("—".to_owned(), |c| c.to_string())
3595 ),
3596 );
3597 turns.push(DeliberationTurn {
3598 judge: j + 1,
3599 agent: agent_id,
3600 body: blind::sanitize_prose(&body, &self.state.config.blind),
3601 tentative,
3602 });
3603 }
3604 self.state
3605 .deliberation
3606 .push(DeliberationRound { round, turns });
3607 self.state.save()?;
3608 }
3609
3610 self.state.status = RunStatus::Voting;
3611 self.state.save()?;
3612 Ok(())
3613 }
3614
3615 async fn vote(&mut self) -> Result<()> {
3618 let run_id = self.state.id.clone();
3623 let prompts = self.state.config.prompts.clone();
3624 if !self.state.votes.is_empty() {
3625 return Ok(());
3626 }
3627 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3628 if viable.len() == 1 {
3629 return Ok(());
3630 }
3631 self.state.status = RunStatus::Voting;
3632
3633 let language = self.state.config.graph.language.clone();
3634 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3635 let sessions = self.state.config.graph.sessions;
3636 let artifacts = agent::artifacts_dir(&self.state.dir());
3637 let root = self.state.worktree_root();
3638 let base_short = short(&self.state.base_commit);
3639 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3640
3641 let mut jobs = Vec::new();
3642 let mut seats_at = Vec::new();
3643 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3644 if self
3645 .state
3646 .judgements
3647 .get(j)
3648 .is_some_and(|r| r.failed.is_some())
3649 {
3650 continue;
3651 }
3652 let seat_key = format!("judge-{}", j + 1);
3653 let spec = self.occupant(&seat_key, spec);
3654 let seat = self.seat(&seat_key, &spec.id);
3655 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3656 let text = if has_context(&spec, &seat, sessions) {
3657 prompt::final_vote(&viable, &language)
3658 } else {
3659 full.clone()
3660 };
3661 jobs.push(SeatJob {
3662 spec,
3663 seat,
3664 prompt: text,
3665 cwd: root.join(format!("judge-{}", j + 1)),
3666 timeout,
3667 allow_write: false,
3668 sessions,
3669 artifacts: artifacts.clone(),
3670 stem: format!("vote-judge-{}", j + 1),
3671 handover: Some(full),
3672 });
3673 seats_at.push(j);
3674 }
3675
3676 self.state.event(
3677 "vote",
3678 format!(
3679 "collecting {} final votes one by one, privately",
3680 jobs.len()
3681 ),
3682 );
3683 let allowed = viable.clone();
3684 let mut quota_losses = Vec::new();
3685 let cache = self.state.config.cache_dir();
3686 let ctx = WaveCtx {
3687 carry_seats: false,
3688 pauses: Some((&self.pause, &self.interrupt)),
3689 run: &run_id,
3690 node: "vote",
3691 prompts: &prompts,
3692 cache: cache.as_deref(),
3693 round: None,
3694 };
3695 let results = ask_json_wave::<FinalVote>(
3696 jobs,
3697 Arc::clone(&self.sem),
3698 self.state.config.graph.retries,
3699 &self.roles.judge_roster,
3700 &ctx,
3701 &mut quota_losses,
3702 &mut self.state,
3703 &move |v: &FinalVote| match v.label() {
3704 Some(c) if allowed.contains(&c) => Ok(()),
3705 other => bail!("vote {other:?} is not one of {allowed:?}"),
3706 },
3707 )
3708 .await;
3709 if self.state.resource_block.is_some() {
3710 return self.stop_for_resource();
3711 }
3712 self.state.quota.extend(quota_losses);
3713
3714 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3715 let agent_id = seat.agent.clone();
3716 self.state.seats.insert(seat.key.clone(), seat);
3717 let initial = self
3718 .state
3719 .judgements
3720 .get(j)
3721 .and_then(|r| r.ranking.first().copied());
3722 let mut record = VoteRecord {
3723 judge: j + 1,
3724 agent: agent_id,
3725 vote: None,
3726 reason: String::new(),
3727 changed: false,
3728 };
3729 match res {
3730 Ok((v, _)) => {
3731 record.vote = v.label();
3732 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3733 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3734 self.state.event(
3735 "vote",
3736 format!(
3737 "judge {} voted {}{}",
3738 j + 1,
3739 record.vote.unwrap_or('?'),
3740 if record.changed { " (changed)" } else { "" }
3741 ),
3742 );
3743 }
3744 Err(e) => {
3745 self.state
3746 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3747 }
3748 }
3749 self.state.votes.push(record);
3750 self.state.save()?;
3751 }
3752 Ok(())
3753 }
3754
3755 fn tally(&mut self) -> Result<()> {
3758 if self.state.tally.is_some() {
3759 return Ok(());
3760 }
3761 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3762 let tops: Vec<char> = self
3763 .state
3764 .judgements
3765 .iter()
3766 .filter_map(|j| j.ranking.first().copied())
3767 .collect();
3768 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3769
3770 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3773 let mut cast: Vec<char> = Vec::new();
3774 for (i, j) in self.state.judgements.iter().enumerate() {
3775 let vote = self
3776 .state
3777 .votes
3778 .iter()
3779 .find(|v| v.judge == i + 1)
3780 .and_then(|v| v.vote)
3781 .or_else(|| j.ranking.first().copied());
3782 if let Some(v) = vote {
3783 *first_choice.entry(v).or_insert(0) += 1;
3784 cast.push(v);
3785 }
3786 }
3787
3788 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3789 for j in &self.state.judgements {
3790 let n = j.ranking.len();
3791 for (pos, label) in j.ranking.iter().enumerate() {
3792 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3793 }
3794 }
3795
3796 let best = first_choice.values().copied().max().unwrap_or(0);
3797 let mut leaders: Vec<char> = first_choice
3798 .iter()
3799 .filter(|(_, v)| **v == best)
3800 .map(|(k, _)| *k)
3801 .collect();
3802 let mut tie_break = None;
3803 if leaders.len() > 1 {
3804 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3805 let borda_leaders: Vec<char> = leaders
3806 .iter()
3807 .copied()
3808 .filter(|l| borda[l] == top_borda)
3809 .collect();
3810 tie_break = Some(if borda_leaders.len() == 1 {
3811 format!(
3812 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3813 leaders.len()
3814 )
3815 } else {
3816 format!(
3817 "{} way tie on both first-choice votes and Borda points, broken by label order",
3818 leaders.len()
3819 )
3820 });
3821 leaders = borda_leaders;
3822 leaders.sort_unstable();
3823 }
3824 let winner = *leaders
3825 .first()
3826 .or(viable.first())
3827 .context("no candidate to declare a winner from")?;
3828
3829 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3830 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3831 let deliberated = !self.state.deliberation.is_empty();
3832
3833 let quota_seats: std::collections::BTreeSet<&str> =
3837 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3838 let mut present = 0usize;
3839 for (i, j) in self.state.judgements.iter().enumerate() {
3840 if quota_seats.contains(j.seat.as_str()) {
3841 continue;
3842 }
3843 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3844 let voted = self
3845 .state
3846 .votes
3847 .iter()
3848 .any(|v| v.judge == i + 1 && v.vote.is_some());
3849 if ranked || voted {
3850 present += 1;
3851 }
3852 }
3853 let needs_quorum = viable.len() > 1;
3859 let judges_total = if needs_quorum {
3860 self.roles.judges.len()
3861 } else {
3862 0
3863 };
3864 let quorum = if needs_quorum {
3865 judges_total / 2 + 1
3866 } else {
3867 0
3868 };
3869 let met_quorum = !needs_quorum || present >= quorum;
3870 let uncontested = (!needs_quorum).then(|| {
3871 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3872 });
3873
3874 self.state.event(
3875 "tally",
3876 match &uncontested {
3877 Some(reason) => format!("winner {winner} — {reason}"),
3878 None => format!(
3879 "winner {winner} — votes {} | initial {} | {} changed | \
3880 {present}/{judges_total} judges{}",
3881 first_choice
3882 .iter()
3883 .map(|(k, v)| format!("{k}:{v}"))
3884 .collect::<Vec<_>>()
3885 .join(" "),
3886 if unanimous_initial {
3887 "unanimous"
3888 } else {
3889 "split"
3890 },
3891 changed_votes,
3892 if met_quorum {
3893 String::new()
3894 } else {
3895 format!(" — below quorum ({quorum} required)")
3896 },
3897 ),
3898 },
3899 );
3900 if !met_quorum {
3901 self.state.event(
3902 "stall",
3903 format!(
3904 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3905 the run stops here, resumable"
3906 ),
3907 );
3908 }
3909 self.state.tally = Some(Tally {
3910 first_choice,
3911 borda,
3912 winner,
3913 rankings: tops.len(),
3914 unanimous_initial,
3915 deliberated,
3916 changed_votes,
3917 unanimous_final,
3918 tie_break,
3919 judges: judges_total,
3920 present,
3921 quorum,
3922 met_quorum,
3923 uncontested,
3924 });
3925 self.state.status = if met_quorum {
3926 RunStatus::Reviewing
3927 } else {
3928 RunStatus::Stalled
3929 };
3930 self.state.save()?;
3931 Ok(())
3932 }
3933
3934 #[allow(clippy::too_many_lines)]
3955 async fn recover_stall(&mut self) -> Result<bool> {
3956 let run_id = self.state.id.clone();
3961 let prompts = self.state.config.prompts.clone();
3962 let quota_seats: BTreeSet<&str> =
3967 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3968 let absent: Vec<String> = self
3969 .state
3970 .judgements
3971 .iter()
3972 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3973 .map(|j| j.seat.clone())
3974 .collect();
3975 if absent.is_empty() {
3976 return Ok(false);
3977 }
3978 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3979 if viable.len() <= 1 {
3980 return Ok(false);
3981 }
3982 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983 let language = self.state.config.graph.language.clone();
3984 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985 let sessions = self.state.config.graph.sessions;
3986 let artifacts = agent::artifacts_dir(&self.state.dir());
3987 let root = self.state.worktree_root();
3988 let base_short = short(&self.state.base_commit);
3989 let candidates: Vec<Candidate> = viable.clone();
3990
3991 let mut positions: Vec<usize> = absent
3993 .iter()
3994 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3995 .collect();
3996 if positions.is_empty() {
3997 return Ok(false);
3998 }
3999 positions.sort_unstable();
4000 positions.dedup();
4001
4002 let mut judge_jobs = Vec::new();
4004 for &j in &positions {
4005 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4006 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4007 let seat_key = format!("judge-{}", j + 1);
4008 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4009 let seat = self.seat(&seat_key, &spec.id);
4010 judge_jobs.push(SeatJob {
4011 spec,
4012 seat,
4013 prompt: prompt::judge(
4014 &self.state.instruction,
4015 &views,
4016 self.roles.judges.len(),
4017 &base_short,
4018 &language,
4019 ),
4020 cwd: root.join(seat_key),
4021 timeout,
4022 allow_write: false,
4023 sessions,
4024 artifacts: artifacts.clone(),
4025 stem: format!("judge-{}-recover", j + 1),
4026 handover: None,
4027 });
4028 }
4029
4030 let labels_for_check = labels.clone();
4031 let mut judge_losses = Vec::new();
4032 let retries = self.state.config.graph.retries;
4033 let cache = self.state.config.cache_dir();
4034 let ctx = WaveCtx {
4035 carry_seats: false,
4036 pauses: Some((&self.pause, &self.interrupt)),
4037 run: &run_id,
4038 node: "judge",
4039 prompts: &prompts,
4040 cache: cache.as_deref(),
4041 round: None,
4042 };
4043 let results = ask_json_wave::<Ranking>(
4044 judge_jobs,
4045 Arc::clone(&self.sem),
4046 retries,
4047 &self.roles.judge_roster,
4048 &ctx,
4049 &mut judge_losses,
4050 &mut self.state,
4051 &move |r: &Ranking| r.validate(&labels_for_check),
4052 )
4053 .await;
4054 if self.state.resource_block.is_some() {
4055 self.stop_for_resource()?;
4056 return Ok(false);
4057 }
4058
4059 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4061 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4062 let agent_id = seat.agent.clone();
4063 self.state.seats.insert(seat.key.clone(), seat);
4064 let record = &mut self.state.judgements[j];
4065 match res {
4066 Ok((ranking, out)) => {
4067 record.agent = agent_id;
4068 record.ranking = ranking.normalized();
4069 record.reasons = ranking.reasons;
4070 record.strengths = ranking.strengths;
4071 record.confidence = ranking.confidence;
4072 record.failed = None;
4073 record.duration_ms = out.duration_ms;
4074 recovered.insert(j);
4075 self.state.event(
4076 "recover",
4077 format!("judge {} ranked again after the limit", j + 1),
4078 );
4079 }
4080 Err(e) => {
4081 self.state
4082 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4083 }
4084 }
4085 }
4086
4087 let mut vote_jobs = Vec::new();
4089 let mut vote_pos: Vec<usize> = Vec::new();
4090 for &j in &recovered {
4091 let seat_key = format!("judge-{}", j + 1);
4092 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4093 let seat = self.seat(&seat_key, &spec.id);
4094 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4095 let text = if has_context(&spec, &seat, sessions) {
4096 prompt::final_vote(&labels, &language)
4097 } else {
4098 full.clone()
4099 };
4100 vote_jobs.push(SeatJob {
4101 spec,
4102 seat,
4103 prompt: text,
4104 cwd: root.join(seat_key),
4105 timeout,
4106 allow_write: false,
4107 sessions,
4108 artifacts: artifacts.clone(),
4109 stem: format!("vote-judge-{}-recover", j + 1),
4110 handover: Some(full),
4111 });
4112 vote_pos.push(j);
4113 }
4114 let allowed = labels.clone();
4115 let mut vote_losses = Vec::new();
4116 let vote_retries = self.state.config.graph.retries;
4117 let vote_cache = self.state.config.cache_dir();
4118 let ctx = WaveCtx {
4119 carry_seats: false,
4120 pauses: Some((&self.pause, &self.interrupt)),
4121 run: &run_id,
4122 node: "vote",
4123 prompts: &prompts,
4124 cache: vote_cache.as_deref(),
4125 round: None,
4126 };
4127 let votes = ask_json_wave::<FinalVote>(
4128 vote_jobs,
4129 Arc::clone(&self.sem),
4130 vote_retries,
4131 &self.roles.judge_roster,
4132 &ctx,
4133 &mut vote_losses,
4134 &mut self.state,
4135 &move |v: &FinalVote| match v.label() {
4136 Some(c) if allowed.contains(&c) => Ok(()),
4137 other => bail!("vote {other:?} is not one of {allowed:?}"),
4138 },
4139 )
4140 .await;
4141 if self.state.resource_block.is_some() {
4142 self.stop_for_resource()?;
4143 return Ok(false);
4144 }
4145 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4146 let agent_id = seat.agent.clone();
4147 self.state.seats.insert(seat.key.clone(), seat);
4148 match res {
4149 Ok((v, _)) => {
4150 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4151 rec.vote = v.label();
4152 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4153 } else {
4154 self.state.votes.push(VoteRecord {
4155 judge: j + 1,
4156 agent: agent_id,
4157 vote: v.label(),
4158 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4159 changed: false,
4160 });
4161 }
4162 self.state.event(
4163 "recover",
4164 format!("judge {} voted again after the limit", j + 1),
4165 );
4166 }
4167 Err(e) => {
4168 self.state
4169 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4170 }
4171 }
4172 }
4173
4174 let recovered_keys: BTreeSet<String> = recovered
4178 .iter()
4179 .map(|&j| format!("judge-{}", j + 1))
4180 .collect();
4181 self.state
4182 .quota
4183 .retain(|q| !recovered_keys.contains(&q.seat));
4184 for loss in judge_losses.into_iter().chain(vote_losses) {
4188 if recovered_keys.contains(&loss.seat) {
4189 continue;
4190 }
4191 self.state.quota.retain(|q| q.seat != loss.seat);
4192 self.state.quota.push(loss);
4193 }
4194
4195 self.state.tally = None;
4197 self.tally()?;
4198 Ok(self
4199 .state
4200 .tally
4201 .as_ref()
4202 .map(|t| t.met_quorum)
4203 .unwrap_or(false))
4204 }
4205
4206 async fn graft(&mut self) -> Result<()> {
4222 let Err(e) = self.graft_inner().await else {
4223 return Ok(());
4224 };
4225 if e.is::<crate::repo_lock::ResourceBlocked>() {
4226 return Err(e);
4227 }
4228 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4229 self.state.status = RunStatus::Blocked;
4230 self.state.event("graft", format!("blocked: {e:#}"));
4231 self.state.save()?;
4232 return Err(e);
4233 }
4234 if let Some(rec) = self.state.graft.clone()
4239 && !rec.outcome.settled()
4240 {
4241 if let Some(winner) = self.state.winner().cloned() {
4242 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4243 if re.is::<crate::repo_lock::ResourceBlocked>() {
4244 return Err(re);
4245 }
4246 self.state.status = RunStatus::Blocked;
4247 self.state.event("graft", format!("blocked: {re:#}"));
4248 self.state.save()?;
4249 return Err(re);
4250 }
4251 }
4252 if let Some(g) = self.state.graft.as_mut() {
4253 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4254 }
4255 }
4256 self.state.event("graft", format!("skipped: {e:#}"));
4257 if let Err(e) = self.state.save() {
4258 tracing::warn!("save after graft: {e:#}");
4259 }
4260 Ok(())
4261 }
4262
4263 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4266 let _coordination = self.coordinate(wt).await?;
4267 crate::recovery::restore_repair(wt, pre)
4268 .await
4269 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4270 }
4271
4272 fn graft_skipped(&mut self, why: &str) {
4274 let message = format!("skipped: {why}");
4275 if !self
4276 .state
4277 .events
4278 .iter()
4279 .any(|e| e.node == "graft" && e.message == message)
4280 {
4281 self.state.event("graft", message);
4282 }
4283 }
4284
4285 async fn graft_inner(&mut self) -> Result<()> {
4286 if let Some(rec) = self.state.graft.clone()
4289 && !rec.outcome.settled()
4290 {
4291 if self.state.reviews.is_empty()
4295 && let Some(winner) = self.state.winner().cloned()
4296 {
4297 self.graft_restore(&winner.worktree, &rec.pre).await?;
4298 }
4299 if rec.outcome == GraftOutcome::WaitingForRestore {
4300 self.state.graft = None;
4301 self.state.save()?;
4302 } else {
4303 if let Some(g) = self.state.graft.as_mut() {
4304 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4305 }
4306 self.state.event(
4307 "graft",
4308 "rolled back: the previous round was interrupted; the winner is untouched",
4309 );
4310 self.state.save()?;
4311 return Ok(());
4312 }
4313 }
4314 let plan = match graft::plan(&self.state) {
4315 Ok(plan) => plan,
4316 Err(skip) => {
4317 if skip != graft::Skip::AlreadyDone {
4318 self.graft_skipped(skip.reason());
4319 self.state.save()?;
4320 }
4321 return Ok(());
4322 }
4323 };
4324 let Some(winner) = self.state.winner().cloned() else {
4325 return Ok(());
4326 };
4327 let wt = winner.worktree.clone();
4328 if !git::is_clean(&wt).await? {
4329 self.graft_skipped("the winner's worktree has uncommitted changes");
4330 self.state.save()?;
4331 return Ok(());
4332 }
4333 let pre = git::rev_parse(&wt, "HEAD").await?;
4334 self.state.graft = Some(GraftRecord {
4337 pre: pre.clone(),
4338 post: None,
4339 lines: None,
4340 limit: None,
4341 agent: None,
4342 outcome: GraftOutcome::Running,
4343 requested: graft::requested(&plan),
4344 adoption: None,
4345 adoption_error: None,
4346 });
4347 self.state.event(
4348 "graft",
4349 format!(
4350 "porting strengths of candidate(s) {} onto {}",
4351 plan.strengths
4352 .iter()
4353 .map(|(l, _, _)| l.to_string())
4354 .collect::<Vec<_>>()
4355 .join(", "),
4356 plan.winner
4357 ),
4358 );
4359 self.state.save()?;
4360
4361 let fix_prompt = prompt::graft(
4362 &self.state.instruction,
4363 plan.winner,
4364 &plan.strengths,
4365 &self.state.config.graph.language,
4366 );
4367 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4368 let sessions = self.state.config.graph.sessions;
4369 let artifacts = agent::artifacts_dir(&self.state.dir());
4370 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4371 let (_, seat, out) = self
4372 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4373 SeatJob {
4374 prompt: fix_prompt.clone(),
4375 spec,
4376 seat,
4377 cwd: wt.clone(),
4378 timeout,
4379 allow_write: true,
4380 sessions,
4381 artifacts: artifacts.clone(),
4382 stem: "graft".to_owned(),
4383 handover: None,
4384 }
4385 })
4386 .await;
4387 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4388 self.state.graft.as_mut().expect("graft record").outcome =
4391 GraftOutcome::WaitingForRestore;
4392 return self.stop_for_resource();
4393 }
4394 let mut why: Option<String> = None;
4395 match &out {
4396 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4397 why = Some("the fixer produced nothing usable".to_owned());
4398 }
4399 AgentOutcome::Ok(_) => {}
4400 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4401 AgentOutcome::Quota(o) => {
4402 self.state.quota.push(QuotaLoss {
4403 seat: seat.key.clone(),
4404 node: "graft".to_owned(),
4405 at: Timestamp::now(),
4406 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4407 });
4408 why = Some("rate limited (quota)".to_owned());
4409 }
4410 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4411 why = Some(format!(
4412 "the fixer failed: {}",
4413 e.lines()
4414 .next()
4415 .unwrap_or("")
4416 .chars()
4417 .take(160)
4418 .collect::<String>()
4419 ));
4420 }
4421 }
4422 let agent_id = seat.agent.clone();
4423 self.state.seats.insert(seat.key.clone(), seat);
4424
4425 let mut post = None;
4426 let mut lines = None;
4427 let _coordination = self.coordinate(&wt).await?;
4428 if why.is_none() {
4429 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4430 if let Ok(r) = &rescued {
4431 self.state.note_withheld("graft", &r.withheld);
4432 }
4433 let head = git::rev_parse(&wt, "HEAD").await?;
4434 let dirty = match &rescued {
4438 Ok(r) => git::status_porcelain(&wt)
4439 .await
4440 .map(|st| {
4441 st.lines().any(|l| {
4442 !r.withheld
4443 .iter()
4444 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4445 })
4446 })
4447 .unwrap_or(true),
4448 Err(_) => true,
4449 };
4450 if dirty {
4451 why = Some("the fixer left changes that could not be committed".to_owned());
4452 } else if head == pre {
4453 why = Some("the fixer changed nothing".to_owned());
4454 } else {
4455 lines = git::changed_lines(&wt, &pre, &head).await;
4456 post = Some(head);
4457 }
4458 }
4459 let (adoption, adoption_error) = match &out {
4460 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4461 _ => (None, None),
4462 };
4463 let requested = graft::requested(&plan);
4464 let outcome = match why {
4465 None => {
4466 self.state.event(
4467 "graft",
4468 format!(
4469 "applied: {}",
4470 lines.map_or_else(
4471 || "changed lines unavailable".to_owned(),
4472 |n| format!("{n} changed lines")
4473 )
4474 ),
4475 );
4476 GraftOutcome::Applied
4477 }
4478 Some(why) => {
4479 crate::recovery::restore_repair(&wt, &pre)
4482 .await
4483 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4484 self.state.event(
4485 "graft",
4486 format!("rolled back: {why}; the winner is untouched"),
4487 );
4488 GraftOutcome::RolledBack(why)
4489 }
4490 };
4491 self.state.graft = Some(GraftRecord {
4492 pre,
4493 post,
4494 lines,
4495 limit: None,
4496 agent: Some(agent_id),
4497 outcome,
4498 requested,
4499 adoption,
4500 adoption_error,
4501 });
4502 self.state.save()?;
4503 Ok(())
4504 }
4505
4506 async fn fold_losers(&mut self) -> Result<()> {
4509 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4510 return Ok(());
4511 };
4512 if !self.state.candidate_preservation.is_empty() {
4513 bail!("candidate preservation incomplete; refusing cleanup");
4514 }
4515 let repo = self.state.repo.clone();
4516 let _coordination = self.coordinate(&repo).await?;
4517 let mut folded = Vec::new();
4518 for i in 0..self.state.candidates.len() {
4519 let c = &self.state.candidates[i];
4520 if c.label == winner || c.folded {
4521 continue;
4522 }
4523 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4524 remove_preserved_worktree(&repo, &wt).await?;
4525 if git::branch_exists(&repo, &branch).await?
4526 && !git::branch_delete(&repo, &branch).await?
4527 {
4528 bail!("candidate branch removal refused");
4529 }
4530 self.state.candidates[i].folded = true;
4531 self.state.save()?;
4532 folded.push(label.to_string());
4533 }
4534 let root = self.state.worktree_root();
4536 for j in 1..=self.roles.judges.len() {
4537 let wt = root.join(format!("judge-{j}"));
4538 if wt.exists() {
4539 remove_preserved_worktree(&repo, &wt).await?;
4540 }
4541 }
4542 if self.state.config.graph.advise {
4545 for k in 1..=self.state.config.graph.advisors {
4546 let wt = root.join(format!("advisor-{k}"));
4547 if wt.exists() {
4548 remove_preserved_worktree(&repo, &wt).await?;
4549 }
4550 }
4551 }
4552 if !folded.is_empty() {
4553 self.state
4554 .event("fold", format!("folded candidates {}", folded.join(", ")));
4555 self.state.save()?;
4556 }
4557 Ok(())
4558 }
4559
4560 async fn sync_to_base(&mut self) -> Result<()> {
4600 if self.state.status == RunStatus::AlreadyInBase {
4601 return Ok(());
4602 }
4603 let conflicted = self
4604 .state
4605 .base_sync
4606 .as_ref()
4607 .is_some_and(|s| s.conflict.is_some());
4608 let Some(winner) = self.state.winner().cloned() else {
4609 return Ok(());
4610 };
4611
4612 let repo = self.state.repo.clone();
4613 let remote = self.state.config.merge.remote.clone();
4614 let base_branch = self.state.base_branch.clone();
4615 let tracking = format!("{remote}/{base_branch}");
4616
4617 git::fetch(&repo, &remote, &base_branch).await.ok();
4618 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4623 return Ok(());
4624 };
4625
4626 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4627 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4628 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4629
4630 if (behind > 0 || conflicted || head == tip)
4639 && self
4640 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4641 .await?
4642 {
4643 return Ok(());
4644 }
4645 if conflicted {
4646 return Ok(());
4647 }
4648
4649 if behind == 0 {
4650 if let Some(from) = self
4659 .state
4660 .rebase_fixes
4661 .iter()
4662 .rev()
4663 .find_map(|r| r.from.clone())
4664 && from != head
4665 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4666 .await
4667 .is_ok_and(|o| o.ok())
4668 {
4669 git::sync_to_head(&winner.worktree).await?;
4670 }
4671 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4677 if let Some(why) = &conflict {
4678 self.state.status = RunStatus::Blocked;
4679 self.state.event("land", why.clone());
4680 }
4681 self.state.base_sync = Some(BaseSync {
4682 tip,
4683 behind: 0,
4684 attempts,
4685 conflict,
4686 already_in: None,
4687 });
4688 self.state.save()?;
4689 return Ok(());
4690 }
4691
4692 if attempts >= BASE_SYNC_ROUNDS {
4693 let why = format!(
4694 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4695 rebase(s); rebasing again would only race it",
4696 winner.branch
4697 );
4698 self.state.status = RunStatus::Blocked;
4699 self.state.base_sync = Some(BaseSync {
4700 tip,
4701 behind,
4702 attempts,
4703 conflict: Some(why.clone()),
4704 already_in: None,
4705 });
4706 self.state.event("land", why);
4707 self.state.save()?;
4708 return Ok(());
4709 }
4710
4711 self.state.event(
4712 "land",
4713 format!(
4714 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4715 winner.branch
4716 ),
4717 );
4718 self.state.save()?;
4719
4720 let branch_tracking = format!("{remote}/{}", winner.branch);
4726 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4727 let remote_tip = if self.state.task_pr_branch().is_none()
4728 && matches!(&fetched_branch, Ok(o) if o.ok())
4729 {
4730 git::rev_parse(&repo, &branch_tracking).await.ok()
4731 } else {
4732 None
4733 };
4734 if let Some(theirs) = &remote_tip
4738 && !git::is_ancestor(&repo, theirs, &head).await
4739 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4740 .await
4741 .is_ok_and(|missing| missing.is_empty())
4742 {
4743 let why = format!(
4744 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4745 them",
4746 short(theirs),
4747 winner.branch
4748 );
4749 self.state.status = RunStatus::Blocked;
4750 self.state.base_sync = Some(BaseSync {
4751 tip,
4752 behind,
4753 attempts,
4754 conflict: Some(why.clone()),
4755 already_in: None,
4756 });
4757 self.state.event("land", why);
4758 self.state.save()?;
4759 return Ok(());
4760 }
4761
4762 let scratch = self.state.dir().join("base-sync");
4763 let rebased = match crate::rebase::rebase_with_fixer(
4764 &mut self.state,
4765 &scratch,
4766 &winner.branch,
4767 &tracking,
4768 )
4769 .await
4770 {
4771 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4772 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4773 Err(e) => Err(e),
4774 };
4775 let attempts = attempts + 1;
4776 match rebased {
4777 Ok(None) => {
4778 git::sync_to_head(&winner.worktree).await?;
4782 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4783 let mut conflict = None;
4784 if let Some(pinned) = &remote_tip {
4785 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4786 match pushed {
4787 Ok(o) if o.ok() => self.state.event(
4788 "land",
4789 format!("pushed rebased {} to {remote}", winner.branch),
4790 ),
4791 Ok(o) => {
4792 conflict = Some(format!(
4793 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4794 winner.branch,
4795 short(pinned),
4796 o.stderr.chars().take(600).collect::<String>()
4797 ));
4798 }
4799 Err(e) => {
4800 conflict = Some(format!(
4801 "rebased {} locally but could not push it: {e:#}",
4802 winner.branch
4803 ));
4804 }
4805 }
4806 }
4807 if let Some(why) = &conflict {
4808 self.state.status = RunStatus::Blocked;
4809 self.state.event("land", why.clone());
4810 }
4811 self.state.base_sync = Some(BaseSync {
4812 tip: tip.clone(),
4813 behind: 0,
4814 attempts,
4815 conflict,
4816 already_in: None,
4817 });
4818 self.state
4819 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4820 }
4821 Ok(Some(conflict)) => {
4822 let why = format!(
4823 "{} conflicts with {tracking} and did not rebase: {}",
4824 winner.branch,
4825 conflict.chars().take(600).collect::<String>()
4826 );
4827 self.state.status = RunStatus::Blocked;
4828 self.state.base_sync = Some(BaseSync {
4829 tip,
4830 behind,
4831 attempts,
4832 conflict: Some(why.clone()),
4833 already_in: None,
4834 });
4835 self.state.event("land", why);
4836 }
4837 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4841 self.state.save()?;
4842 return Ok(());
4843 }
4844 Err(e) => {
4845 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4846 self.state.status = RunStatus::Blocked;
4847 self.state.base_sync = Some(BaseSync {
4848 tip,
4849 behind,
4850 attempts,
4851 conflict: Some(why.clone()),
4852 already_in: None,
4853 });
4854 self.state.event("land", why);
4855 }
4856 }
4857 self.state.save()?;
4858 Ok(())
4859 }
4860
4861 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4867 if self.state.task_pr_branch().is_some() {
4870 return None;
4871 }
4872 let repo = self.state.repo.clone();
4873 let remote = self.state.config.merge.remote.clone();
4874 let fetched = git::fetch(&repo, &remote, branch).await;
4875 if !matches!(&fetched, Ok(o) if o.ok()) {
4876 return None;
4877 }
4878 let branch_tracking = format!("{remote}/{branch}");
4879 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4880 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4881 return None;
4882 }
4883 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4884 .await
4885 .is_ok_and(|missing| missing.is_empty())
4886 {
4887 return Some(format!(
4888 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4889 them",
4890 short(&theirs)
4891 ));
4892 }
4893 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4894 Ok(o) if o.ok() => {
4895 self.state
4896 .event("land", format!("pushed rebased {branch} to {remote}"));
4897 None
4898 }
4899 Ok(o) => Some(format!(
4900 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4901 someone may have pushed): {}",
4902 short(&theirs),
4903 o.stderr.chars().take(600).collect::<String>()
4904 )),
4905 Err(e) => Some(format!(
4906 "{branch} is rebased locally but could not be pushed: {e:#}"
4907 )),
4908 }
4909 }
4910
4911 async fn settle_already_in(
4928 &mut self,
4929 branch: &str,
4930 tip: &str,
4931 head: &str,
4932 attempts: usize,
4933 behind: usize,
4934 ) -> Result<bool> {
4935 let repo = self.state.repo.clone();
4936 let remote = self.state.config.merge.remote.clone();
4937 let start = self.state.base_commit.clone();
4938 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4939 Ok(Some(e)) => e,
4940 Ok(None) => return Ok(false),
4941 Err(e) => {
4942 tracing::warn!("already-in-base check for {branch}: {e:#}");
4943 return Ok(false);
4944 }
4945 };
4946 let mut verified = vec![head.to_owned()];
4947 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4948 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4949 if matches!(&fetched, Ok(o) if o.ok())
4950 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4951 && theirs != head
4952 {
4953 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4954 Ok(Some(_)) => verified.push(theirs),
4955 _ => return Ok(false),
4956 }
4957 }
4958 let base_branch = self.state.base_branch.clone();
4959 let message = format!(
4960 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4961 land",
4962 evidence.names(),
4963 evidence.proof.as_str()
4964 );
4965 let closed =
4966 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4967 .await;
4968 match closed {
4969 Ok(Ok(url)) => self
4970 .state
4971 .event("land", format!("closed {url}: superseded on {base_branch}")),
4972 Ok(Err(why)) => self
4973 .state
4974 .event("land", format!("did not close a pull request: {why}")),
4975 Err(e) => {
4976 let why = format!(
4977 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4978 not be closed ({e:#}); resume to retry"
4979 );
4980 self.state.status = RunStatus::Blocked;
4981 self.state.base_sync = Some(BaseSync {
4982 tip: tip.to_owned(),
4983 behind,
4984 attempts,
4985 conflict: Some(why.clone()),
4986 already_in: None,
4987 });
4988 self.state.event("land", why);
4989 self.state.save()?;
4990 return Ok(true);
4991 }
4992 }
4993 self.state.status = RunStatus::AlreadyInBase;
4994 self.state.base_sync = Some(BaseSync {
4995 tip: tip.to_owned(),
4996 behind,
4997 attempts,
4998 conflict: None,
4999 already_in: Some(evidence),
5000 });
5001 self.state.event("land", message);
5002 self.state.save()?;
5003 self.settle_questions();
5004 Ok(true)
5005 }
5006
5007 fn landing_base(&self) -> String {
5017 self.state
5018 .base_sync
5019 .as_ref()
5020 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5021 }
5022
5023 pub async fn fix_selected(
5056 &mut self,
5057 ids: &[String],
5058 reason: &str,
5059 allow_stale: bool,
5060 ) -> Result<()> {
5061 let reason = reason.trim();
5062 if reason.is_empty() {
5063 bail!("a fix request needs a reason — that is the operator's own record of why");
5064 }
5065 if ids.is_empty() {
5066 bail!("no finding id given");
5067 }
5068 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5069 bail!(
5070 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5071 has already concluded — can be given a targeted fix. A run still \
5072 in progress should simply be resumed; a `merged` run's branch has \
5073 already landed, so its answer is a fresh `magi review <branch>`, \
5074 not reopening this run's own record",
5075 self.state.id,
5076 self.state.status.as_str()
5077 );
5078 }
5079 let Some(winner) = self.state.winner().cloned() else {
5080 bail!("run {} has no winning candidate to fix", self.state.id);
5081 };
5082 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5083 bail!(
5084 "branch `{}` no longer exists; this run cannot be extended",
5085 winner.branch
5086 );
5087 }
5088 let home = crate::run::home();
5089 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5090 bail!(
5091 "run {} is currently being worked on by another magi process",
5092 self.state.id
5093 );
5094 }
5095 let _claim = FixClaim::acquire(&self.state.dir())?;
5101
5102 let mut seen = BTreeSet::new();
5106 let mut findings = Vec::new();
5107 let mut missing = Vec::new();
5108 for id in ids {
5109 if !seen.insert(id.clone()) {
5110 continue;
5111 }
5112 match self.state.finding(id) {
5113 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5114 id: f.id.clone(),
5115 severity: f.severity,
5116 reviewer_vote: rec.vote,
5117 round: round.round,
5118 round_head: round.head.clone(),
5119 reviewer: rec.reviewer,
5120 agent: rec.agent.clone(),
5121 file: f.file.clone(),
5122 line: f.line,
5123 title: f.title.clone(),
5124 detail: f.detail.clone(),
5125 outcome: OperatorFixOutcome::Pending,
5126 }),
5127 None => missing.push(id.clone()),
5128 }
5129 }
5130 if !missing.is_empty() {
5131 bail!(
5132 "unknown finding id(s): {}; nothing was changed",
5133 missing.join(", ")
5134 );
5135 }
5136
5137 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5138 let stale_details: Vec<(String, String)> = findings
5139 .iter()
5140 .filter(|f| f.round_head != head_at_request)
5141 .map(|f| (f.id.clone(), f.round_head.clone()))
5142 .collect();
5143 let stale = !stale_details.is_empty();
5144 if stale && !allow_stale {
5145 bail!(
5146 "the branch has moved since some finding(s) were raised — {} — now \
5147 at {}; pass --allow-stale to fix anyway, or re-run review first",
5148 stale_details
5149 .iter()
5150 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5151 .collect::<Vec<_>>()
5152 .join(", "),
5153 short(&head_at_request)
5154 );
5155 }
5156
5157 let request = OperatorFixRequest {
5158 requested_at: Timestamp::now(),
5159 reason: reason.to_owned(),
5160 findings,
5161 head_at_request: head_at_request.clone(),
5162 allow_stale,
5163 stale,
5164 fix: None,
5165 result_head: None,
5166 follow_up_review_run: None,
5167 };
5168 self.state.event(
5169 "fix",
5170 format!(
5171 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5172 request.findings.len(),
5173 request
5174 .findings
5175 .iter()
5176 .map(|f| f.id.as_str())
5177 .collect::<Vec<_>>()
5178 .join(", "),
5179 ),
5180 );
5181 self.state.operator_fixes.push(request);
5188 self.state.save()?;
5189 let request_index = self.state.operator_fixes.len() - 1;
5190
5191 let repo = self.state.repo.clone();
5200 let coordination = self.coordinate(&repo).await?;
5201 if winner.worktree.exists() {
5202 let dirty = git::git(
5205 &winner.worktree,
5206 &["status", "--porcelain", "--untracked-files=all"],
5207 )
5208 .await?;
5209 let only_withheld = dirty.lines().all(|l| {
5210 l.strip_prefix("?? ")
5211 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5212 });
5213 if !only_withheld {
5214 bail!(
5215 "`{}` has uncommitted changes; refusing to touch it — commit or \
5216 discard them first",
5217 winner.worktree.display()
5218 );
5219 }
5220 git::worktree_remove(&self.state.repo, &winner.worktree)
5221 .await
5222 .ok();
5223 }
5224 let fix_worktree = self.state.worktree_root().join("operator-fix");
5225 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5226 git::git(
5227 &self.state.repo,
5228 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5229 )
5230 .await
5231 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5232 if !git::is_clean(&fix_worktree).await? {
5233 git::worktree_remove(&self.state.repo, &fix_worktree)
5234 .await
5235 .ok();
5236 bail!(
5237 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5238 winner.branch
5239 );
5240 }
5241 let repo_for_setup = self.state.repo.clone();
5242 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5243 Ok(()) => {
5244 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5245 }
5246 Err(e) => Err(e),
5247 };
5248 if let Err(e) = setup {
5249 git::worktree_remove(&self.state.repo, &fix_worktree)
5250 .await
5251 .ok();
5252 return Err(e);
5253 }
5254
5255 let run_id = self.state.id.clone();
5256 let prompts = self.state.config.prompts.clone();
5257 let language = self.state.config.graph.language.clone();
5258 let sessions = self.state.config.graph.sessions;
5259 let artifacts = agent::artifacts_dir(&self.state.dir());
5260 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5261 .findings
5262 .iter()
5263 .map(|f| Finding {
5264 id: f.id.clone(),
5265 severity: f.severity,
5266 file: f.file.clone(),
5267 line: f.line,
5268 title: f.title.clone(),
5269 detail: f.detail.clone(),
5270 })
5271 .collect();
5272 let fix_prompt = prompt::operator_fix_in(
5273 &self.state.instruction,
5274 &finding_list,
5275 reason,
5276 &stale_details,
5277 &head_at_request,
5278 &language,
5279 &self.state.config.graph.github_language,
5280 );
5281 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5282 drop(coordination);
5283 let (job, seat, out) = self
5284 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5285 prompt: fix_prompt.clone(),
5286 spec,
5287 seat,
5288 cwd: fix_worktree.clone(),
5289 timeout,
5290 allow_write: true,
5291 sessions,
5292 artifacts: artifacts.clone(),
5293 stem: "operator-fix".to_owned(),
5294 handover: None,
5295 })
5296 .await;
5297 let agent_id = seat.agent.clone();
5298
5299 let mut fix = FixRecord {
5300 agent: agent_id,
5301 addressed: Vec::new(),
5302 rejected: Vec::new(),
5303 notes: String::new(),
5304 committed: false,
5305 failed: None,
5306 duration_ms: 0,
5307 continuation: None,
5308 };
5309 let mut final_seat = seat.clone();
5310 match out {
5311 AgentOutcome::Ok(o) => {
5312 fix.duration_ms = o.duration_ms;
5313 let parsed = verdict::extract_json::<FixReport>(&o.text);
5314 let incomplete_reason = match &parsed {
5315 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5316 "the reply parsed, but it reported a command whose own CLI \
5317 never confirmed an exit status"
5318 .to_owned(),
5319 ),
5320 Ok(_) => None,
5321 Err(e) => Some(e.to_string()),
5322 };
5323 match incomplete_reason {
5324 None => {
5325 let report = parsed.expect("checked Ok above");
5326 fix.addressed = report.addressed;
5327 fix.rejected = report.rejected;
5328 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5329 }
5330 Some(reason) => {
5331 let (resumed_seat, resolved, failure, cont) = self
5332 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5333 .await;
5334 fix.duration_ms += cont.cumulative_wait_ms;
5335 fix.continuation = Some(cont);
5336 final_seat = resumed_seat;
5337 match resolved {
5338 Some(report) => {
5339 fix.addressed = report.addressed;
5340 fix.rejected = report.rejected;
5341 fix.notes =
5342 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5343 }
5344 None => fix.failed = failure,
5345 }
5346 }
5347 }
5348 }
5349 AgentOutcome::Dropped(o) => {
5350 fix.duration_ms = o.duration_ms;
5351 let why = o
5352 .dropped
5353 .as_ref()
5354 .map(|d| d.why.as_str())
5355 .unwrap_or("the CLI ended the stream without delivering its answer");
5356 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5357 }
5358 AgentOutcome::Quota(o) => {
5359 self.state.quota.push(QuotaLoss {
5360 seat: final_seat.key.clone(),
5361 node: "fix".to_owned(),
5362 at: Timestamp::now(),
5363 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5364 });
5365 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5366 }
5367 AgentOutcome::Failed(e) => fix.failed = Some(e),
5368 AgentOutcome::ResourceBlocked(e) => {
5369 fix.failed = Some(format!("the fixer was not started: {e}"));
5370 }
5371 }
5372 if fix.continuation.is_none() {
5373 fix.continuation = Some(ContinuationRecord::not_needed());
5374 }
5375 self.state.seats.insert(final_seat.key.clone(), final_seat);
5376
5377 let rescue_message = format!(
5378 "magi: operator-selected fix ({}) (uncommitted work)",
5379 self.state.operator_fixes[request_index]
5380 .findings
5381 .iter()
5382 .map(|f| f.id.as_str())
5383 .collect::<Vec<_>>()
5384 .join(", ")
5385 );
5386 let repo = self.state.repo.clone();
5387 let coordination = self.coordinate(&repo).await?;
5388 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5389 self.state.note_withheld("fix", &r.withheld);
5390 }
5391 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5392 fix.committed = after != head_at_request;
5393 git::worktree_remove(&self.state.repo, &fix_worktree)
5394 .await
5395 .ok();
5396 drop(coordination);
5397
5398 self.state.event(
5399 "fix",
5400 match &fix.failed {
5401 Some(reason) => format!(
5402 "operator fix: adoption report was lost ({reason}); {}",
5403 if fix.committed {
5404 "committed"
5405 } else {
5406 "NO new commit"
5407 }
5408 ),
5409 None => format!(
5410 "operator fix: {} addressed, {} rejected, {}",
5411 fix.addressed.len(),
5412 fix.rejected.len(),
5413 if fix.committed {
5414 "committed"
5415 } else {
5416 "NO new commit"
5417 }
5418 ),
5419 },
5420 );
5421
5422 for f in &mut self.state.operator_fixes[request_index].findings {
5429 f.outcome = if fix.failed.is_some() {
5430 OperatorFixOutcome::Unreported
5431 } else if fix.addressed.contains(&f.id) {
5432 OperatorFixOutcome::Addressed
5433 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5434 OperatorFixOutcome::Rejected { why: r.why.clone() }
5435 } else {
5436 OperatorFixOutcome::Unreported
5437 };
5438 }
5439
5440 let committed = fix.committed;
5441 if committed {
5442 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5443 }
5444 self.state.operator_fixes[request_index].fix = Some(fix);
5445 self.state.save()?;
5448
5449 if committed {
5450 self.state.event(
5451 "fix",
5452 format!(
5453 "operator fix committed {}; opening a follow-up review-only run",
5454 short(&after)
5455 ),
5456 );
5457 let origin =
5460 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5461 match Self::review(
5462 &self.state.repo,
5463 &winner.branch,
5464 self.state.config.clone(),
5465 origin,
5466 )
5467 .await
5468 {
5469 Ok(mut follow_up) => {
5470 follow_up.state.event(
5471 "start",
5472 format!(
5473 "requested by an operator fix on run {} for finding(s) {}",
5474 self.state.id,
5475 self.state.operator_fixes[request_index]
5476 .findings
5477 .iter()
5478 .map(|f| f.id.as_str())
5479 .collect::<Vec<_>>()
5480 .join(", "),
5481 ),
5482 );
5483 follow_up.state.save()?;
5484 let follow_up_id = follow_up.state.id.clone();
5485 let adopted = match crate::direct::adopt(
5488 &follow_up.state,
5489 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5490 ) {
5491 Ok(a) => a,
5492 Err(e) => {
5493 self.state.event(
5496 "fix",
5497 format!(
5498 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5499 ),
5500 );
5501 self.state.save()?;
5502 return Ok(());
5503 }
5504 };
5505 let executed = follow_up.execute().await;
5506 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5507 if let Some(a) = adopted {
5508 a.finish(&follow_up.state, executed);
5509 }
5510 if let Some(e) = failure {
5511 self.state.event(
5512 "fix",
5513 format!(
5514 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5515 ),
5516 );
5517 }
5518 self.state.operator_fixes[request_index].follow_up_review_run =
5519 Some(follow_up_id);
5520 }
5521 Err(e) => {
5522 self.state.event(
5523 "fix",
5524 format!("committed the fix but could not open a follow-up review: {e:#}"),
5525 );
5526 }
5527 }
5528 self.state.save()?;
5529 }
5530
5531 Ok(())
5532 }
5533
5534 async fn ask_fixer(
5554 &mut self,
5555 winner: &Candidate,
5556 node: &'static str,
5557 round: Option<usize>,
5558 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5559 ) -> (SeatJob, SeatState, AgentOutcome) {
5560 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5561 self.ask_fixer_among(attempts, node, round, None, build)
5562 .await
5563 }
5564
5565 async fn ask_fixer_among(
5570 &mut self,
5571 attempts: Vec<(AgentSpec, String)>,
5572 node: &'static str,
5573 round: Option<usize>,
5574 restore: Option<(&Path, &str)>,
5575 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5576 ) -> (SeatJob, SeatState, AgentOutcome) {
5577 let run_id = self.state.id.clone();
5578 let prompts = self.state.config.prompts.clone();
5579 let cache = self.state.config.cache_dir();
5580 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5581 let mut last = None;
5582 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5583 let seat = self.seat(&key, &spec.id);
5584 let mut job = build(spec.clone(), seat);
5585 if i > 0 {
5586 job.stem = format!("{}-{}", job.stem, spec.id);
5587 }
5588 let ctx = WaveCtx {
5589 carry_seats: false,
5590 pauses: Some((&self.pause, &self.interrupt)),
5591 run: &run_id,
5592 node,
5593 prompts: &prompts,
5594 cache: cache.as_deref(),
5595 round,
5596 };
5597 let (seat, out) =
5598 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5599 let advances = match &out {
5602 AgentOutcome::Ok(o) => agent::output_advances(o),
5603 AgentOutcome::ResourceBlocked(_) => false,
5604 _ => true,
5605 };
5606 if let Some(next) = ids.get(i + 1)
5607 && advances
5608 {
5609 self.state.seats.insert(seat.key.clone(), seat.clone());
5610 let class =
5611 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5612 let reason = match &out {
5613 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5614 other => fail_reason(other),
5615 };
5616 if let Some((wt, rev)) = restore
5620 && let Err(e) = self.graft_restore(wt, rev).await
5621 {
5622 let why =
5623 format!("could not restore the worktree before the next fixer: {e:#}");
5624 tracing::warn!("{why}");
5625 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5626 AgentOutcome::ResourceBlocked(why)
5627 } else {
5628 AgentOutcome::Failed(why)
5629 };
5630 last = Some((job, seat, out));
5631 break;
5632 }
5633 record_handover(
5634 &mut self.state,
5635 node,
5636 &seat.key,
5637 &spec.id,
5638 next,
5639 &class,
5640 &reason,
5641 );
5642 continue;
5643 }
5644 last = Some((job, seat, out));
5645 break;
5646 }
5647 last.expect("the fixer chain always has an entry")
5648 }
5649
5650 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5661 use crate::recovery::{Mode, Repair};
5662 if !self.state.reviews.is_empty() {
5663 return Ok(());
5664 }
5665 let Some(mut rec) = self.state.recovery.clone() else {
5666 return Ok(());
5667 };
5668 if rec.mode != Mode::VerifyRepair {
5669 return Ok(());
5670 }
5671 let wt = winner.worktree.clone();
5672 let mut head = git::rev_parse(&wt, "HEAD").await?;
5673 match rec.repair.clone() {
5674 Repair::Done { .. } => return Ok(()),
5675 Repair::WaitingForRestore { before } => {
5676 self.graft_restore(&wt, &before).await?;
5677 head = before;
5678 rec.repair = Repair::NotStarted;
5679 self.state.recovery = Some(rec.clone());
5680 self.state.save()?;
5681 }
5682 Repair::Started { before } => {
5683 if let Err(e) = self.graft_restore(&wt, &before).await {
5687 if e.is::<crate::repo_lock::ResourceBlocked>() {
5688 return Err(e);
5689 }
5690 return self
5691 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5692 .await;
5693 }
5694 rec.repair = Repair::Done {
5695 before: before.clone(),
5696 after: before,
5697 committed: false,
5698 failed: Some("interrupted; rolled back".to_owned()),
5699 };
5700 self.state
5701 .event("recovery", "the repair was interrupted and rolled back");
5702 self.state.recovery = Some(rec);
5703 self.state.save()?;
5704 return Ok(());
5705 }
5706 Repair::NotStarted => {}
5707 }
5708 if crate::recovery::recheck(&mut rec, &head) {
5709 self.state.event("recovery", rec.summary());
5710 self.state.recovery = Some(rec);
5711 self.state.save()?;
5712 return Ok(());
5713 }
5714 rec.repair = Repair::Started {
5715 before: head.clone(),
5716 };
5717 self.state.recovery = Some(rec.clone());
5718 self.state.save()?;
5719
5720 let language = self.state.config.graph.language.clone();
5721 let sessions = self.state.config.graph.sessions;
5722 let artifacts = agent::artifacts_dir(&self.state.dir());
5723 let fix_prompt = prompt::recovery_fix(
5724 &self.state.instruction,
5725 &rec,
5726 self.state.config.graph.review_rounds,
5727 &language,
5728 );
5729 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5730 self.state.event(
5731 "recovery",
5732 format!(
5733 "repair: the fixer starts from the failed verification on {}",
5734 short(&head)
5735 ),
5736 );
5737 let (_job, seat, out) = self
5738 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5739 prompt: fix_prompt.clone(),
5740 spec,
5741 seat,
5742 cwd: wt.clone(),
5743 timeout,
5744 allow_write: true,
5745 sessions,
5746 artifacts: artifacts.clone(),
5747 stem: "fix-recovery".to_owned(),
5748 handover: None,
5749 })
5750 .await;
5751 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5752 rec.repair = Repair::WaitingForRestore { before: head };
5753 self.state.recovery = Some(rec);
5754 return self.stop_for_resource();
5755 }
5756 let failed = match &out {
5757 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5758 AgentOutcome::Quota(o) => {
5759 self.state.quota.push(QuotaLoss {
5760 seat: seat.key.clone(),
5761 node: "fix".to_owned(),
5762 at: Timestamp::now(),
5763 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5764 });
5765 Some("rate limited (quota); the fixer could not run".to_owned())
5766 }
5767 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5768 AgentOutcome::Failed(e) => Some(e.clone()),
5769 };
5770 self.state.seats.insert(seat.key.clone(), seat);
5771 let coordination = self.coordinate(&wt).await?;
5772 let failed = match failed {
5773 Some(why) => {
5775 if let Err(e) = git::restore_to(&wt, &head).await {
5776 return self
5777 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5778 .await;
5779 }
5780 Some(why)
5781 }
5782 None => {
5783 if let Ok(r) = git::rescue_commit(
5784 &wt,
5785 "magi: recovery repair of the failed verification (uncommitted work)",
5786 )
5787 .await
5788 {
5789 self.state.note_withheld("fix", &r.withheld);
5790 }
5791 None
5792 }
5793 };
5794 let after = git::rev_parse(&wt, "HEAD").await?;
5795 drop(coordination);
5796 self.state.event(
5797 "recovery",
5798 match &failed {
5799 Some(why) => format!("repair: nothing kept ({why})"),
5800 None if after != head => {
5801 format!("repair: committed {}", short(&after))
5802 }
5803 None => "repair: the fixer made no commit".to_owned(),
5804 },
5805 );
5806 rec.repair = Repair::Done {
5807 before: head.clone(),
5808 committed: after != head,
5809 after,
5810 failed,
5811 };
5812 self.state.recovery = Some(rec);
5813 self.state.save()?;
5814 Ok(())
5815 }
5816
5817 async fn stop_recovery(
5820 &mut self,
5821 mut rec: crate::recovery::RecoveryRecord,
5822 head: &str,
5823 why: String,
5824 ) -> Result<()> {
5825 rec.repair = crate::recovery::Repair::Started {
5827 before: head.to_owned(),
5828 };
5829 self.state.recovery = Some(rec);
5830 self.state
5831 .event("recovery", format!("repair could not be undone: {why}"));
5832 self.state.status = RunStatus::Blocked;
5833 self.state.save()?;
5834 bail!("recovery repair could not be rolled back: {why}")
5835 }
5836
5837 async fn review_loop(&mut self) -> Result<()> {
5838 if self
5843 .state
5844 .base_sync
5845 .as_ref()
5846 .is_some_and(|s| s.conflict.is_some())
5847 {
5848 return Ok(());
5849 }
5850 let run_id = self.state.id.clone();
5855 let prompts = self.state.config.prompts.clone();
5856 let Some(winner) = self.state.winner().cloned() else {
5857 return Ok(());
5858 };
5859 let max_rounds = self.state.config.graph.review_rounds;
5860 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5870 if status == RunStatus::Gating {
5873 self.record_contested_handoff();
5874 }
5875 self.state.status = status;
5876 self.state.save()?;
5877 return Ok(());
5878 }
5879 self.state.status = RunStatus::Reviewing;
5880 if self
5890 .state
5891 .reviews
5892 .last()
5893 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5894 {
5895 let shell = self.state.config.shell();
5896 return self
5897 .stop_reviewing(
5898 "the last round's own verification never resolved",
5899 &shell,
5900 &winner.worktree,
5901 )
5902 .await;
5903 }
5904
5905 self.recovery_repair(&winner).await?;
5906 if self.resource_halt()? {
5907 return Ok(());
5908 }
5909
5910 let repo = self.state.repo.clone();
5911 let root = self.state.worktree_root();
5912 let language = self.state.config.graph.language.clone();
5913 let sessions = self.state.config.graph.sessions;
5914 let artifacts = self.merge_repair_artifacts();
5915 let base = self.landing_base();
5916 let base_short = short(&base);
5917 let reviewers = self.roles.reviewers.clone();
5918 let shell = self.state.config.shell();
5919
5920 for round in (self.state.reviews.len() + 1)..=max_rounds {
5921 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5922 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5923 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5924 let prev_verification = self
5933 .state
5934 .reviews
5935 .last()
5936 .and_then(|r| r.verification_summary(&head));
5937 let prior_section = match (&self.state.recovery, round) {
5940 (Some(rec), 1) if self.state.reviews.is_empty() => {
5941 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5942 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5943 Some(crate::recovery::head_of(
5944 &git::diff(&winner.worktree, reviewed, "HEAD")
5945 .await
5946 .unwrap_or_default(),
5947 crate::recovery::SINCE_DIFF_MAX,
5948 ))
5949 }
5950 _ => None,
5951 };
5952 Some(prompt::recovery_review(rec, since.as_deref()))
5953 }
5954 _ => None,
5955 };
5956
5957 let mut jobs = Vec::new();
5961 let coordination = self.coordinate(&repo).await?;
5962 for (r, spec) in reviewers.iter().cloned().enumerate() {
5963 let wt = root.join(format!("review-{}", r + 1));
5964 if wt.exists() {
5967 git::reset_detached(&wt, &head).await?;
5968 } else {
5969 git::worktree_add_detached(&repo, &wt, &head).await?;
5970 }
5971 self.setup_seat_worktree(&repo, &wt, false).await?;
5972 let seat_key = format!("review-{}", r + 1);
5973 let spec = pick_start_spec(
5976 &self.roles.reviewer_roster,
5977 spec,
5978 self.state.seat_history.get(&seat_key),
5979 );
5980 let seat = self.seat(&seat_key, &spec.id);
5981 jobs.push(SeatJob {
5982 prompt: prompt::review(&prompt::ReviewCtx {
5983 instruction: &self.state.instruction,
5984 branch: &winner.branch,
5985 base_short: &base_short,
5986 stat: &stat,
5987 patch: &patch,
5988 verification: prev_verification.as_ref(),
5989 prior: prior_section.as_deref(),
5990 reviewers: reviewers.len(),
5991 round,
5992 rounds: max_rounds,
5993 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5996 lens: Lens::for_seat(r),
5997 language: &language,
5998 github_language: &self.state.config.graph.github_language,
5999 }),
6000 spec,
6001 seat,
6002 cwd: wt,
6003 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6004 allow_write: false,
6005 sessions,
6006 artifacts: artifacts.clone(),
6007 stem: format!("review-{round}-{}", r + 1),
6008 handover: None,
6009 });
6010 }
6011
6012 drop(coordination);
6013 self.state.event(
6014 "review",
6015 format!(
6016 "round {round}: {} reviewers on {}",
6017 jobs.len(),
6018 short(&head)
6019 ),
6020 );
6021 let mut quota_losses = Vec::new();
6022 let review_retries = self.state.config.graph.retries;
6023 let review_cache = self.state.config.cache_dir();
6024 let ctx = WaveCtx {
6025 carry_seats: true,
6026 pauses: Some((&self.pause, &self.interrupt)),
6027 run: &run_id,
6028 node: "review",
6029 prompts: &prompts,
6030 cache: review_cache.as_deref(),
6031 round: Some(round),
6032 };
6033 let results = ask_json_wave::<Review>(
6034 jobs,
6035 Arc::clone(&self.sem),
6036 review_retries,
6037 &self.roles.reviewer_roster,
6038 &ctx,
6039 &mut quota_losses,
6040 &mut self.state,
6041 &|_: &Review| Ok(()),
6042 )
6043 .await;
6044 if self.state.resource_block.is_some() {
6045 return self.stop_for_resource();
6046 }
6047 let round_quota_missing = quota_losses.len();
6051 self.state.quota.extend(quota_losses);
6052
6053 let mut records = Vec::new();
6054 let mut all_findings = Vec::new();
6055 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6056 let agent_id = seat.agent.clone();
6057 self.state.seats.insert(seat.key.clone(), seat);
6058 let mut record = ReviewRecord {
6059 reviewer: r + 1,
6060 agent: agent_id,
6061 summary: String::new(),
6062 findings: Vec::new(),
6063 vote: None,
6064 failed: None,
6065 duration_ms: 0,
6066 attempts,
6072 };
6073 match res {
6074 Ok((review, out)) => {
6075 record.summary =
6083 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6084 record.vote = Some(review.vote);
6085 record.duration_ms = out.duration_ms;
6086 for (n, mut f) in review.findings.into_iter().enumerate() {
6087 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6090 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6091 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6092 f.file = f
6098 .file
6099 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6100 all_findings.push(f.clone());
6101 record.findings.push(f);
6102 }
6103 self.state.event(
6104 "review",
6105 format!(
6106 "round {round}: reviewer {} voted {} with {} finding(s)",
6107 r + 1,
6108 review.vote.label(),
6109 record.findings.len()
6110 ),
6111 );
6112 }
6113 Err(e) => {
6114 record.failed = Some(e.to_string());
6115 self.state.event(
6116 "review",
6117 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6118 );
6119 }
6120 }
6121 records.push(record);
6122 }
6123
6124 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6131 let vote_split =
6132 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6133 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6134 if vote_split {
6135 self.state.event(
6136 "review",
6137 format!(
6138 "round {round}: votes split ({}) — one round of reconsideration",
6139 initial_votes
6140 .iter()
6141 .map(|v| v.label())
6142 .collect::<Vec<_>>()
6143 .join(", ")
6144 ),
6145 );
6146 let panel: Vec<ReviewSeatReport<'_>> = records
6149 .iter()
6150 .filter_map(|r| {
6151 r.vote.map(|vote| ReviewSeatReport {
6152 reviewer: r.reviewer,
6153 vote,
6154 summary: &r.summary,
6155 findings: &r.findings,
6156 })
6157 })
6158 .collect();
6159
6160 let mut jobs = Vec::new();
6161 let mut seats_at = Vec::new();
6162 for (r, spec) in reviewers.iter().cloned().enumerate() {
6163 if records[r].vote.is_none() {
6167 continue;
6168 }
6169 let wt = root.join(format!("review-{}", r + 1));
6170 let seat_key = format!("review-{}", r + 1);
6171 let spec = self.occupant(&seat_key, spec);
6172 let seat = self.seat(&seat_key, &spec.id);
6173 let build = |with_patch: bool| {
6181 prompt::review_reconsider(&ReviewReconsiderCtx {
6182 instruction: &self.state.instruction,
6183 reviewer: r + 1,
6184 lens: Lens::for_seat(r),
6185 panel: &panel,
6186 patch: with_patch.then_some(ReviewPatch {
6187 branch: &winner.branch,
6188 base_short: &base_short,
6189 stat: &stat,
6190 patch: &patch,
6191 }),
6192 round,
6193 rounds: max_rounds,
6194 language: &language,
6195 })
6196 };
6197 let full = build(true);
6198 let prompt = if has_context(&spec, &seat, sessions) {
6199 build(false)
6200 } else {
6201 full.clone()
6202 };
6203 jobs.push(SeatJob {
6204 prompt,
6205 spec,
6206 seat,
6207 cwd: wt,
6208 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6209 allow_write: false,
6210 sessions,
6211 artifacts: artifacts.clone(),
6212 stem: format!("review-{round}-reconsider-{}", r + 1),
6213 handover: Some(full),
6214 });
6215 seats_at.push(r);
6216 }
6217
6218 let mut recon_quota_losses = Vec::new();
6219 let recon_cache = self.state.config.cache_dir();
6220 let recon_ctx = WaveCtx {
6221 carry_seats: true,
6222 pauses: Some((&self.pause, &self.interrupt)),
6223 run: &run_id,
6224 node: "review",
6225 prompts: &prompts,
6226 cache: recon_cache.as_deref(),
6227 round: Some(round),
6228 };
6229 let recon_results = ask_json_wave::<ReviewRevote>(
6230 jobs,
6231 Arc::clone(&self.sem),
6232 review_retries,
6233 &self.roles.reviewer_roster,
6234 &recon_ctx,
6235 &mut recon_quota_losses,
6236 &mut self.state,
6237 &|_: &ReviewRevote| Ok(()),
6238 )
6239 .await;
6240 if self.state.resource_block.is_some() {
6241 return self.stop_for_resource();
6242 }
6243 self.state.quota.extend(recon_quota_losses);
6244
6245 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6246 let agent_id = seat.agent.clone();
6247 self.state.seats.insert(seat.key.clone(), seat);
6248 let mut rec = ReviewRevoteRecord {
6249 reviewer: r + 1,
6250 agent: agent_id,
6251 vote: None,
6252 reason: String::new(),
6253 failed: None,
6254 };
6255 match res {
6256 Ok((rv, _)) => {
6257 rec.vote = Some(rv.vote);
6258 rec.reason =
6259 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6260 self.state.event(
6261 "review",
6262 format!(
6263 "round {round}: reviewer {} revoted {}",
6264 r + 1,
6265 rv.vote.label()
6266 ),
6267 );
6268 }
6269 Err(e) => {
6270 rec.failed = Some(e.to_string());
6271 self.state.event(
6272 "review",
6273 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6274 );
6275 }
6276 }
6277 reconsideration.push(rec);
6278 }
6279 } else if initial_votes.len() > 1 {
6280 self.state.event(
6281 "review",
6282 format!(
6283 "round {round}: votes agreed ({}) — no reconsideration",
6284 initial_votes[0].label()
6285 ),
6286 );
6287 }
6288
6289 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6290 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6291 let defer_e2e =
6302 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6303 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6304 let reason =
6305 format!("{blocking} blocking finding(s) already required a fix this round");
6306 self.state.event(
6307 "verify",
6308 format!(
6309 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6310 {}); it will run once a round has none left",
6311 short(&head)
6312 ),
6313 );
6314 (Vec::new(), false, true, Some(reason))
6315 } else {
6316 let e2e_commands = self.state.config.verify.e2e.clone();
6317 let cache_dir = self.state.config.cache_dir();
6318 let context = format!("round {round}");
6319 let wait = self.repository_wait();
6320 let (e2e, verify_retried) = with_cache_lease(
6321 &mut self.state,
6322 cache_dir.as_deref(),
6323 "e2e",
6324 "e2e",
6325 &winner.worktree,
6326 &head,
6327 verify_timeout,
6328 &context,
6329 &wait,
6330 |state, budget| {
6331 let shell = shell.clone();
6332 let e2e_commands = e2e_commands.clone();
6333 let worktree = winner.worktree.clone();
6334 let context = context.clone();
6335 async move {
6336 run_e2e_with_retry(
6337 state,
6338 &shell,
6339 &e2e_commands,
6340 &worktree,
6341 budget,
6342 &context,
6343 )
6344 .await
6345 }
6346 },
6347 )
6348 .await?;
6349 (e2e, verify_retried, false, None)
6350 };
6351
6352 let expected = records.len();
6353 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6354 let incomplete = answered < expected;
6355 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6356 let policy = self.state.config.graph.incomplete_review;
6357 let clean = round_is_clean(
6358 blocking,
6359 e2e_ok,
6360 answered,
6361 expected,
6362 round_quota_missing,
6363 policy,
6364 );
6365
6366 let mut round_record = ReviewRound {
6367 round,
6368 head: head.clone(),
6369 verified_head: None,
6370 verified_at: None,
6371 reviews: records,
6372 e2e,
6373 verify_retried,
6374 e2e_deferred,
6375 e2e_defer_reason,
6376 fix: None,
6377 blocking,
6378 answered,
6379 expected,
6380 clean,
6381 progressed: false,
6382 vote_split,
6383 reconsideration,
6384 verdict: None,
6385 };
6386 round_record.verdict = ReviewVote::worst(
6390 round_record
6391 .final_votes()
6392 .into_iter()
6393 .map(|(_, _, vote)| vote),
6394 );
6395 if !matches!(
6406 round_record.e2e_status(),
6407 E2eStatus::Deferred | E2eStatus::NotConfigured
6408 ) {
6409 round_record.verified_head = Some(head.clone());
6410 round_record.verified_at = Some(Timestamp::now());
6411 }
6412 let this_round_verification = round_record.verification_summary(&head);
6413
6414 if incomplete {
6415 let missing: Vec<String> = round_record
6416 .reviews
6417 .iter()
6418 .filter(|r| r.failed.is_some())
6419 .map(|r| format!("review-{}", r.reviewer))
6420 .collect();
6421 self.state.event(
6422 "review",
6423 format!(
6424 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6425 missing.join(", ")
6426 ),
6427 );
6428 }
6429
6430 if clean {
6431 self.state.event(
6432 "review",
6433 if incomplete && policy == IncompleteReviewPolicy::Warn {
6434 format!(
6435 "round {round}: clean (warn policy, incomplete panel) — no \
6436 blocking findings from the seats that answered, verification green"
6437 )
6438 } else if incomplete {
6439 format!(
6440 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6441 quorum) — no blocking findings from the seats that answered, \
6442 verification green",
6443 expected - answered
6444 )
6445 } else {
6446 format!("round {round}: clean — no blocking findings, verification green")
6447 },
6448 );
6449 self.state.reviews.push(round_record);
6450 self.state.status = RunStatus::Gating;
6451 self.state.save()?;
6452 return Ok(());
6453 }
6454
6455 if incomplete && blocking == 0 && e2e_ok {
6463 self.state.reviews.push(round_record);
6464 self.state.save()?;
6465 if round == max_rounds {
6466 self.state.status = RunStatus::Blocked;
6467 self.state.event(
6468 "review",
6469 format!(
6470 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6471 refusing to call it clean",
6472 expected - answered
6473 ),
6474 );
6475 return Ok(());
6476 }
6477 continue;
6478 }
6479
6480 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6492 self.state.reviews.push(round_record);
6493 return self
6494 .stop_reviewing(
6495 "the round's own verification could not run",
6496 &shell,
6497 &winner.worktree,
6498 )
6499 .await;
6500 }
6501
6502 if round == max_rounds {
6503 self.state.reviews.push(round_record);
6504 return self
6505 .stop_reviewing(
6506 &format!(
6507 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6508 ),
6509 &shell,
6510 &winner.worktree,
6511 )
6512 .await;
6513 }
6514
6515 let blocking_findings: Vec<_> = all_findings
6518 .iter()
6519 .filter(|f| f.severity.blocks())
6520 .cloned()
6521 .collect();
6522 let fix_prompt = prompt::fix_in(
6523 &self.state.instruction,
6524 &blocking_findings,
6525 this_round_verification.as_ref(),
6526 round,
6527 max_rounds,
6528 &language,
6529 &self.state.config.graph.github_language,
6530 );
6531 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6532 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6533 let (job, seat, out) = self
6534 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6535 prompt: fix_prompt.clone(),
6536 spec,
6537 seat,
6538 cwd: winner.worktree.clone(),
6539 timeout,
6540 allow_write: true,
6541 sessions,
6542 artifacts: artifacts.clone(),
6543 stem: format!("fix-{round}"),
6544 handover: None,
6545 })
6546 .await;
6547 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6550 return self.stop_for_resource();
6551 }
6552 let agent_id = seat.agent.clone();
6553
6554 let mut fix = FixRecord {
6555 agent: agent_id,
6556 addressed: Vec::new(),
6557 rejected: Vec::new(),
6558 notes: String::new(),
6559 committed: false,
6560 failed: None,
6561 duration_ms: 0,
6562 continuation: None,
6563 };
6564 let mut continuation = ContinuationRecord::not_needed();
6565 let mut final_seat = seat.clone();
6566 match out {
6567 AgentOutcome::Ok(o) => {
6568 fix.duration_ms = o.duration_ms;
6569 let parsed = verdict::extract_json::<FixReport>(&o.text);
6570 let incomplete_reason = match &parsed {
6577 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6578 "the reply parsed, but it reported a command whose own CLI never \
6579 confirmed an exit status"
6580 .to_owned(),
6581 ),
6582 Ok(_) => None,
6583 Err(e) => Some(e.to_string()),
6584 };
6585 match incomplete_reason {
6586 None => {
6587 let report = parsed.expect("checked Ok above");
6588 fix.addressed = report.addressed;
6589 fix.rejected = report.rejected;
6590 fix.notes =
6591 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6592 }
6593 Some(reason) => {
6594 let (resumed_seat, resolved, failure, cont) = self
6595 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6596 .await;
6597 fix.duration_ms += cont.cumulative_wait_ms;
6598 continuation = cont;
6599 final_seat = resumed_seat;
6600 match resolved {
6601 Some(report) => {
6602 fix.addressed = report.addressed;
6603 fix.rejected = report.rejected;
6604 fix.notes = blind::sanitize_prose(
6605 &report.notes,
6606 &self.state.config.blind,
6607 );
6608 }
6609 None => fix.failed = failure,
6610 }
6611 }
6612 }
6613 }
6614 AgentOutcome::Dropped(o) => {
6616 fix.duration_ms = o.duration_ms;
6617 let why = o
6618 .dropped
6619 .as_ref()
6620 .map(|d| d.why.as_str())
6621 .unwrap_or("the CLI ended the stream without delivering its answer");
6622 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6623 }
6624 AgentOutcome::Quota(o) => {
6625 self.state.quota.push(QuotaLoss {
6626 seat: final_seat.key.clone(),
6627 node: "fix".to_owned(),
6628 at: Timestamp::now(),
6629 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6630 });
6631 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6632 }
6633 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6634 }
6635 fix.continuation = Some(continuation);
6636 self.state.seats.insert(final_seat.key.clone(), final_seat);
6637 let coordination = self.coordinate(&winner.worktree).await?;
6638 if let Ok(r) = git::rescue_commit(
6639 &winner.worktree,
6640 &format!("magi: review round {round} fixes (uncommitted work)"),
6641 )
6642 .await
6643 {
6644 self.state.note_withheld("fix", &r.withheld);
6645 }
6646 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6647 fix.committed = after != before;
6648 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6656 drop(coordination);
6657 let progressed = diff_after != patch;
6658 let commit_note = if fix.committed {
6659 "committed"
6660 } else {
6661 "NO new commit"
6662 };
6663 let tree_note = if progressed {
6664 "changed vs base"
6665 } else {
6666 "unchanged vs base"
6667 };
6668 self.state.event(
6669 "fix",
6670 match &fix.failed {
6671 Some(reason) => {
6677 format!(
6678 "round {round}: fixer's adoption report was lost ({reason}); \
6679 {commit_note}, tree {tree_note}"
6680 )
6681 }
6682 None => format!(
6683 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6684 {tree_note}{}",
6685 fix.addressed.len(),
6686 fix.rejected.len(),
6687 if continuation.outcome == ContinuationOutcome::Resumed {
6688 format!(
6689 " (adoption report recovered after {} continuation(s))",
6690 continuation.attempts
6691 )
6692 } else {
6693 String::new()
6694 },
6695 ),
6696 },
6697 );
6698 round_record.fix = Some(fix);
6699 round_record.progressed = progressed;
6700 self.state.reviews.push(round_record);
6701 self.state.save()?;
6702
6703 if matches!(
6716 continuation.outcome,
6717 ContinuationOutcome::Exhausted
6718 | ContinuationOutcome::QuotaLost
6719 | ContinuationOutcome::NoSession
6720 ) {
6721 return self
6722 .stop_reviewing(
6723 "the fixer's adoption report never came back, even after resuming its \
6724 own seat; refusing to start another round against the same worktree \
6725 while that is unresolved",
6726 &shell,
6727 &winner.worktree,
6728 )
6729 .await;
6730 }
6731
6732 let streak = self
6733 .state
6734 .reviews
6735 .iter()
6736 .rev()
6737 .take_while(|r| !r.progressed)
6738 .count();
6739 if streak >= STAGNANT_LIMIT {
6740 return self
6741 .stop_reviewing(
6742 &format!(
6743 "the tree has not moved against base for {streak} round(s) in a row"
6744 ),
6745 &shell,
6746 &winner.worktree,
6747 )
6748 .await;
6749 }
6750 }
6751 Ok(())
6752 }
6753
6754 fn record_contested_handoff(&mut self) {
6788 if self.state.contested_handoff.is_some() {
6789 return;
6790 }
6791 let Some(contested) = self
6792 .state
6793 .reviews
6794 .last()
6795 .and_then(ReviewRound::contested_handoff)
6796 else {
6797 return;
6798 };
6799 self.state.event(
6800 "review",
6801 format!(
6802 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6803 wait for the owner's approval",
6804 contested.findings.len(),
6805 contested.rejecters.len()
6806 ),
6807 );
6808 self.state.contested_handoff = Some(contested);
6809 }
6810
6811 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6812 let round_idx = self.state.reviews.len() - 1;
6813 let needs_catchup_run = matches!(
6821 self.state.reviews[round_idx].e2e_status(),
6822 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6823 );
6824 if needs_catchup_run {
6825 let round = self.state.reviews[round_idx].round;
6826 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6827 let commands = self.state.config.verify.e2e.clone();
6828 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6829 let cache_dir = self.state.config.cache_dir();
6830 let context = format!(
6831 "round {round}: verification unresolved, catching up before the final decision"
6832 );
6833 let wait = self.repository_wait();
6834 let (outcomes, verify_retried) = with_cache_lease(
6835 &mut self.state,
6836 cache_dir.as_deref(),
6837 "e2e",
6838 "e2e",
6839 worktree,
6840 &attempted_head,
6841 timeout,
6842 &context,
6843 &wait,
6844 |state, budget| {
6845 let shell = shell.to_vec();
6846 let commands = commands.clone();
6847 let context = context.clone();
6848 async move {
6849 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6850 .await
6851 }
6852 },
6853 )
6854 .await?;
6855 let last = &mut self.state.reviews[round_idx];
6856 last.e2e = outcomes;
6857 last.verify_retried = verify_retried;
6858 last.verified_head = Some(attempted_head);
6865 last.verified_at = Some(Timestamp::now());
6866 if verify_inconclusive(&last.e2e) {
6867 self.state.save()?;
6874 return Ok(());
6875 }
6876 last.e2e_deferred = false;
6877 }
6878 let last = &self.state.reviews[round_idx];
6879 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6880
6881 match last.e2e_status() {
6882 E2eStatus::Failed => {
6883 let red: Vec<String> = last
6884 .e2e
6885 .iter()
6886 .filter(|o| !o.ok())
6887 .map(|o| {
6888 format!(
6889 "`{}` -> {:?}\n{}",
6890 o.command,
6891 o.code,
6892 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6893 )
6894 })
6895 .collect();
6896 self.state
6897 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6898 self.state.status = RunStatus::Blocked;
6899 }
6900 E2eStatus::ResourceBlocked => {
6905 self.state.event(
6906 "review",
6907 format!(
6908 "{why}; e2e could not run (shared build cache unavailable); not \
6909 deciding yet"
6910 ),
6911 );
6912 }
6913 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6914 self.state.event(
6915 "review",
6916 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6917 );
6918 self.record_contested_handoff();
6919 self.state.status = RunStatus::Gating;
6920 }
6921 }
6922 self.state.save()?;
6923 Ok(())
6924 }
6925
6926 async fn gate(&mut self) -> Result<()> {
6929 if self.state.status == RunStatus::Failed
6941 || self
6942 .state
6943 .base_sync
6944 .as_ref()
6945 .is_some_and(|s| s.conflict.is_some())
6946 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6947 != Some(RunStatus::Gating)
6948 {
6949 return Ok(());
6950 }
6951 if self.state.gate_ran {
6952 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6963 self.state.status = RunStatus::Blocked;
6964 self.state.save()?;
6965 }
6966 return Ok(());
6967 }
6968 let Some(winner) = self.state.winner().cloned() else {
6969 return Ok(());
6970 };
6971 self.state.status = RunStatus::Gating;
6972 let mut outcomes = self.run_gate(&winner).await?;
6973 loop {
6974 if verify_inconclusive(&outcomes) {
6985 self.state.save()?;
6986 return Ok(());
6987 }
6988 if outcomes.iter().all(CommandOutcome::ok) {
6989 break;
6990 }
6991 match self.gate_fix_round(&winner, &outcomes).await? {
6992 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6993 GateFix::Stop => break,
6994 GateFix::Defer => {
6995 self.state.save()?;
6996 return Ok(());
6997 }
6998 }
6999 }
7000 let passed = outcomes.iter().all(CommandOutcome::ok);
7001 self.state.gate = outcomes;
7002 self.state.gate_ran = true;
7003 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7004 if !passed {
7005 self.state.status = RunStatus::Blocked;
7006 let spent = self.state.gate_fixes.len();
7007 self.state.event(
7008 "gate",
7009 if spent == 0 {
7010 "gate failed; not merging".to_owned()
7011 } else {
7012 format!("gate failed after {spent} gate-fix round(s); not merging")
7013 },
7014 );
7015 }
7016 self.state.save()?;
7017 Ok(())
7018 }
7019
7020 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7030 let commands = state.config.verify.pre_gate.clone();
7031 if commands.is_empty() {
7032 return;
7033 }
7034 let shell = state.config.shell();
7035 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7036 let (outcomes, _) = run_commands(
7037 state,
7038 "pre_gate",
7039 "pre_gate",
7040 0,
7041 &shell,
7042 &commands,
7043 &winner.worktree,
7044 timeout,
7045 )
7046 .await;
7047 for o in &outcomes {
7048 if !o.ok() {
7049 tracing::warn!(
7050 "pre_gate `{}` failed ({:?}); the gate decides",
7051 o.command,
7052 o.code
7053 );
7054 }
7055 state.event(
7056 "pre_gate",
7057 format!(
7058 "`{}` -> {}",
7059 o.command,
7060 if o.ok() {
7061 "pass".to_owned()
7062 } else {
7063 format!(
7064 "FAIL ({:?})\n{}",
7065 o.code,
7066 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7067 )
7068 }
7069 ),
7070 );
7071 }
7072 state.pre_gate = outcomes;
7073 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7074 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7075 Ok(head) => {
7076 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7077 state.pre_gate_commit = Some(head);
7078 }
7079 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7080 },
7081 Ok(false) => {}
7082 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7083 }
7084 if let Err(e) = state.save() {
7085 tracing::warn!("could not persist the pre_gate record: {e:#}");
7086 }
7087 }
7088
7089 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7092 let wait = self.repository_wait();
7093 run_landing_gate(&mut self.state, winner, &wait).await
7094 }
7095
7096 async fn gate_fix_round(
7109 &mut self,
7110 winner: &Candidate,
7111 outcomes: &[CommandOutcome],
7112 ) -> Result<GateFix> {
7113 let cap = self.state.config.graph.gate_fix_rounds;
7114 let spent = self.state.gate_fixes.len();
7115 if spent >= cap {
7116 if cap > 0 {
7117 self.state.event(
7118 "gate",
7119 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7120 );
7121 }
7122 return Ok(GateFix::Stop);
7123 }
7124 if !gate_fixable(outcomes) {
7125 self.state.event(
7126 "gate",
7127 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7128 command or similar); not spending a fix round on it",
7129 );
7130 return Ok(GateFix::Stop);
7131 }
7132 let min_free = self.state.config.disk.min_free_bytes;
7133 if min_free > 0 {
7134 match crate::disk::free_bytes(&winner.worktree) {
7135 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7136 Ok(free) => {
7137 self.state.event(
7138 "gate",
7139 format!(
7140 "only {free} bytes free ({min_free} required by `[disk] \
7141 min_free_bytes`); not spending a fix round on a failure the disk \
7142 may explain"
7143 ),
7144 );
7145 return Ok(GateFix::Stop);
7146 }
7147 Err(e) => {
7148 self.state.event(
7149 "gate",
7150 format!("free disk space could not be measured ({e:#}); no fix round"),
7151 );
7152 return Ok(GateFix::Stop);
7153 }
7154 }
7155 }
7156
7157 let attempt = spent + 1;
7158 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7159 let base = self.landing_base();
7160 let fix_prompt = prompt::gate_fix_in(
7161 &self.state.instruction,
7162 &failed,
7163 attempt,
7164 cap,
7165 &self.state.config.graph.language,
7166 &self.state.config.graph.github_language,
7167 );
7168 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7169 let sessions = self.state.config.graph.sessions;
7170 let artifacts = self.merge_repair_artifacts();
7171 self.state.event(
7172 "gate",
7173 format!("gate failed; gate-fix round {attempt} of {cap}"),
7174 );
7175 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7176 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7177 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7180 head: before.clone(),
7181 passed: false,
7182 outcomes: Vec::new(),
7183 });
7184 self.state.save()?;
7185 let (_, seat, out) = self
7186 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7187 prompt: fix_prompt.clone(),
7188 spec,
7189 seat,
7190 cwd: winner.worktree.clone(),
7191 timeout,
7192 allow_write: true,
7193 sessions,
7194 artifacts: artifacts.clone(),
7195 stem: format!("gate-fix-{attempt}"),
7196 handover: None,
7197 })
7198 .await;
7199 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7202 return Ok(GateFix::Defer);
7203 }
7204 let mut record = GateFixRecord {
7205 agent: seat.agent.clone(),
7206 failed,
7207 notes: String::new(),
7208 committed: false,
7209 error: None,
7210 };
7211 match out {
7212 AgentOutcome::Ok(o) => {
7213 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7216 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7217 }
7218 }
7219 AgentOutcome::Dropped(_) => {
7220 record.error = Some("the CLI dropped the stream".to_owned());
7221 }
7222 AgentOutcome::Quota(o) => {
7223 self.state.quota.push(QuotaLoss {
7224 seat: seat.key.clone(),
7225 node: "gate-fix".to_owned(),
7226 at: Timestamp::now(),
7227 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7228 });
7229 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7230 }
7231 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7232 }
7233 self.state.seats.insert(seat.key.clone(), seat);
7234 let coordination = self.coordinate(&winner.worktree).await?;
7235 if let Ok(r) = git::rescue_commit(
7236 &winner.worktree,
7237 &format!("magi: gate fix {attempt} (uncommitted work)"),
7238 )
7239 .await
7240 {
7241 self.state.note_withheld("gate-fix", &r.withheld);
7242 }
7243 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7244 record.committed = after != before;
7245 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7246 let note = record.error.clone();
7247 self.state.gate_fixes.push(record);
7248 self.state.save()?;
7249 if !changed {
7250 self.state.event(
7251 "gate",
7252 match note {
7253 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7254 None => format!("gate-fix round {attempt}: the tree did not change"),
7255 },
7256 );
7257 return Ok(GateFix::Stop);
7258 }
7259 self.state.event(
7260 "gate",
7261 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7262 );
7263
7264 drop(coordination);
7265 let wait = self.repository_wait();
7266 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7267 }
7268
7269 async fn guarded_pr_message(
7276 &mut self,
7277 winner: &Candidate,
7278 facts: Option<&BranchFacts>,
7279 posting: bool,
7280 ) -> Result<Option<PrMessage>> {
7281 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7282 let head = match final_head.as_deref() {
7283 Some(h) => crate::pr_outcome::FinalHead::At(h),
7284 None => crate::pr_outcome::FinalHead::Unknown,
7285 };
7286 let summarized = if posting {
7288 Some(self.summarize_change(winner, final_head.as_deref()).await)
7289 } else {
7290 None
7291 };
7292 if self.state.resource_block.is_some() {
7297 self.state.event(
7298 "pr-summary",
7299 "a shared resource was busy; nothing was published, resume to summarize again",
7300 );
7301 return Ok(None);
7302 }
7303 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7304 anyhow::bail!(
7305 "the final commit moved during public description generation; retry before publishing"
7306 );
7307 }
7308 let note = match (&summarized, &self.state.change_summary) {
7309 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7310 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7311 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7312 (None, _) => ChangeNote::NotAsked,
7313 };
7314 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7315 if !posting {
7316 let identity = crate::scrub::Identity::current();
7317 return Ok(Some(PrMessage {
7318 title: crate::scrub::scrub(&pr.title, &identity),
7319 body: crate::scrub::scrub(&pr.body, &identity),
7320 }));
7321 }
7322 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7325 if let Some(known) = self
7326 .state
7327 .github_text
7328 .clone()
7329 .filter(|g| g.fingerprint == raw_fp)
7330 {
7331 return self.settle_withheld(winner, &pr, known).await;
7332 }
7333 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7334 let mut violations =
7335 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7336 let identity = crate::scrub::Identity::current();
7337 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7338 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7339 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7340 {
7341 violations.push(crate::github_text::Violation::SensitiveData);
7342 }
7343 let has_language = violations
7347 .iter()
7348 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7349 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7350 self.state.event(
7351 "github-text",
7352 if has_language {
7353 format!("description rejected: {violations:?}; requesting one rewrite")
7354 } else {
7355 format!("description rejected: {violations:?}; asking the owner")
7356 },
7357 );
7358 let mut rewritten = false;
7359 let mut rejected_rewrite = None;
7360 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7361 {
7362 let output = text;
7363 if let Some(title) = summary_title(&output) {
7364 let raw_body = summary_without_title(&output);
7367 let clean_title = crate::scrub::scrub(&title, &identity);
7368 let mut body = crate::scrub::scrub(&raw_body, &identity);
7369 let redacted = clean_title != title || body != raw_body;
7370 let title = clean_title;
7371 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7372 if !complete {
7373 self.state.event(
7374 "github-text",
7375 "description rewrite omitted required findings",
7376 );
7377 }
7378 if !body.trim().is_empty() {
7379 body.push_str("\n\n");
7380 body.push_str(&public_pr_evidence(&self.state, head));
7381 }
7382 let marker = format!("magi:run/{}", self.state.id);
7383 if !body.trim().is_empty() && !body.contains(&marker) {
7384 body.push_str(&format!("\n\n{marker}"));
7385 }
7386 let rewrite = PrMessage {
7387 title: title.clone(),
7388 body: body.clone(),
7389 };
7390 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7391 let mut failed = crate::github_text::check_for(
7392 &self.state.config.graph,
7393 &title,
7394 &body,
7395 redecision,
7396 );
7397 if (!crate::github_text::shareable(&title)
7398 || !crate::github_text::shareable(&body))
7399 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7400 {
7401 failed.push(crate::github_text::Violation::SensitiveData);
7402 }
7403 if complete
7404 && !title.trim().is_empty()
7405 && !body.trim().is_empty()
7406 && !failed.is_empty()
7407 {
7408 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7409 title: title.clone(),
7410 body: body.clone(),
7411 failed: failed
7412 .iter()
7413 .map(|v| crate::github_text::category(*v).to_owned())
7414 .collect(),
7415 });
7416 }
7417 if complete
7418 && !title.trim().is_empty()
7419 && !raw_body.trim().is_empty()
7420 && failed.is_empty()
7421 {
7422 decision = redecision;
7423 pr = PrMessage { title, body };
7424 rewritten = true;
7425 self.state
7426 .event("github-text", "description rewrite passed");
7427 if redacted {
7428 self.state.event(
7429 "github-text",
7430 "sensitive data removed from description rewrite",
7431 );
7432 }
7433 }
7434 }
7435 }
7436 if !rewritten {
7437 self.state.event(
7438 "github-text",
7439 "description rewrite unavailable or rejected; asking the owner",
7440 );
7441 if let Some(w) =
7442 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7443 {
7444 self.retire_github_text_question("the withheld text changed");
7447 let record = crate::run::GithubTextAsk {
7448 rewrite: rejected_rewrite,
7449 fingerprint: raw_fp,
7450 title: w.title,
7451 body: w.body,
7452 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7453 question: None,
7454 asks: 0,
7455 resolved: false,
7456 chosen_title: None,
7457 choice: None,
7458 };
7459 return self.settle_withheld(winner, &pr, record).await;
7460 }
7461 }
7462 }
7463 let (title, body) =
7464 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7465 Ok(Some(PrMessage { title, body }))
7466 }
7467
7468 async fn summarize_change(
7479 &mut self,
7480 winner: &Candidate,
7481 head: Option<&str>,
7482 ) -> std::result::Result<(), String> {
7483 let Some(head) = head else {
7484 return Err("the final commit could not be read".to_owned());
7485 };
7486 let remote = self.state.config.merge.remote.clone();
7487 let tracking = format!("{remote}/{}", self.state.base_branch);
7488 if !git::rev_exists(&winner.worktree, &tracking).await {
7489 return Err(format!("the remote base `{tracking}` could not be read"));
7490 }
7491 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7492 Ok(b) if !b.is_empty() => b,
7493 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7494 };
7495 let evidence = crate::pr_outcome::summary_evidence(
7496 &self.state,
7497 crate::pr_outcome::FinalHead::At(head),
7498 );
7499 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7500 if self
7501 .state
7502 .change_summary
7503 .as_ref()
7504 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7505 {
7506 return Ok(());
7507 }
7508 if self.state.change_summary.take().is_some() {
7509 self.state.event(
7510 "pr-summary",
7511 "the final diff or evidence changed; the earlier change summary was discarded",
7512 );
7513 }
7514 if let Err(e) = write_artifact(
7516 &self.state,
7517 &format!("public-description-{fingerprint}.md"),
7518 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7519 ) {
7520 tracing::warn!("could not save public description evidence: {e:#}");
7521 }
7522 let chain = match agent::pick_chain(
7523 &self.state.config.agents,
7524 self.state.config.roles.synthesizer.as_ref(),
7525 &agent::installed,
7526 "synthesizer",
7527 ) {
7528 Ok(chain) => chain,
7529 Err(e) => {
7530 tracing::warn!("no agent for the change summary: {e:#}");
7531 self.state.event("pr-summary", "no summary agent available");
7532 return Err("no agent was available to summarize it".to_owned());
7533 }
7534 };
7535 let subjects = git::subjects(&winner.worktree, &base, head)
7536 .await
7537 .unwrap_or_default();
7538 let stat = git::diff_stat(&winner.worktree, &base, head)
7539 .await
7540 .unwrap_or_default();
7541 let stat: String = stat.chars().take(6_000).collect();
7542 let mut prompt = crate::prompt::change_summary_in(
7543 &base,
7544 head,
7545 &subjects,
7546 stat.trim(),
7547 &self.state.config.graph.github_language,
7548 );
7549 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7550 let sessions = self.state.config.graph.sessions;
7551 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7552 let prompts = self.state.config.prompts.clone();
7553 let cache = self.state.config.cache_dir();
7554 let run = self.state.id.clone();
7555 for (i, spec) in chain.into_iter().enumerate() {
7556 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7557 let job = SeatJob {
7558 spec: spec.clone(),
7559 seat,
7560 cwd: winner.worktree.clone(),
7561 prompt: prompt.clone(),
7562 timeout,
7563 allow_write: false,
7564 sessions,
7565 artifacts: agent::artifacts_dir(&self.state.dir()),
7566 stem: if i == 0 {
7567 "change-summary".to_owned()
7568 } else {
7569 format!("change-summary-{}", spec.id)
7570 },
7571 handover: None,
7572 };
7573 let ctx = WaveCtx {
7574 run: &run,
7575 node: "pr-summary",
7576 prompts: &prompts,
7577 cache: cache.as_deref(),
7578 round: None,
7579 carry_seats: false,
7580 pauses: Some((&self.pause, &self.interrupt)),
7581 };
7582 let (seat, outcome) =
7583 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7584 self.state.seats.insert(seat.key.clone(), seat);
7585 let text = match outcome {
7586 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7587 AgentOutcome::ResourceBlocked(why) => {
7588 if self.state.resource_block.is_none() {
7591 self.state.resource_block = Some(crate::run::ResourceBlock {
7592 node: "pr-summary".to_owned(),
7593 kind: "shared resource".to_owned(),
7594 detail: why,
7595 at: Timestamp::now(),
7596 });
7597 }
7598 return Err("a shared resource was busy".to_owned());
7599 }
7600 _ => continue,
7601 };
7602 let blind = &self.state.config.blind;
7603 let Some(title) = summary_title(&text) else {
7604 continue;
7605 };
7606 let body: String = summary_without_title(&text)
7607 .lines()
7608 .filter(|l| !l.contains("magi:run/"))
7609 .collect::<Vec<_>>()
7610 .join("\n");
7611 let body = blind::sanitize_prose(body.trim(), blind);
7612 let title = blind::sanitize_prose(&title, blind);
7613 if body.trim().is_empty() || title.trim().is_empty() {
7614 continue;
7615 }
7616 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7617 continue;
7618 }
7619 if git::rev_parse(&winner.worktree, "HEAD")
7621 .await
7622 .ok()
7623 .as_deref()
7624 != Some(head)
7625 {
7626 return Err("the branch moved while the summary was written".to_owned());
7627 }
7628 if git::merge_base(&winner.worktree, &tracking, head)
7629 .await
7630 .ok()
7631 .as_deref()
7632 != Some(base.as_str())
7633 || crate::pr_outcome::summary_evidence(
7634 &self.state,
7635 crate::pr_outcome::FinalHead::At(head),
7636 ) != evidence
7637 {
7638 return Err("the summary evidence changed while it was written".to_owned());
7639 }
7640 self.state.change_summary = Some(crate::run::ChangeSummary {
7641 head: head.to_owned(),
7642 base,
7643 evidence: fingerprint,
7644 title,
7645 body,
7646 });
7647 self.state.event(
7648 "pr-summary",
7649 format!("change summary written by {} for the final diff", spec.id),
7650 );
7651 if let Err(e) = self.state.save() {
7652 tracing::warn!("could not save the change summary: {e:#}");
7653 }
7654 return Ok(());
7655 }
7656 self.state
7657 .event("pr-summary", "no agent produced a change summary");
7658 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7659 }
7660
7661 async fn rewrite_pr_text(
7669 &mut self,
7670 winner: &Candidate,
7671 pr: &PrMessage,
7672 violations: &[crate::github_text::Violation],
7673 ) -> Option<String> {
7674 let prompt = format!(
7675 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7676 pr.title, pr.body
7677 );
7678 let sessions = self.state.config.graph.sessions;
7679 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7680 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7682 let key = format!("impl-{}", winner.label);
7683 if let Some(spec) = self
7684 .state
7685 .config
7686 .agents
7687 .iter()
7688 .find(|a| a.id == winner.agent)
7689 .cloned()
7690 && let Some(existing) = self.state.seats.get(&key)
7691 && existing.agent == spec.id
7692 && agent::has_session(spec.kind, existing, sessions)
7693 {
7694 attempts.push((spec, existing.clone(), true));
7695 } else {
7696 let chain = agent::pick_chain(
7697 &self.state.config.agents,
7698 self.state.config.roles.language_judge.as_ref(),
7699 &agent::installed,
7700 "language_judge",
7701 );
7702 match chain {
7703 Ok(chain) => {
7704 for spec in chain {
7705 let seat = crate::fixer::seat_for(
7706 &mut self.state,
7707 "github-text-rewrite",
7708 &spec.id,
7709 );
7710 attempts.push((spec, seat, false));
7711 }
7712 }
7713 Err(e) => {
7714 tracing::warn!("no agent for the description rewrite: {e:#}");
7715 self.state
7716 .event("github-text", "no rewrite agent available");
7717 return None;
7718 }
7719 }
7720 }
7721 let prompts = self.state.config.prompts.clone();
7722 let cache = self.state.config.cache_dir();
7723 let run = self.state.id.clone();
7724 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7725 let job = SeatJob {
7726 spec: spec.clone(),
7727 seat,
7728 cwd: winner.worktree.clone(),
7729 prompt: prompt.clone(),
7730 timeout: retry_budget(timeout_full, resumed),
7731 allow_write: false,
7732 sessions,
7733 artifacts: agent::artifacts_dir(&self.state.dir()),
7734 stem: if i == 0 {
7735 "github-text-rewrite".to_owned()
7736 } else {
7737 format!("github-text-rewrite-{}", spec.id)
7738 },
7739 handover: None,
7740 };
7741 let ctx = WaveCtx {
7742 run: &run,
7743 node: "github-text",
7744 prompts: &prompts,
7745 cache: cache.as_deref(),
7746 round: None,
7747 carry_seats: false,
7748 pauses: Some((&self.pause, &self.interrupt)),
7749 };
7750 let (seat, outcome) =
7751 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7752 self.state.seats.insert(seat.key.clone(), seat);
7753 match outcome {
7754 AgentOutcome::Ok(o)
7755 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7756 {
7757 return Some(o.text);
7758 }
7759 AgentOutcome::ResourceBlocked(_) => return None,
7760 _ => {}
7761 }
7762 }
7763 None
7764 }
7765
7766 fn retire_github_text_question(&mut self, why: &str) {
7768 let Some(id) = self
7769 .state
7770 .github_text
7771 .as_ref()
7772 .and_then(|g| g.question.clone())
7773 else {
7774 return;
7775 };
7776 let _ = ask::Questions::open().update(&id, |q| {
7777 q.abandon(why);
7778 Ok(())
7779 });
7780 }
7781
7782 async fn settle_withheld(
7786 &mut self,
7787 winner: &Candidate,
7788 pr: &PrMessage,
7789 mut g: crate::run::GithubTextAsk,
7790 ) -> Result<Option<PrMessage>> {
7791 use crate::github_text as gt;
7792 let store = ask::Questions::open();
7793 let timeout = self.state.config.graph.answer_timeout;
7794 if !g.resolved {
7795 let mut standing = g
7796 .question
7797 .as_ref()
7798 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7799 if let Some(q) = standing.as_ref()
7800 && q.status.open()
7801 && gt::expired(q, timeout)
7802 {
7803 let why = format!("no answer within {}s of asking", timeout.max(1));
7804 standing = store
7805 .update(&q.id, |q| {
7806 q.abandon(&why);
7807 Ok(())
7808 })
7809 .ok()
7810 .map(|(q, ())| q);
7811 }
7812 let mut ask_again: Option<Option<Vec<String>>> = None;
7813 match standing {
7814 None if g.asks <= 1 => {
7817 g.asks = 0;
7818 ask_again = Some(None);
7819 }
7820 None => {
7821 g.resolved = true;
7822 g.chosen_title = None;
7823 g.choice = Some(GithubTextChoice::Fallback);
7824 }
7825 Some(q) if q.status.open() => {
7826 self.park_on_github_text(&g)?;
7827 return Ok(None);
7828 }
7829 Some(q) => match gt::read_reply(&q) {
7830 gt::Reply::Fallback => {
7831 g.resolved = true;
7832 g.chosen_title = None;
7833 g.choice = Some(GithubTextChoice::Fallback);
7834 }
7835 gt::Reply::Original => {
7836 g.resolved = true;
7837 g.chosen_title = None;
7838 g.choice = Some(GithubTextChoice::Original);
7839 }
7840 gt::Reply::Rewrite => {
7841 g.resolved = true;
7842 g.chosen_title = None;
7843 g.choice = Some(if g.rewrite.is_some() {
7844 GithubTextChoice::Rewrite
7845 } else {
7846 GithubTextChoice::Fallback
7847 });
7848 }
7849 gt::Reply::Title(text) => {
7850 let first = text
7851 .lines()
7852 .map(str::trim)
7853 .find(|l| !l.is_empty())
7854 .unwrap_or("");
7855 let decision = if first.is_empty() {
7856 None
7857 } else {
7858 let candidate = PrMessage {
7859 title: first.to_owned(),
7860 body: String::new(),
7861 };
7862 self.judge_pr_language(&winner.worktree, &candidate).await
7863 };
7864 match gt::vet_title(
7865 &text,
7866 decision,
7867 self.state.config.graph.github_english(),
7868 ) {
7869 Ok(title) => {
7870 g.resolved = true;
7871 g.chosen_title = Some(title);
7872 g.choice = Some(GithubTextChoice::MyTitle);
7873 }
7874 Err(cats) if g.asks < 2 => {
7875 ask_again = Some(Some(
7876 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7877 ))
7878 }
7879 Err(_) => {
7880 g.resolved = true;
7881 g.chosen_title = None;
7882 g.choice = Some(GithubTextChoice::Fallback);
7883 }
7884 }
7885 }
7886 },
7887 }
7888 if let Some(rejected) = ask_again {
7889 let w = gt::Withheld {
7892 title: g.title,
7893 body: g.body,
7894 categories: g.categories.clone(),
7895 };
7896 let artifact = match crate::run::write_artifact(
7899 &self.state,
7900 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7901 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7902 ) {
7903 Ok(p) => Some(p),
7904 Err(e) => {
7905 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7906 None
7907 }
7908 };
7909 let (detail, comparison) = gt::question_content(
7910 &w,
7911 &pr.title,
7912 &pr.body,
7913 &self.state.id,
7914 rejected.as_deref(),
7915 artifact.as_deref(),
7916 g.rewrite.as_ref(),
7917 );
7918 let mut q = ask::Question::new(
7919 self.state.id.clone(),
7920 gt::ASK_NODE.to_owned(),
7921 gt::ASK_SEAT.to_owned(),
7922 gt::question_summary(&self.state.config.graph.language, &w),
7923 detail,
7924 gt::question_choices(&w, g.rewrite.as_ref()),
7925 );
7926 q.posting_comparison = Some(comparison);
7927 q.answer_timeout = timeout;
7928 g.question = Some(q.id.clone());
7931 g.asks += 1;
7932 self.state.github_text = Some(g.clone());
7933 self.state.event(
7934 "github-text",
7935 format!("asking the owner about the withheld text ({})", q.short()),
7936 );
7937 self.state.save()?;
7938 store
7939 .put_executable(&mut q)
7940 .context("file the posting-gate question")?;
7941 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7942 tracing::warn!(
7943 "could not notify about posting-gate question {}: {e:#}",
7944 q.short()
7945 );
7946 }
7947 self.park_on_github_text(&g)?;
7948 return Ok(None);
7949 }
7950 }
7951 let resolved = g.chosen_title.clone();
7952 let original = g.choice == Some(GithubTextChoice::Original);
7953 self.state.event(
7955 "github-text",
7956 if original {
7957 format!(
7958 "posting the original text for: {}",
7959 match (g.title, g.body) {
7960 (true, true) => "title, description",
7961 (true, false) => "title",
7962 _ => "description",
7963 }
7964 )
7965 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7966 "posting the saved rewrite title and description".to_owned()
7967 } else if resolved.is_some() {
7968 "posting the owner's replacement title".to_owned()
7969 } else {
7970 "posting the neutral text for the withheld fields".to_owned()
7971 },
7972 );
7973 self.state.github_text = Some(g.clone());
7974 self.state.save()?;
7975 if g.choice == Some(GithubTextChoice::Rewrite)
7976 && let Some(r) = g.rewrite.as_ref()
7977 {
7978 return Ok(Some(PrMessage {
7979 title: r.title.clone(),
7980 body: r.body.clone(),
7981 }));
7982 }
7983 let w = gt::Withheld {
7984 title: g.title,
7985 body: g.body,
7986 categories: g.categories.clone(),
7987 };
7988 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7989 if let Some(t) = resolved {
7990 title = t;
7991 }
7992 let english = gt::LanguageDecision {
7994 title_english: true,
7995 body_english: true,
7996 body_complete: true,
7997 };
7998 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7999 if original {
8000 if g.title {
8003 title = pr.title.clone();
8004 }
8005 if g.body {
8006 body = pr.body.clone();
8007 }
8008 }
8009 Ok(Some(PrMessage { title, body }))
8010 }
8011
8012 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8013 self.state.github_text = Some(g.clone());
8014 self.state.end_pass_parked(ParkCause::Question);
8015 self.state.parked = true;
8016 self.state.event(
8017 "github-text",
8018 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8019 );
8020 self.state.save()
8021 }
8022
8023 async fn judge_pr_language(
8026 &mut self,
8027 cwd: &Path,
8028 pr: &PrMessage,
8029 ) -> Option<crate::github_text::LanguageDecision> {
8030 self.state.config.roles.language_judge.as_ref()?;
8031 if !self.state.config.graph.language_gate() {
8032 return None;
8033 }
8034 let decision =
8035 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8036 self.state.event(
8037 "github-text",
8038 if decision.is_some() {
8039 "language decided by the language judge"
8040 } else {
8041 "language judge unavailable; using built-in heuristics"
8042 },
8043 );
8044 decision
8045 }
8046
8047 async fn merge(&mut self) -> Result<()> {
8048 if self
8063 .state
8064 .base_sync
8065 .as_ref()
8066 .is_some_and(|s| s.conflict.is_some())
8067 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8068 != Some(RunStatus::Gating)
8069 || !self.state.gate_allows_progress()
8077 {
8078 return Ok(());
8079 }
8080 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8089 return Ok(());
8090 }
8091 let Some(winner) = self.state.winner().cloned() else {
8092 return Ok(());
8093 };
8094 let repo = self.state.repo.clone();
8095 let base = self.state.base_branch.clone();
8096 let mode = self.state.config.merge.mode;
8097 let style = self.state.config.merge.style;
8098 let facts = if is_review_run(&self.state) {
8099 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8100 let start = review_base(
8101 &repo,
8102 &self.state.config.merge.remote,
8103 &base,
8104 &self.state.base_commit,
8105 &winner.branch,
8106 )
8107 .await;
8108 branch_facts(&repo, &start, &winner.branch).await
8109 } else {
8110 None
8111 };
8112 let leaked = if is_review_run(&self.state) {
8115 leaked_subjects(&self.state, &winner.branch).await
8116 } else {
8117 Some(Vec::new())
8118 };
8119 let Some(pr) = self
8123 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8124 .await?
8125 else {
8126 return Ok(());
8127 };
8128 let message = pr.commit_message();
8129
8130 let outcome = match mode {
8131 MergeMode::None => MergeOutcome {
8132 mode,
8133 ok: true,
8134 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8135 empty: false,
8136 },
8137 MergeMode::Pr | MergeMode::Local
8138 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8139 {
8140 MergeOutcome {
8141 mode,
8142 ok: false,
8143 detail: empty_candidate_detail(&self.state, &base),
8144 empty: true,
8145 }
8146 }
8147 MergeMode::Local => {
8148 let on = git::current_branch(&repo).await?;
8149 if on.as_deref() != Some(base.as_str()) {
8150 MergeOutcome {
8151 mode,
8152 ok: false,
8153 detail: format!(
8154 "{} has {} checked out, not the base branch {base}",
8155 repo.display(),
8156 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8157 ),
8158 empty: false,
8159 }
8160 } else if !git::is_clean(&repo).await? {
8161 MergeOutcome {
8162 mode,
8163 ok: false,
8164 detail: format!("{} is dirty; refusing to merge", repo.display()),
8165 empty: false,
8166 }
8167 } else {
8168 let out = match style {
8169 MergeStyle::Merge => {
8170 git::merge_no_ff(&repo, &winner.branch, &message).await?
8171 }
8172 MergeStyle::Squash => {
8173 git::merge_squash(&repo, &winner.branch, &message).await?
8174 }
8175 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8176 };
8177 MergeOutcome {
8178 mode,
8179 ok: out.ok(),
8180 detail: if out.ok() { out.stdout } else { out.stderr },
8181 empty: false,
8182 }
8183 }
8184 }
8185 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8186 self.state.pr_publication_pending = true;
8187 self.state.save()?;
8188 let out =
8189 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8190 self.state.pr_publication_pending = out.is_err();
8191 MergeOutcome {
8192 mode,
8193 ok: out.is_ok(),
8194 detail: match out {
8195 Ok(url) => url,
8196 Err(e) => format!("{e:#}"),
8197 },
8198 empty: false,
8199 }
8200 }
8201 MergeMode::Pr => {
8202 let remote = self.state.config.merge.remote.clone();
8203 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8204 if !pushed.ok() {
8205 MergeOutcome {
8206 mode,
8207 ok: false,
8208 detail: pushed.stderr,
8209 empty: false,
8210 }
8211 } else {
8212 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8218 let out = match pr_merge_plan(found) {
8219 PrPlan::Create => {
8220 gh_pr_create(
8221 &winner.worktree,
8222 &base,
8223 &winner.branch,
8224 &pr.title,
8225 &pr.body,
8226 )
8227 .await
8228 }
8229 PrPlan::Adopt { url, title } => {
8230 self.state
8231 .event("merge", format!("Pr: adopted open pull request {url}"));
8232 if self.state.github_text.as_ref().is_some_and(|g| {
8233 g.resolved
8234 && g.choice == Some(GithubTextChoice::Rewrite)
8235 && g.rewrite.is_some()
8236 }) {
8237 land::set_pr_text_verbatim(
8240 &winner.worktree,
8241 &url,
8242 &pr.title,
8243 &pr.body,
8244 )
8245 .await
8246 .map(|()| url)
8247 } else {
8248 if title != pr.title
8249 && (!is_review_run(&self.state)
8250 || leaked
8251 .as_deref()
8252 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8253 && let Err(e) =
8254 if self.state.github_text.as_ref().is_some_and(|g| {
8255 g.resolved
8256 && g.title
8257 && g.choice == Some(GithubTextChoice::Original)
8258 }) {
8259 land::set_pr_title_verbatim(
8261 &winner.worktree,
8262 &url,
8263 &pr.title,
8264 )
8265 .await
8266 } else {
8267 land::set_pr_title(
8268 &mut self.state,
8269 &winner.worktree,
8270 &url,
8271 &pr.title,
8272 )
8273 .await
8274 }
8275 {
8276 tracing::warn!("could not refresh title of {url}: {e:#}");
8277 self.state
8278 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8279 }
8280 Ok(url)
8281 }
8282 }
8283 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8284 };
8285 match out {
8286 Ok(url) => MergeOutcome {
8287 mode,
8288 ok: true,
8289 detail: url,
8290 empty: false,
8291 },
8292 Err(e) => MergeOutcome {
8293 mode,
8294 ok: false,
8295 detail: e.to_string(),
8296 empty: false,
8297 },
8298 }
8299 }
8300 }
8301 };
8302
8303 self.state.status = match (mode, outcome.ok) {
8304 (MergeMode::None, _) => RunStatus::Ready,
8305 (_, true) if outcome.empty => RunStatus::Ready,
8306 (MergeMode::Pr, true) => RunStatus::Ready,
8307 (_, true) => RunStatus::Merged,
8308 (_, false) => RunStatus::Blocked,
8309 };
8310 self.state.event(
8311 "merge",
8312 format!(
8313 "{:?}: {}",
8314 mode,
8315 outcome.detail.lines().next().unwrap_or("")
8316 ),
8317 );
8318 self.state.merge = Some(outcome);
8319 self.state.save()?;
8320
8321 if self.state.config.graph.land
8327 && mode == MergeMode::Pr
8328 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8329 {
8330 self.run_land().await?;
8331 }
8332 self.settle_questions();
8337 Ok(())
8338 }
8339
8340 async fn run_land(&mut self) -> Result<()> {
8351 crate::pr_head::ensure_owner(&self.state)?;
8352 let url = self
8353 .state
8354 .pr
8355 .as_ref()
8356 .map(|pr| pr.url.clone())
8357 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8358 .unwrap_or_default();
8359 let url = url.lines().next().unwrap_or("").trim().to_owned();
8360 if !url.starts_with("http") {
8361 return Ok(());
8362 }
8363 loop {
8366 match land::land(&mut self.state, &url).await {
8367 Ok(_)
8368 if self
8369 .state
8370 .merge_repairs
8371 .last()
8372 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8373 {
8374 if self.repair_merge_review().await? {
8375 continue;
8376 }
8377 }
8378 Ok(pr) if self.state.parked => {
8379 let _ = pr;
8383 }
8384 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8385 self.state.status = RunStatus::Superseded;
8386 self.state.save()?;
8387 }
8388 Ok(pr) => {
8389 self.state.status = match pr.state {
8390 land::PrLifecycle::Merged => RunStatus::Merged,
8391 _ => RunStatus::Blocked,
8392 };
8393 if bump::should_release_bump(self.state.status)
8400 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8401 {
8402 self.state
8408 .event("bump", format!("release bump skipped: {e:#}"));
8409 }
8410 if self.state.status == RunStatus::Merged {
8413 crate::followup::after_merge(&mut self.state, &pr.url).await;
8414 }
8415 self.state.save()?;
8416 }
8417 Err(e) => {
8418 if e.is::<crate::repo_lock::ResourceBlocked>() {
8419 return Err(e);
8420 }
8421 self.state.status = RunStatus::Blocked;
8422 self.state.event("land", format!("gave up: {e}"));
8423 self.state.save()?;
8424 }
8425 }
8426 break;
8427 }
8428 Ok(())
8429 }
8430
8431 async fn repair_merge_review(&mut self) -> Result<bool> {
8434 use crate::run::MergeRepairPhase as Phase;
8435 crate::pr_head::ensure_owner(&self.state)?;
8436 let i = self.state.merge_repairs.len() - 1;
8437 let request = self.state.merge_repairs[i].clone();
8438 if request.phase == Phase::Fixing {
8439 let restored = match self.state.winner().cloned() {
8440 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8441 None => Err(anyhow::anyhow!("repair has no winning branch")),
8442 };
8443 let why = match restored {
8444 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8445 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8446 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8447 };
8448 self.hold_merge_repair(i, why)?;
8449 return Ok(false);
8450 }
8451 let failure = if i >= self.state.config.graph.land_rounds
8452 || self.state.config.graph.review_rounds == 0
8453 {
8454 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8455 } else {
8456 None
8457 };
8458 if let Some(why) = failure {
8459 self.hold_merge_repair(i, why)?;
8460 return Ok(false);
8461 }
8462 if request.phase == Phase::Requested {
8463 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8464 Ok(pr) => pr,
8465 Err(e) => {
8466 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8467 return Ok(false);
8468 }
8469 };
8470 self.state.merge_repairs[i].phase = Phase::Fixing;
8471 self.state.land_approval = None;
8472 self.state.land_armed_head = None;
8473 self.state.save()?;
8474 let evidence = serde_json::to_string_pretty(&request)?;
8477 let reason = format!(
8478 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8479 );
8480 let budget = self.state.config.graph.land_rounds;
8481 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8482 match result {
8483 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8484 Ok(land::Fixed::Failed(why)) => {
8485 self.hold_merge_repair(
8486 i,
8487 format!(
8488 "repair failed: {why}; inspect the branch and run an explicit review"
8489 ),
8490 )?;
8491 return Ok(false);
8492 }
8493 Err(e) => {
8494 self.hold_merge_repair(
8495 i,
8496 format!(
8497 "repair failed: {e:#}; inspect the branch and run an explicit review"
8498 ),
8499 )?;
8500 return Ok(false);
8501 }
8502 }
8503 self.begin_merge_repair_review(i)?;
8507 }
8508 self.review_loop().await?;
8509 self.gate().await?;
8510 if self.resource_halt()? || self.state.parked {
8511 return Ok(false);
8512 }
8513 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8514 != Some(RunStatus::Gating)
8515 || !self.state.gate_allows_progress()
8516 {
8517 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8518 return Ok(false);
8519 }
8520 let winner = self
8523 .state
8524 .winner()
8525 .cloned()
8526 .context("repair has no winning branch")?;
8527 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8528 if self.state.gate_head.as_deref() != Some(after.as_str())
8529 || !git::is_clean(&winner.worktree).await?
8530 {
8531 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8532 return Ok(false);
8533 }
8534 let expected = self
8535 .state
8536 .pr_pushed_head
8537 .as_deref()
8538 .unwrap_or(&request.head)
8539 .to_owned();
8540 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8541 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8544 let refspec = format!("{after}:refs/heads/{public}");
8545 let push = git::git_raw(
8546 &winner.worktree,
8547 &["push", &lease, &self.state.config.merge.remote, &refspec],
8548 )
8549 .await;
8550 match push {
8551 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8552 Ok(out) => {
8553 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8554 return Ok(false);
8555 }
8556 Err(e) => {
8557 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8558 return Ok(false);
8559 }
8560 }
8561 self.state.merge_repairs[i].phase = Phase::Complete;
8562 self.state.status = RunStatus::Landing;
8563 self.state.event(
8564 "repair",
8565 "new review and gate completed; fresh owner merge decision required",
8566 );
8567 self.state.save()?;
8568 Ok(true)
8569 }
8570
8571 fn merge_repair_artifacts(&self) -> PathBuf {
8572 let artifacts = agent::artifacts_dir(&self.state.dir());
8573 match self
8574 .state
8575 .merge_repairs
8576 .last()
8577 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8578 {
8579 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8580 None => artifacts,
8581 }
8582 }
8583
8584 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8585 self.state.reviews.clear();
8586 self.state.gate.clear();
8587 self.state.gate_ran = false;
8588 self.state.gate_head = None;
8589 self.state.gate_fixes.clear();
8590 self.state.gate_repair_e2e = None;
8591 self.state.pre_gate.clear();
8592 self.state.pre_gate_commit = None;
8593 self.state.contested_handoff = None;
8594 self.state.land_reruns = None;
8595 self.state.land_approval = None;
8596 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8597 self.state.status = RunStatus::Reviewing;
8598 self.state.save()
8599 }
8600
8601 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8602 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8603 self.state.merge_repairs[index].failure = Some(why.clone());
8604 self.state.status = RunStatus::Blocked;
8605 self.state.event("repair", why);
8606 self.state.save()
8607 }
8608
8609 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8613 if let Some(existing) = self.state.seats.get(key)
8614 && existing.agent == agent
8615 {
8616 return existing.clone();
8617 }
8618 let fresh = if self.state.seats.contains_key(key) {
8621 handover_seat(key, agent, self.state.next_seat_seed())
8622 } else {
8623 SeatState::new(key, agent, self.state.seed)
8624 };
8625 self.state.seats.insert(key.to_owned(), fresh.clone());
8626 fresh
8627 }
8628
8629 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8635 match self.state.seats.get(key) {
8636 Some(s) if s.agent != spec.id => {
8637 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8638 }
8639 _ => spec,
8640 }
8641 }
8642
8643 fn view(&self, c: &Candidate) -> CandidateView {
8645 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8646 .unwrap_or_default();
8647 let (patch, _) = blind::sanitize_patch(
8648 &format!("candidate {} patch", c.label),
8649 &raw,
8650 &self.state.config.blind,
8651 );
8652 CandidateView {
8653 label: c.label,
8654 branch: c.branch.clone(),
8655 summary: c.summary.clone(),
8656 stat: c.stat.clone(),
8657 patch,
8658 }
8659 }
8660
8661 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8663 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8664 prompt::judge(
8665 "(see above)",
8666 &views,
8667 self.roles.judges.len(),
8668 base_short,
8669 "en",
8670 )
8671 }
8672
8673 fn vote_prompt_full(
8679 &self,
8680 j: usize,
8681 labels: &[char],
8682 language: &str,
8683 candidates: &[Candidate],
8684 base_short: &str,
8685 ) -> String {
8686 let mut text = format!(
8687 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8688 prompt::final_vote(labels, language),
8689 self.state.instruction,
8690 self.candidate_block(candidates, base_short)
8691 );
8692 if let Some(own) = self
8693 .state
8694 .judgements
8695 .get(j)
8696 .filter(|r| !r.ranking.is_empty())
8697 {
8698 let reasons = own
8699 .reasons
8700 .iter()
8701 .map(|(k, v)| format!("- {k}: {v}"))
8702 .collect::<Vec<_>>()
8703 .join("\n");
8704 text.push_str(&format!(
8705 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8706 own.ranking.iter().collect::<String>(),
8707 if reasons.is_empty() {
8708 ""
8709 } else {
8710 ", because:\n"
8711 }
8712 ));
8713 }
8714 if !self.state.deliberation.is_empty() {
8715 text.push_str("\n# What was argued before this vote\n");
8716 for t in self.transcript(&[], j) {
8717 text.push_str(&format!(
8718 "\n## {}{}\n\n{}\n",
8719 t.who,
8720 if t.is_self { " (you)" } else { "" },
8721 t.body.trim()
8722 ));
8723 }
8724 }
8725 text
8726 }
8727
8728 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8735 let mut turns = Vec::new();
8736 for j in &self.state.judgements {
8737 if j.ranking.is_empty() {
8738 continue;
8739 }
8740 let reasons = j
8741 .reasons
8742 .iter()
8743 .map(|(k, v)| format!("- {k}: {v}"))
8744 .collect::<Vec<_>>()
8745 .join("\n");
8746 turns.push(Turn {
8747 who: format!("Judge {} (opening ranking)", j.judge),
8748 is_self: j.judge == self_idx + 1,
8749 body: format!(
8750 "Ranked {}{}{reasons}",
8751 j.ranking.iter().collect::<String>(),
8752 if reasons.is_empty() {
8753 ""
8754 } else {
8755 ", because:\n"
8756 }
8757 ),
8758 });
8759 }
8760 for t in self
8761 .state
8762 .deliberation
8763 .iter()
8764 .flat_map(|r| r.turns.iter())
8765 .chain(current)
8766 {
8767 turns.push(Turn {
8768 who: format!("Judge {}", t.judge),
8769 is_self: t.judge == self_idx + 1,
8770 body: t.body.clone(),
8771 });
8772 }
8773 turns
8774 }
8775}
8776
8777fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8779 agent::has_session(spec.kind, seat, sessions)
8780}
8781
8782fn next_untried_in_roster<'a>(
8803 roster: &'a [AgentSpec],
8804 start: usize,
8805 tried: &BTreeSet<String>,
8806) -> Option<&'a AgentSpec> {
8807 roster
8808 .get(start + 1..)?
8809 .iter()
8810 .find(|s| !tried.contains(&s.id))
8811}
8812
8813fn pick_successor<'a>(
8834 roster: &'a [AgentSpec],
8835 start: usize,
8836 tried: &BTreeSet<String>,
8837 carried: Option<&BTreeSet<String>>,
8838 others: &BTreeSet<String>,
8839) -> Option<&'a AgentSpec> {
8840 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8841 roster
8842 .get(start + 1..)
8843 .and_then(|tail| {
8844 tail.iter()
8845 .filter(free)
8846 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8847 })
8848 .or_else(|| {
8849 carried
8850 .is_some()
8851 .then(|| roster.iter().find(free))
8852 .flatten()
8853 })
8854 .or_else(|| match carried {
8855 Some(c) => next_for_seat(roster, start, tried, c),
8856 None => next_untried_in_roster(roster, start, tried),
8857 })
8858}
8859
8860fn next_for_seat<'a>(
8870 roster: &'a [AgentSpec],
8871 start: usize,
8872 round_tried: &BTreeSet<String>,
8873 carried_failed: &BTreeSet<String>,
8874) -> Option<&'a AgentSpec> {
8875 roster
8876 .get(start + 1..)?
8877 .iter()
8878 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8879 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8880}
8881
8882fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8888 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8889 return spec;
8890 };
8891 let ok = |id: &str| !h.failed.contains(id);
8892 if let Some(last) = h.last_ok.as_deref()
8893 && ok(last)
8894 && let Some(s) = roster.iter().find(|s| s.id == last)
8895 {
8896 return s.clone();
8897 }
8898 if ok(&spec.id) {
8899 return spec;
8900 }
8901 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8902 roster
8903 .get(from + 1..)
8904 .into_iter()
8905 .flatten()
8906 .chain(roster.iter())
8907 .find(|s| ok(&s.id))
8908 .cloned()
8909 .unwrap_or(spec)
8910}
8911
8912pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8916 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8917}
8918
8919const TIMED_OUT: &str = "timed out";
8922
8923impl FailClass {
8924 fn of(out: &AgentOutcome) -> Option<Self> {
8926 match out {
8927 AgentOutcome::Ok(_) => None,
8928 AgentOutcome::Quota(_) => Some(Self::Quota),
8929 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8930 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8931 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8932 AgentOutcome::ResourceBlocked(_) => None,
8935 }
8936 }
8937
8938 fn stem_word(&self) -> &'static str {
8940 match self {
8941 Self::Quota => "quota",
8942 _ => "handover",
8943 }
8944 }
8945}
8946
8947fn failure_signature(msg: &str) -> String {
8951 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8952 if let Some(code) = line
8955 .strip_prefix("process exited with no usable output (exit code ")
8956 .and_then(|s| s.strip_suffix(')'))
8957 .and_then(|s| s.parse::<i32>().ok())
8958 {
8959 line = format!("exited with some({code}) and no usable output");
8960 } else if matches!(
8961 line.as_str(),
8962 "process exited with no usable output (no exit code reported)"
8963 | "process exited with no usable output (exit code unavailable)"
8964 ) {
8965 line = "exited with none and no usable output".to_owned();
8966 }
8967 let mut out = Vec::new();
8968 for word in line.split_whitespace() {
8969 if word.contains('/') || word.contains('\\') {
8970 out.push("<path>".to_owned());
8971 continue;
8972 }
8973 let mut w = String::new();
8974 let mut in_digits = false;
8975 for c in word.chars() {
8976 if c.is_ascii_digit() {
8977 if !in_digits {
8978 w.push('#');
8979 }
8980 in_digits = true;
8981 } else {
8982 in_digits = false;
8983 w.push(c);
8984 }
8985 }
8986 out.push(w);
8987 }
8988 out.join(" ").chars().take(120).collect()
8989}
8990
8991fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8998 match cur {
8999 FailClass::Quota | FailClass::Timeout => true,
9000 FailClass::Other(_) => prev != Some(cur),
9001 }
9002}
9003
9004fn fail_reason(out: &AgentOutcome) -> String {
9006 match out {
9007 AgentOutcome::Ok(_) => String::new(),
9008 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9009 AgentOutcome::Dropped(o) => format!(
9010 "the CLI dropped the stream ({})",
9011 o.dropped
9012 .as_ref()
9013 .map(|d| d.why.as_str())
9014 .unwrap_or("it ended without delivering its answer")
9015 ),
9016 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9017 e.lines().next().unwrap_or("").chars().take(160).collect()
9018 }
9019 }
9020}
9021
9022pub(crate) fn record_handover(
9025 state: &mut RunState,
9026 node: &str,
9027 seat: &str,
9028 from: &str,
9029 to: &str,
9030 class: &FailClass,
9031 reason: &str,
9032) {
9033 let message = if *class == FailClass::Quota {
9034 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9035 } else {
9036 format!("{seat}: handed over {from} -> {to} ({reason})")
9037 };
9038 state.event(node, message);
9039 state.handovers.push(Handover {
9040 at: Timestamp::now(),
9041 node: node.to_owned(),
9042 seat: seat.to_owned(),
9043 from: from.to_owned(),
9044 to: to.to_owned(),
9045 reason: reason.to_owned(),
9046 });
9047}
9048
9049fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9063 commands.iter().any(|c| c.exit_code.is_none())
9064}
9065
9066fn verified_noop_claim(
9079 usable: bool,
9080 commands: &[agent::CommandEvidence],
9081 text: &str,
9082) -> Option<String> {
9083 (usable && !has_unconfirmed_command(commands))
9084 .then(|| verdict::verified_noop(text))
9085 .flatten()
9086}
9087
9088fn short(commit: &str) -> String {
9089 commit.chars().take(7).collect()
9090}
9091
9092fn make_executable(path: &Path) -> Result<()> {
9093 #[cfg(unix)]
9094 {
9095 use std::os::unix::fs::PermissionsExt as _;
9096 let mut perms = std::fs::metadata(path)?.permissions();
9097 perms.set_mode(0o755);
9098 std::fs::set_permissions(path, perms)?;
9099 }
9100 #[cfg(not(unix))]
9101 {
9102 let _ = path;
9103 }
9104 Ok(())
9105}
9106
9107struct WaveCtx<'a> {
9114 run: &'a str,
9117 node: &'a str,
9119 prompts: &'a Prompts,
9120 cache: Option<&'a Path>,
9122 round: Option<usize>,
9125 carry_seats: bool,
9129 pauses: Option<(&'a Pause, &'a Pause)>,
9132}
9133
9134fn repository_wait(
9136 state: &RunState,
9137 pauses: Option<(&Pause, &Pause)>,
9138) -> crate::repo_lock::WaitPolicy {
9139 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9140 let inherited = crate::repo_lock::WaitPolicy::current();
9141 crate::repo_lock::WaitPolicy::new(
9142 Duration::from_secs(state.config.graph.timeout_resource_wait),
9143 move || {
9144 inherited.cancelled()
9145 || pauses
9146 .as_ref()
9147 .is_some_and(|(a, b)| a.parked() || b.parked())
9148 },
9149 )
9150}
9151
9152fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9153 if error.is::<crate::repo_lock::ResourceBlocked>() {
9154 state.resource_block = Some(crate::run::ResourceBlock {
9155 node: "repository".into(),
9156 kind: if error.to_string().contains("disk pressure") {
9157 "disk pressure"
9158 } else {
9159 "repository coordination"
9160 }
9161 .into(),
9162 detail: error.to_string(),
9163 at: Timestamp::now(),
9164 });
9165 state.event("resource", error.to_string());
9166 }
9167}
9168
9169async fn coordinate_operation(
9170 state: &mut RunState,
9171 cwd: &Path,
9172 wait: &crate::repo_lock::WaitPolicy,
9173) -> Result<Option<crate::repo_lock::SeatLease>> {
9174 let result =
9175 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9176 .await;
9177 if let Err(e) = &result {
9178 record_repository_block(state, e);
9179 }
9180 result
9181}
9182
9183async fn run_one(
9185 job: SeatJob,
9186 sem: Arc<Semaphore>,
9187 ctx: &WaveCtx<'_>,
9188 state: &mut RunState,
9189 attempt: usize,
9190) -> (SeatState, AgentOutcome) {
9191 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9192 .await
9193 .pop()
9194 .expect("one job in, one result out");
9195 (seat, out)
9196}
9197
9198async fn wave(
9204 jobs: Vec<SeatJob>,
9205 sem: Arc<Semaphore>,
9206 ctx: &WaveCtx<'_>,
9207 state: &mut RunState,
9208 attempt: usize,
9209) -> Vec<(usize, SeatState, AgentOutcome)> {
9210 let WaveCtx {
9211 run,
9212 node,
9213 prompts,
9214 cache,
9215 round,
9216 carry_seats: _,
9217 pauses,
9218 } = *ctx;
9219 for job in &jobs {
9220 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9221 }
9222 if let Err(e) = state.save() {
9223 tracing::warn!("could not persist in-progress seats: {e:#}");
9228 }
9229 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9245 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9249 let cache_guard = if let Some(cache_dir) = cache {
9250 if jobs_had_a_writer {
9251 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9252 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9253 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9254 {
9255 Ok(Some(g)) => Some(g),
9256 Ok(None) => {
9259 state.resource_block = Some(crate::run::ResourceBlock {
9260 node: node.to_owned(),
9261 kind: "shared build cache".to_owned(),
9262 detail: "the run is parking".to_owned(),
9263 at: Timestamp::now(),
9264 });
9265 return block_wave(
9266 jobs,
9267 state,
9268 node,
9269 attempt,
9270 "parked while waiting for the shared build cache",
9271 );
9272 }
9273 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9276 Err(e) => {
9277 let why = format!("{e:#}");
9281 state.resource_block = Some(crate::run::ResourceBlock {
9282 node: node.to_owned(),
9283 kind: "shared build cache".to_owned(),
9284 detail: why.clone(),
9285 at: Timestamp::now(),
9286 });
9287 state.event(
9288 "resource",
9289 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9290 );
9291 return block_wave(jobs, state, node, attempt, &why);
9292 }
9293 }
9294 } else {
9295 None
9296 }
9297 } else {
9298 None
9299 };
9300 state.seats_spawning(node, attempt);
9303 if let Err(e) = state.save() {
9304 tracing::warn!("could not persist the start of a wave: {e:#}");
9305 }
9306 let slot_limit = resource_limit
9310 + jobs
9311 .iter()
9312 .map(|j| j.timeout)
9313 .max()
9314 .unwrap_or_default()
9315 .saturating_mul(jobs.len() as u32);
9316 let mut set = tokio::task::JoinSet::new();
9317 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9320 let overlay = prompts.overlay(node);
9321 let repository_wait = repository_wait(state, pauses);
9322 for (i, mut job) in jobs.into_iter().enumerate() {
9323 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9324 if cache.is_some() {
9325 job.prompt.push('\n');
9326 job.prompt
9327 .push_str(&prompt::build_cache_note(node, job.allow_write));
9328 }
9329 let sem = Arc::clone(&sem);
9330 let run = run.to_owned();
9331 let node = node.to_owned();
9332 let attachments = if node == "implement" {
9335 state.attachments.clone()
9336 } else {
9337 Vec::new()
9338 };
9339 let cache = cache
9350 .filter(|_| job.allow_write && cache_guard.is_some())
9351 .map(Path::to_path_buf);
9352 let spawned_tx = spawned_tx.clone();
9353 let repository_wait = repository_wait.clone();
9354 let coordination = state.config.coordination.clone();
9355 let fetched_base = state.fetched_base.clone();
9356 set.spawn(async move {
9357 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9361 Ok(p) => p,
9362 Err(_) => {
9363 return (
9364 i,
9365 job.seat,
9366 AgentOutcome::ResourceBlocked(format!(
9367 "no parallel slot became free within {}s",
9368 slot_limit.as_secs()
9369 )),
9370 false,
9371 );
9372 }
9373 };
9374 spawned_tx.send(job.seat.key.clone()).ok();
9375 let mut seat = job.seat;
9376 let out = repository_wait
9377 .scope(agent::invoke(
9378 &job.spec,
9379 &mut seat,
9380 &Invocation {
9381 scope: crate::agent::InvocationScope::Repository,
9382 coordination: coordination.as_ref(),
9383 fetched_base: fetched_base.as_ref(),
9384 cwd: &job.cwd,
9385 prompt: &job.prompt,
9386 timeout: job.timeout,
9387 allow_write: job.allow_write,
9388 unsandboxed: false,
9389 sessions: job.sessions,
9390 artifacts: &job.artifacts,
9391 stem: &job.stem,
9392 run: &run,
9393 node: &node,
9394 cache_dir: cache.as_deref(),
9395 attachments: &attachments,
9396 writable: &[],
9397 },
9398 ))
9399 .await;
9400 let interrupted = out
9401 .as_ref()
9402 .err()
9403 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9404 let out = match out {
9405 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9406 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9407 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9415 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9416 Ok(o) => AgentOutcome::Failed(format!(
9417 "process exited with no usable output ({})",
9418 crate::notices::exit_status(o.exit_code)
9419 )),
9420 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9421 AgentOutcome::ResourceBlocked(e.to_string())
9422 }
9423 Err(e) => AgentOutcome::Failed(e.to_string()),
9424 };
9425 (i, seat, out, interrupted)
9426 });
9427 }
9428 drop(spawned_tx);
9429 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9430 loop {
9431 let joined = tokio::select! {
9432 Some(key) = spawned_rx.recv() => {
9433 state.seat_spawned(&key);
9435 if let Err(e) = state.save() {
9436 tracing::warn!("could not persist a seat's start: {e:#}");
9437 }
9438 continue;
9439 }
9440 joined = set.join_next() => joined,
9441 };
9442 let Some(joined) = joined else {
9443 break;
9444 };
9445 let (i, seat, out, interrupted) = match joined {
9446 Ok(v) => v,
9447 Err(e) => {
9451 tracing::error!("agent task panicked: {e}");
9452 continue;
9453 }
9454 };
9455 state.seat_finished(&seat.key);
9456 if interrupted
9457 && node == "implement"
9458 && let Some(label) = state
9459 .candidates
9460 .iter()
9461 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9462 .map(|candidate| candidate.label)
9463 {
9464 state.candidate_preservation.insert(
9465 label,
9466 "interrupted implementation awaiting persistence".into(),
9467 );
9468 state.interrupted_implementations.insert(label);
9469 }
9470 if let AgentOutcome::ResourceBlocked(why) = &out
9471 && state.resource_block.is_none()
9472 {
9473 state.resource_block = Some(crate::run::ResourceBlock {
9474 node: node.to_owned(),
9475 kind: if why.contains("disk pressure") {
9476 "disk pressure"
9477 } else if why.contains("repository coordination") {
9478 "repository coordination"
9479 } else {
9480 "parallel slot"
9481 }
9482 .to_owned(),
9483 detail: why.clone(),
9484 at: Timestamp::now(),
9485 });
9486 let activity = if interrupted {
9487 "agent execution interrupted"
9488 } else {
9489 "no agent was started"
9490 };
9491 state.event("resource", format!("{node}: {activity} - {why}"));
9492 }
9493 record_jobs(state, node, round, &seat.key, &out);
9494 if let Err(e) = state.save() {
9495 tracing::warn!("could not persist a seat's completion: {e:#}");
9496 }
9497 if collected.len() <= i {
9498 collected.resize_with(i + 1, || None);
9499 }
9500 collected[i] = Some((i, seat, out));
9501 }
9502 if state
9508 .active
9509 .values()
9510 .any(|a| a.node == node && a.attempt == attempt)
9511 {
9512 state
9513 .active
9514 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9515 if let Err(e) = state.save() {
9516 tracing::warn!("could not persist the end of a wave: {e:#}");
9517 }
9518 }
9519 if let Some(cache_dir) = cache
9526 && jobs_had_a_writer
9527 {
9528 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9529 }
9530 if let Some(guard) = cache_guard {
9531 guard.release();
9532 }
9533 collected.into_iter().flatten().collect()
9534}
9535
9536fn record_jobs(
9547 state: &mut RunState,
9548 node: &str,
9549 round: Option<usize>,
9550 seat: &str,
9551 out: &AgentOutcome,
9552) {
9553 let commands: &[agent::CommandEvidence] = match out {
9554 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9555 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9556 };
9557 let checked_at = Timestamp::now();
9558 for c in commands {
9559 state.jobs.push(JobRecord {
9560 node: node.to_owned(),
9561 round,
9562 seat: seat.to_owned(),
9563 id: c.id.clone(),
9564 description: c.description.clone(),
9565 checked_at,
9566 status: match c.exit_code {
9567 Some(0) => JobStatus::Completed,
9568 Some(_) => JobStatus::Failed,
9569 None => JobStatus::Unknown,
9570 },
9571 exit_code: c.exit_code,
9572 result_summary: c.result_summary.clone(),
9573 source: c.source.clone(),
9574 });
9575 }
9576}
9577
9578fn round_is_clean(
9599 blocking: usize,
9600 e2e_ok: bool,
9601 answered: usize,
9602 expected: usize,
9603 quota_missing: usize,
9604 policy: IncompleteReviewPolicy,
9605) -> bool {
9606 if blocking != 0 || !e2e_ok {
9607 return false;
9608 }
9609 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9610 return true;
9611 }
9612 answered > 0 && expected - answered <= quota_missing
9613}
9614
9615pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9639 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9640 return Some(RunStatus::Gating);
9641 }
9642 let last = reviews.last()?;
9643 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9644 if reviews.len() < max_rounds && !stagnant {
9645 return None;
9646 }
9647 if last.incomplete() && last.blocking == 0 {
9648 return Some(RunStatus::Blocked);
9649 }
9650 if last.e2e_status() == E2eStatus::ResourceBlocked {
9651 return None;
9652 }
9653 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9654 RunStatus::Gating
9655 } else {
9656 RunStatus::Blocked
9657 })
9658}
9659
9660pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9675 if nudged {
9676 (full / 4).max(Duration::from_secs(120)).min(full)
9677 } else {
9678 full
9679 }
9680}
9681
9682#[allow(clippy::too_many_arguments)]
9711async fn ask_json_wave<T>(
9712 jobs: Vec<SeatJob>,
9713 sem: Arc<Semaphore>,
9714 retries: usize,
9715 roster: &[AgentSpec],
9716 ctx: &WaveCtx<'_>,
9717 losses: &mut Vec<QuotaLoss>,
9718 state: &mut RunState,
9719 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9720) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9721where
9722 T: serde::de::DeserializeOwned + Send + 'static,
9723{
9724 ask_wave_with(
9725 jobs,
9726 sem,
9727 retries,
9728 roster,
9729 ctx,
9730 losses,
9731 state,
9732 &|text: &str| {
9733 let v = verdict::extract_json::<T>(text)?;
9734 validate(&v)?;
9735 Ok(v)
9736 },
9737 )
9738 .await
9739}
9740
9741#[allow(clippy::too_many_arguments)]
9750async fn ask_wave_with<T>(
9751 jobs: Vec<SeatJob>,
9752 sem: Arc<Semaphore>,
9753 retries: usize,
9754 roster: &[AgentSpec],
9755 ctx: &WaveCtx<'_>,
9756 losses: &mut Vec<QuotaLoss>,
9757 state: &mut RunState,
9758 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9759) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9760where
9761 T: Send + 'static,
9762{
9763 let n = jobs.len();
9764 let originals: Vec<SeatJob> = jobs;
9765 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9766 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9767 let mut nudges: Vec<usize> = vec![0; n];
9774 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9778 let mut tried: Vec<BTreeSet<String>> = specs
9779 .iter()
9780 .map(|s| BTreeSet::from([s.id.clone()]))
9781 .collect();
9782 let starts: Vec<usize> = specs
9783 .iter()
9784 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9785 .collect();
9786 let carry = ctx.carry_seats && !roster.is_empty();
9787 let carried: Vec<BTreeSet<String>> = originals
9790 .iter()
9791 .map(|j| {
9792 state
9793 .seat_history
9794 .get(&j.seat.key)
9795 .filter(|_| carry)
9796 .map(|h| h.failed.clone())
9797 .unwrap_or_default()
9798 })
9799 .collect();
9800 let mut prev: Vec<Option<FailClass>> = originals
9801 .iter()
9802 .map(|j| {
9803 state
9804 .seat_history
9805 .get(&j.seat.key)
9806 .filter(|_| carry)
9807 .and_then(|h| h.last_fail.clone())
9808 })
9809 .collect();
9810 let next_agent =
9811 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9812 let others: BTreeSet<String> = specs
9813 .iter()
9814 .enumerate()
9815 .filter(|(j, _)| *j != i)
9816 .map(|(_, s)| s.id.clone())
9817 .collect();
9818 pick_successor(
9819 roster,
9820 starts[i],
9821 tried,
9822 carry.then(|| &carried[i]),
9823 &others,
9824 )
9825 .cloned()
9826 };
9827 let mut fresh: Vec<Option<String>> = vec![None; n];
9828 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9829 let mut pending: Vec<usize> = (0..n).collect();
9830
9831 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9836 for round in 0..max_rounds {
9837 if pending.is_empty() {
9838 break;
9839 }
9840 let mut batch = Vec::with_capacity(pending.len());
9841 let mut renudged: Vec<&str> = Vec::new();
9842 for &i in &pending {
9843 let src = &originals[i];
9844 let full: &str = match &src.handover {
9849 Some(h) if specs[i].id != src.spec.id => h,
9850 _ => &src.prompt,
9851 };
9852 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9855 (
9856 full.to_owned(),
9857 src.timeout,
9858 format!("{}-{word}-{}", src.stem, specs[i].id),
9859 )
9860 } else if nudges[i] == 0 {
9861 (full.to_owned(), src.timeout, src.stem.clone())
9862 } else {
9863 renudged.push(src.seat.key.as_str());
9864 let why = done[i]
9865 .as_ref()
9866 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9867 .unwrap_or_else(|| "no parsable answer".to_owned());
9868 let nudge = prompt::nudge(&why);
9869 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9870 let prompt = if nudged {
9871 nudge
9872 } else {
9873 format!("{full}\n\n---\n\n{nudge}")
9874 };
9875 (
9876 prompt,
9877 retry_budget(src.timeout, nudged),
9878 format!("{}-retry{}", src.stem, nudges[i]),
9879 )
9880 };
9881 batch.push(SeatJob {
9882 spec: specs[i].clone(),
9883 seat: seats[i].clone(),
9884 cwd: src.cwd.clone(),
9885 prompt,
9886 timeout,
9887 allow_write: src.allow_write,
9888 sessions: src.sessions,
9889 artifacts: src.artifacts.clone(),
9890 stem,
9891 handover: None,
9892 });
9893 }
9894
9895 if !renudged.is_empty() {
9896 state.event(
9897 ctx.node,
9898 format!("retry {round}: re-asking {}", renudged.join(", ")),
9899 );
9900 }
9901 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9902 let mut still = Vec::new();
9903 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9904 seats[i] = seat;
9905 if let AgentOutcome::ResourceBlocked(why) = &out {
9908 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9909 continue;
9910 }
9911 let class = FailClass::of(&out);
9912 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9915 && nudges[i] < retries
9916 && !roster.is_empty();
9917 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9918 let next = should_hand_over(prev[i].as_ref(), &cur)
9919 .then(|| next_agent(i, &tried[i], &specs))
9920 .flatten();
9921 if carry {
9922 let h = state
9923 .seat_history
9924 .entry(originals[i].seat.key.clone())
9925 .or_default();
9926 h.failed.insert(specs[i].id.clone());
9927 h.last_fail = Some(cur.clone());
9928 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9929 h.last_ok = None;
9930 }
9931 if let Err(e) = state.save() {
9934 tracing::warn!("could not persist a seat's failure history: {e:#}");
9935 }
9936 }
9937 if let Some(next) = next {
9938 record_handover(
9939 state,
9940 ctx.node,
9941 &originals[i].seat.key,
9942 &specs[i].id,
9943 &next.id,
9944 &cur,
9945 &fail_reason(&out),
9946 );
9947 tried[i].insert(next.id.clone());
9948 prev[i] = Some(cur.clone());
9949 seats[i] =
9950 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9951 specs[i] = next;
9952 fresh[i] = Some(cur.stem_word().to_owned());
9953 nudges[i] = 0;
9954 done[i] = Some(Err(anyhow::anyhow!(
9955 "handed over after: {}",
9956 fail_reason(&out)
9957 )));
9958 still.push(i);
9959 continue;
9960 }
9961 }
9962 if carry
9963 && !nudge_first
9964 && let Some(cur) = class.clone()
9965 {
9966 let h = state
9968 .seat_history
9969 .entry(originals[i].seat.key.clone())
9970 .or_default();
9971 h.failed.insert(specs[i].id.clone());
9972 h.last_fail = Some(cur);
9973 }
9974 let parsed = match out {
9975 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9976 AgentOutcome::Quota(o) => {
9977 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9978 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9979 }
9980 AgentOutcome::Dropped(o) => {
9984 let why = o
9985 .dropped
9986 .as_ref()
9987 .map(|d| d.why.as_str())
9988 .unwrap_or("the CLI ended the stream without delivering its answer");
9989 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9990 }
9991 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9992 Err(anyhow::anyhow!(e))
9993 }
9994 };
9995 let quota = class == Some(FailClass::Quota);
9996 let failed = parsed.is_err();
9997 done[i] = Some(parsed);
9998 if carry && !failed {
9999 let h = state
10000 .seat_history
10001 .entry(originals[i].seat.key.clone())
10002 .or_default();
10003 h.failed.remove(&specs[i].id);
10004 h.last_ok = Some(specs[i].id.clone());
10005 h.last_fail = None;
10006 }
10007 let agent_failure = class.is_some()
10015 && !nudge_first
10016 && !roster.is_empty()
10017 && next_agent(i, &tried[i], &specs).is_some();
10018 if failed && !quota && !agent_failure && nudges[i] < retries {
10019 nudges[i] += 1;
10020 still.push(i);
10021 }
10022 }
10023 pending = still;
10024 }
10025
10026 for (i, q) in last_quota.into_iter().enumerate() {
10029 if let Some(reset) = q {
10030 losses.push(QuotaLoss {
10031 seat: originals[i].seat.key.clone(),
10032 node: ctx.node.to_owned(),
10033 at: Timestamp::now(),
10034 reset,
10035 });
10036 }
10037 }
10038
10039 seats
10040 .into_iter()
10041 .zip(done)
10042 .zip(nudges)
10043 .map(|((seat, res), attempts)| {
10044 (
10045 seat,
10046 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10047 attempts,
10048 )
10049 })
10050 .collect()
10051}
10052
10053fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10055 results
10056 .iter()
10057 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10058}
10059
10060fn block_wave(
10064 jobs: Vec<SeatJob>,
10065 state: &mut RunState,
10066 node: &str,
10067 attempt: usize,
10068 why: &str,
10069) -> Vec<(usize, SeatState, AgentOutcome)> {
10070 state.resource_wait = None;
10071 state
10072 .active
10073 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10074 if let Err(e) = state.save() {
10075 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10076 }
10077 jobs.into_iter()
10078 .enumerate()
10079 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10080 .collect()
10081}
10082
10083async fn acquire_cache_lease(
10096 state: &mut RunState,
10097 cache_dir: &Path,
10098 owner: &crate::cache::Owner,
10099 budget: Duration,
10100 context: &str,
10101 cancel: &(dyn Fn() -> bool + Sync),
10102) -> Result<Option<crate::cache::Guard>> {
10103 let home = crate::run::home();
10104 let started = Instant::now();
10105 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10106 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10107 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10108 Err(e) => {
10109 state.event(
10110 "verify",
10111 format!("{context}: could not check the shared build cache: {e:#}"),
10112 );
10113 if let Err(e2) = state.save() {
10114 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10115 }
10116 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10117 }
10118 };
10119 state.event(
10120 "verify",
10121 format!(
10122 "{context}: waiting for the shared build cache at {} ({})",
10123 cache_dir.display(),
10124 busy.describe()
10125 ),
10126 );
10127 state.resource_wait = Some(crate::run::ResourceWait {
10130 node: context.to_owned(),
10131 kind: "shared build cache".to_owned(),
10132 holder: busy.describe(),
10133 since: Timestamp::now(),
10134 limit_secs: budget.as_secs(),
10135 });
10136 if let Err(e) = state.save() {
10137 tracing::warn!("could not persist a cache wait: {e:#}");
10138 }
10139 let remaining = budget.saturating_sub(started.elapsed());
10140 let waited = crate::cache::wait_for_unless(
10141 &home,
10142 cache_dir,
10143 owner,
10144 remaining,
10145 Duration::from_secs(5),
10146 cancel,
10147 )
10148 .await;
10149 state.resource_wait = None;
10150 match waited {
10151 Ok(g) => Ok(g),
10152 Err(e) => {
10153 state.event("verify", format!("{context}: {e:#}"));
10154 if let Err(e2) = state.save() {
10155 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10156 }
10157 Err(e)
10158 }
10159 }
10160}
10161
10162#[derive(Debug)]
10167struct CacheCheckError(String);
10168
10169impl std::fmt::Display for CacheCheckError {
10170 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10171 f.write_str(&self.0)
10172 }
10173}
10174
10175impl std::error::Error for CacheCheckError {}
10176
10177#[allow(clippy::too_many_arguments)]
10198async fn with_cache_lease<'s, F, Fut>(
10199 state: &'s mut RunState,
10200 cache_dir: Option<&Path>,
10201 node: &str,
10202 seat: &str,
10203 worktree: &Path,
10204 head: &str,
10205 budget: Duration,
10206 context: &str,
10207 wait: &crate::repo_lock::WaitPolicy,
10208 body: F,
10209) -> Result<(Vec<CommandOutcome>, bool)>
10210where
10211 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10212 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10213{
10214 let started = Instant::now();
10217 let home = crate::run::home();
10218 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10219 let guard = if let Some(cache_dir) = cache_dir {
10220 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10221 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10222 wait.cancelled()
10223 })
10224 .await
10225 .and_then(|g| {
10226 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10227 }) {
10228 Ok(g) => Some(g),
10229 Err(e) => {
10230 return Ok((
10231 vec![CommandOutcome {
10232 command: "(waiting for the shared build cache)".into(),
10233 code: None,
10234 output_tail: e.to_string(),
10235 duration_ms: started.elapsed().as_millis() as u64,
10236 resource_blocked: true,
10237 }],
10238 false,
10239 ));
10240 }
10241 }
10242 } else {
10243 None
10244 };
10245 let _coordination = coordinate_operation(state, worktree, wait).await?;
10246 if let Some(cache_dir) = cache_dir {
10247 let identity = crate::cache::Identity::new(worktree, head);
10248 if let Err(e) = crate::cache::ensure_fresh(
10249 &home,
10250 cache_dir,
10251 &identity,
10252 &state.config.verify.cargo_manifests(),
10253 ) {
10254 state.event(
10255 "verify",
10256 format!(
10257 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10258 worktree.display(),
10259 short(head)
10260 ),
10261 );
10262 if let Some(guard) = guard {
10263 guard.release();
10264 }
10265 return Ok((
10266 vec![CommandOutcome {
10267 command: "(confirming the shared build cache is fresh)".into(),
10268 code: None,
10269 output_tail: e.to_string(),
10270 duration_ms: started.elapsed().as_millis() as u64,
10271 resource_blocked: true,
10272 }],
10273 false,
10274 ));
10275 }
10276 }
10277 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10278 if !timed_out_pids.is_empty() {
10279 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10280 }
10281 drop(_coordination);
10284 if let Some(guard) = guard {
10285 guard.release();
10286 }
10287 Ok((outcomes, retried))
10288}
10289
10290async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10302 wait_for_pids_with(
10303 pids,
10304 crate::proc::pid_alive,
10305 LEASE_RELEASE_POLL,
10306 LEASE_RELEASE_MAX_WAIT,
10307 )
10308 .await;
10309}
10310
10311async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10317 pids: &[u32],
10318 alive: F,
10319 poll: Duration,
10320 max_wait: Duration,
10321) {
10322 let deadline = Instant::now() + max_wait;
10323 loop {
10324 if pids.iter().all(|&pid| !alive(pid)) {
10325 return;
10326 }
10327 if Instant::now() >= deadline {
10328 return;
10329 }
10330 tokio::time::sleep(poll).await;
10331 }
10332}
10333
10334fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10341 outcomes.iter().any(|o| o.resource_blocked)
10342}
10343
10344pub(crate) enum GateFix {
10346 Retry,
10348 Stop,
10351 Defer,
10354}
10355
10356fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10364 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10365 red.peek().is_some()
10366 && red.all(|o| {
10367 !o.resource_blocked
10368 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10369 && !o.output_tail.trim().is_empty()
10370 })
10371}
10372
10373fn e2e_outcome_label(o: &CommandOutcome) -> String {
10377 if o.ok() {
10378 return "pass".to_owned();
10379 }
10380 let reason = if o.build_failed() {
10381 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10382 } else {
10383 format!("FAIL ({:?})", o.code)
10384 };
10385 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10386}
10387
10388async fn run_e2e_with_retry(
10396 state: &mut RunState,
10397 shell: &[String],
10398 commands: &[String],
10399 worktree: &Path,
10400 timeout: Duration,
10401 context: &str,
10402) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10403 let (mut e2e, mut timed_out_pids) = run_commands(
10404 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10405 )
10406 .await;
10407 for o in &e2e {
10408 state.event(
10409 "verify",
10410 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10411 );
10412 }
10413 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10417 if verify_retried {
10418 state.event(
10419 "verify",
10420 format!(
10421 "{context}: verify could not build/link, not a test result — retrying once \
10422 before concluding"
10423 ),
10424 );
10425 let retried = run_commands(
10426 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10427 )
10428 .await;
10429 e2e = retried.0;
10430 timed_out_pids.extend(retried.1);
10433 for o in &e2e {
10434 state.event(
10435 "verify",
10436 format!(
10437 "{context}: retry `{}` -> {}",
10438 o.command,
10439 e2e_outcome_label(o)
10440 ),
10441 );
10442 }
10443 }
10444 (e2e, verify_retried, timed_out_pids)
10445}
10446
10447#[allow(clippy::too_many_arguments)]
10463async fn run_commands(
10464 state: &mut RunState,
10465 node: &str,
10466 task: &str,
10467 attempt: usize,
10468 shell: &[String],
10469 commands: &[String],
10470 cwd: &Path,
10471 timeout: Duration,
10472) -> (Vec<CommandOutcome>, Vec<u32>) {
10473 if commands.is_empty() {
10474 return (Vec::new(), Vec::new());
10479 }
10480 let mut out = Vec::new();
10481 let mut timed_out_pids = Vec::new();
10482 let total = commands.len();
10483 for (idx, command) in commands.iter().enumerate() {
10484 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10485 if let Err(e) = state.save() {
10486 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10487 }
10488 let started = Instant::now();
10489 let mut cmd = tokio::process::Command::new(&shell[0]);
10490 cmd.quiet();
10491 cmd.args(&shell[1..])
10492 .arg(command)
10493 .current_dir(cwd)
10494 .stdin(std::process::Stdio::null())
10495 .stdout(std::process::Stdio::piped())
10496 .stderr(std::process::Stdio::piped())
10497 .kill_on_drop(true);
10498 let mut outputs = Vec::new();
10499 let protected = (|| -> Result<()> {
10500 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10501 outputs.push(guard);
10502 }
10503 if let Some(cache) = state.config.cache_dir() {
10504 outputs.push(crate::retention::OutputLease::cache(
10505 &crate::run::home(),
10506 &cache,
10507 false,
10508 )?);
10509 }
10510 Ok(())
10511 })();
10512 if let Err(e) = protected {
10513 out.push(CommandOutcome {
10514 command: command.clone(),
10515 code: None,
10516 output_tail: format!("output ownership unavailable: {e:#}"),
10517 duration_ms: 0,
10518 resource_blocked: true,
10519 });
10520 continue;
10521 }
10522 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10523 let (code, body) = match spawned {
10524 Ok(mut running) => {
10525 let pid = running.child().id();
10526 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10527 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10528 let mut paths = vec![cwd.to_owned()];
10529 if let Some(cache) = state.config.cache_dir() {
10530 paths.push(cache);
10531 }
10532 let mut pressure = None;
10533 let status = tokio::select! {
10534 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10535 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10536 pressure = Some(reason); None
10537 }
10538 };
10539 let (code, failure) = match status {
10540 Some(Ok(Ok(status))) => (
10541 status.code(),
10542 status
10543 .code()
10544 .is_none()
10545 .then(|| format!("command terminated: {status}")),
10546 ),
10547 Some(Ok(Err(error))) => {
10548 (None, Some(format!("failed waiting for command: {error}")))
10549 }
10550 Some(Err(_)) => {
10551 if !running.coordinated()
10555 && let Some(pid) = pid
10556 {
10557 timed_out_pids.push(pid);
10558 }
10559 (
10560 None,
10561 Some(format!("timed out after {}s", timeout.as_secs())),
10562 )
10563 }
10564 None => (None, None),
10565 };
10566 running.finish().await;
10567 if let Some(reason) = pressure {
10568 state.resource_block = Some(crate::run::ResourceBlock {
10569 node: node.to_owned(),
10570 kind: "disk pressure".to_owned(),
10571 detail: reason.clone(),
10572 at: Timestamp::now(),
10573 });
10574 out.push(CommandOutcome {
10575 command: command.clone(),
10576 code: None,
10577 output_tail: reason,
10578 duration_ms: started.elapsed().as_millis() as u64,
10579 resource_blocked: true,
10580 });
10581 break;
10582 }
10583 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10584 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10585 if let Some(failure) = failure {
10586 body.push('\n');
10587 body.push_str(&failure);
10588 }
10589 (code, body)
10590 }
10591 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10592 };
10593 out.push(CommandOutcome {
10594 command: command.clone(),
10595 code,
10596 output_tail: tail(&body, OUTPUT_TAIL),
10597 duration_ms: started.elapsed().as_millis() as u64,
10598 resource_blocked: false,
10599 });
10600 }
10601 state.task_finished(task);
10602 if let Err(e) = state.save() {
10603 tracing::warn!("could not persist the end of {task}: {e:#}");
10604 }
10605 (out, timed_out_pids)
10606}
10607
10608fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10620 let repo = repo.display();
10621 match style {
10622 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10623 MergeStyle::Squash => {
10624 let subject = message
10627 .lines()
10628 .next()
10629 .unwrap_or(branch)
10630 .replace(['\\', '"', '$', '`'], "");
10631 format!(
10632 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10633 )
10634 }
10635 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10636 }
10637}
10638
10639const PR_TITLE_MAX: usize = 240;
10650
10651fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10662 let line = queue::first_line(text)?;
10663 let chars: Vec<char> = line.chars().collect();
10664 let (kept, mark) = if chars.len() <= max {
10665 (line.to_owned(), "")
10666 } else if let Some(end) = sentence_end(&chars, max) {
10667 (chars[..end].iter().collect::<String>(), "")
10668 } else {
10669 let room = max.saturating_sub(3);
10670 let cut = if chars[room].is_whitespace() {
10673 room
10674 } else {
10675 chars[..room]
10676 .iter()
10677 .rposition(|c| c.is_whitespace())
10678 .unwrap_or(room)
10679 };
10680 let head: String = chars[..cut].iter().collect();
10681 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10682 (head.to_owned(), "...")
10683 };
10684 if kept.is_empty() {
10685 return None;
10686 }
10687 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10688 return None;
10689 }
10690 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10691 return None;
10692 }
10693 Some(format!("{kept}{mark}"))
10694}
10695
10696fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10700 const MIN: usize = 20;
10701 for i in MIN..max.min(chars.len()) {
10702 if !matches!(chars[i], '.' | '!' | '?') {
10703 continue;
10704 }
10705 let Some(&next) = chars.get(i + 1) else {
10706 continue;
10707 };
10708 if !next.is_whitespace() {
10709 continue;
10710 }
10711 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10712 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10713 continue;
10714 }
10715 let word: String = chars[..i]
10716 .iter()
10717 .rev()
10718 .take_while(|c| !c.is_whitespace())
10719 .collect::<Vec<_>>()
10720 .into_iter()
10721 .rev()
10722 .collect();
10723 let word = word.to_ascii_lowercase();
10724 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10725 continue;
10726 }
10727 let end = chars[..i]
10728 .iter()
10729 .rposition(|c| !c.is_whitespace())
10730 .map_or(i, |p| p + 1);
10731 return Some(end);
10732 }
10733 None
10734}
10735
10736struct PrMessage {
10741 title: String,
10742 body: String,
10743}
10744
10745impl PrMessage {
10746 fn commit_message(&self) -> String {
10750 format!("{}\n\n{}", self.title, self.body)
10751 }
10752}
10753
10754fn title_marker(line: &str) -> Option<&str> {
10756 let line = line.trim();
10757 let head = line.get(..6)?;
10758 head.eq_ignore_ascii_case("title:")
10759 .then(|| line[6..].trim())
10760}
10761
10762fn summary_title(summary: &str) -> Option<String> {
10767 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10768 let raw = title_marker(first)?;
10769 if raw.is_empty() {
10770 return None;
10771 }
10772 let title = queue::title_from(raw, PR_TITLE_MAX);
10773 let lower = title.to_ascii_lowercase();
10774 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10775 return None;
10776 }
10777 Some(title)
10778}
10779
10780const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10782
10783pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10785
10786pub(crate) fn is_review_run(state: &RunState) -> bool {
10790 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10791}
10792
10793fn review_title(state: &RunState) -> Option<String> {
10798 english_subject(
10799 state.reviewed_commits.as_ref()?.first()?,
10800 state.config.graph.github_english(),
10801 )
10802}
10803
10804fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10807 let raw = raw.trim();
10808 if raw.is_empty() {
10809 return None;
10810 }
10811 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10812 return None;
10813 }
10814 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10815 return None;
10816 }
10817 let title = queue::title_from(raw, PR_TITLE_MAX);
10818 let lower = title.to_ascii_lowercase();
10819 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10820 return None;
10821 }
10822 Some(title)
10823}
10824
10825#[derive(Debug, Clone, PartialEq, Eq)]
10828struct BranchFacts {
10829 commits: Vec<(String, String)>,
10831 stat: String,
10833}
10834
10835const STAT_FILE_LINES: usize = 25;
10837const COMMITS_MAX_CHARS: usize = 20_000;
10839
10840async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10843 let commits = git::commit_log(repo, base, branch).await.ok()?;
10844 if commits.is_empty() {
10845 return None;
10846 }
10847 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10848 let lines: Vec<&str> = stat.lines().collect();
10849 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10850 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10851 let more = format!(" ... {omitted} more file(s)");
10852 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10853 kept.push(&more);
10854 kept.push(lines[lines.len() - 1]);
10855 kept.join("\n")
10856 } else {
10857 lines.join("\n")
10858 };
10859 Some(BranchFacts { commits, stat })
10860}
10861
10862fn markdown_safe(text: &str) -> String {
10865 text.replace("</details>", "</details>")
10866 .replace("\x60\x60\x60", "~~~")
10867}
10868
10869fn neutral_title(state: &RunState, winner: char) -> String {
10870 format!(
10871 "chore: land candidate {} of run {}",
10872 winner.to_ascii_uppercase(),
10873 state.id
10874 )
10875}
10876
10877pub(crate) async fn repair_landing_gate(
10880 state: &mut RunState,
10881 winner: &Candidate,
10882 outcomes: &[CommandOutcome],
10883) -> Result<GateFix> {
10884 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10885 return Ok(GateFix::Stop);
10886 }
10887 let mut runner = Runner {
10888 roles: state.config.resolve_roles()?,
10889 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10890 state: state.clone(),
10891 pause: Pause::new(),
10892 interrupt: Pause::new(),
10893 };
10894 let result = runner.gate_fix_round(winner, outcomes).await;
10895 *state = runner.state;
10896 result
10897}
10898
10899pub(crate) async fn check_gate_repair_e2e(
10902 state: &mut RunState,
10903 winner: &Candidate,
10904 wait: &crate::repo_lock::WaitPolicy,
10905) -> Result<GateFix> {
10906 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10907 return Ok(GateFix::Retry);
10908 }
10909 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10910 if state
10911 .gate_repair_e2e
10912 .as_ref()
10913 .is_some_and(|r| r.head == head && r.passed)
10914 {
10915 return Ok(GateFix::Retry);
10916 }
10917 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10918 head: head.clone(),
10919 passed: false,
10920 outcomes: Vec::new(),
10921 });
10922 state.save()?;
10923 let commands = state.config.verify.e2e.clone();
10924 let outcomes = if commands.is_empty() {
10925 Vec::new()
10926 } else {
10927 let shell = state.config.shell();
10928 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10929 let cache_dir = state.config.cache_dir();
10930 let (outcomes, _) = with_cache_lease(
10931 state,
10932 cache_dir.as_deref(),
10933 "e2e",
10934 "e2e",
10935 &winner.worktree,
10936 &head,
10937 timeout,
10938 "gate repair e2e",
10939 wait,
10940 |state, budget| async move {
10941 run_e2e_with_retry(
10942 state,
10943 &shell,
10944 &commands,
10945 &winner.worktree,
10946 budget,
10947 "gate repair e2e",
10948 )
10949 .await
10950 },
10951 )
10952 .await?;
10953 outcomes
10954 };
10955 let deferred = verify_inconclusive(&outcomes);
10956 let passed = outcomes.iter().all(CommandOutcome::ok)
10957 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10958 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10959 head,
10960 passed,
10961 outcomes,
10962 });
10963 state.save()?;
10964 if deferred {
10965 Ok(GateFix::Defer)
10966 } else if !passed {
10967 state.event(
10968 "gate",
10969 "verify.e2e failed after gate repair; nothing published",
10970 );
10971 Ok(GateFix::Stop)
10972 } else {
10973 Ok(GateFix::Retry)
10974 }
10975}
10976
10977pub(crate) async fn run_landing_gate(
10979 state: &mut RunState,
10980 winner: &Candidate,
10981 wait: &crate::repo_lock::WaitPolicy,
10982) -> Result<Vec<CommandOutcome>> {
10983 let shell = state.config.shell();
10984 let gate_commands = state.config.verify.gate.clone();
10985 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10986 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10987 Vec::new()
10988 } else {
10989 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10990 let cache_dir = state.config.cache_dir();
10991 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10992 let (outcomes, _) = with_cache_lease(
10993 state,
10994 cache_dir.as_deref(),
10995 "gate",
10996 "gate",
10997 &winner.worktree,
10998 &head,
10999 timeout,
11000 "final gate",
11001 wait,
11002 |state, budget| {
11003 let shell = shell.clone();
11004 let gate_commands = gate_commands.clone();
11005 let winner = winner.clone();
11006 async move {
11007 Runner::run_pre_gate(state, &winner).await;
11008 let (outcomes, timed_out_pids) = run_commands(
11009 state,
11010 "gate",
11011 "gate",
11012 0,
11013 &shell,
11014 &gate_commands,
11015 &winner.worktree,
11016 budget,
11017 )
11018 .await;
11019 (outcomes, false, timed_out_pids)
11020 }
11021 },
11022 )
11023 .await?;
11024 outcomes
11025 };
11026 if outcomes.is_empty() {
11027 state.event("gate", "no gate commands configured; not tested");
11032 }
11033 for o in &outcomes {
11034 state.event(
11035 "gate",
11036 format!(
11037 "`{}` -> {}",
11038 o.command,
11039 if o.ok() {
11040 "pass".to_owned()
11041 } else {
11042 format!(
11043 "FAIL ({:?})\n{}",
11044 o.code,
11045 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11046 )
11047 }
11048 ),
11049 );
11050 }
11051 Ok(outcomes)
11052}
11053
11054pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11060 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11061 ""
11062 } else {
11063 pr_title
11064 }
11065}
11066
11067pub fn landing_subject_source(state: &RunState) -> String {
11071 if is_review_run(state) {
11072 let winner = state.candidates.first().map_or('A', |c| c.label);
11073 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11074 }
11075 state.instruction.clone()
11076}
11077
11078fn summary_without_title(summary: &str) -> String {
11081 let mut lines = summary.trim().lines().peekable();
11082 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11083 lines.next();
11084 }
11085 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11086}
11087
11088#[cfg(test)]
11100fn pr_message(state: &RunState, winner: char) -> PrMessage {
11101 pr_message_with(state, winner, None)
11102}
11103
11104#[cfg(test)]
11107fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11108 let raw = pr_message_raw(
11109 state,
11110 winner,
11111 facts,
11112 crate::pr_outcome::FinalHead::Unreported,
11113 ChangeNote::NotAsked,
11114 );
11115 let id = crate::scrub::Identity::current();
11116 PrMessage {
11117 title: crate::scrub::scrub(&raw.title, &id),
11118 body: crate::scrub::scrub(&raw.body, &id),
11119 }
11120}
11121
11122#[derive(Debug, Clone, Copy)]
11124enum ChangeNote<'a> {
11125 NotAsked,
11127 Summary(&'a crate::run::ChangeSummary),
11129 Unavailable(&'a str),
11131}
11132
11133fn pr_message_raw(
11134 state: &RunState,
11135 winner: char,
11136 facts: Option<&BranchFacts>,
11137 head: crate::pr_outcome::FinalHead<'_>,
11138 change: ChangeNote<'_>,
11139) -> PrMessage {
11140 if !matches!(change, ChangeNote::NotAsked) {
11141 let (title, mut body) = match change {
11142 ChangeNote::Summary(c) => (
11143 c.title.clone(),
11144 format!("## Summary\n\n{}\n\n", c.body.trim()),
11145 ),
11146 ChangeNote::Unavailable(why) => (
11147 neutral_title(state, winner),
11148 format!(
11149 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11150 ),
11151 ),
11152 ChangeNote::NotAsked => unreachable!(),
11153 };
11154 body.push_str(&public_pr_evidence(state, head));
11155 body.push_str(&format!(
11156 "\n---\nmagi:run/{} magi:candidate-{}\n",
11157 state.id,
11158 winner.to_ascii_lowercase()
11159 ));
11160 return PrMessage { title, body };
11161 }
11162 pr_preview(state, winner, facts, head)
11163}
11164
11165fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11168 let mut body = crate::pr_outcome::review_rounds(state);
11169 body.push_str(&crate::pr_outcome::final_verification(state, head));
11170 let open = state.last_round_findings();
11171 if !open.is_empty() {
11172 body.push_str("## Open review findings\n\n");
11173 for f in open {
11174 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11175 }
11176 body.push('\n');
11177 }
11178 if let Some(c) = state.change_summary.as_ref()
11179 && head == crate::pr_outcome::FinalHead::At(&c.head)
11180 {
11181 body.push_str(&format!(
11182 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11183 c.base, c.head
11184 ));
11185 }
11186 body
11187}
11188
11189fn pr_preview(
11191 state: &RunState,
11192 winner: char,
11193 facts: Option<&BranchFacts>,
11194 head: crate::pr_outcome::FinalHead<'_>,
11195) -> PrMessage {
11196 let summary = state
11197 .candidates
11198 .iter()
11199 .find(|c| c.label == winner)
11200 .map(|c| c.summary.as_str())
11201 .unwrap_or_default();
11202 let review = is_review_run(state);
11205 let title = if review {
11206 facts
11207 .and_then(|f| {
11208 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11209 })
11210 .or_else(|| review_title(state))
11211 .or_else(|| {
11212 state
11213 .candidates
11214 .iter()
11215 .find(|c| c.label == winner)
11216 .filter(|c| !c.branch.starts_with("magi/"))
11217 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11218 })
11219 .unwrap_or_else(|| neutral_title(state, winner))
11220 } else {
11221 summary_title(summary).unwrap_or_else(|| {
11222 english_title(
11223 &state.instruction,
11224 PR_TITLE_MAX,
11225 state.config.graph.github_english(),
11226 )
11227 .unwrap_or_else(|| neutral_title(state, winner))
11228 })
11229 };
11230
11231 let mut body = String::new();
11232 let what = summary_without_title(summary);
11233 if !what.is_empty() {
11234 body.push_str("## Summary\n\n");
11235 body.push_str(&what);
11236 body.push_str("\n\n");
11237 }
11238
11239 body.push_str(&crate::pr_outcome::review_rounds(state));
11241 body.push_str(&crate::pr_outcome::final_verification(state, head));
11242 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11243
11244 let open = state.open_findings();
11245 if !open.is_empty() {
11246 body.push_str("## Open review findings\n\n");
11247 for f in &open {
11248 body.push_str(&format!(
11249 "- `{}` [{:?}] {}\n",
11250 f.id, f.severity, "Context retained in local review records."
11251 ));
11252 }
11253 body.push('\n');
11254 }
11255
11256 if let Some(fix) = fix
11257 && !fix.rejected.is_empty()
11258 {
11259 body.push_str("## Declined by the fixer\n\n");
11260 for r in &fix.rejected {
11261 body.push_str(&format!(
11262 "- `{}`: {}\n",
11263 r.id, "Reason retained in local review records."
11264 ));
11265 }
11266 body.push('\n');
11267 }
11268
11269 if review {
11270 body.push_str("## Commits under review\n\n");
11272 if let Some(facts) = facts {
11273 let mut left = COMMITS_MAX_CHARS;
11274 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11275 let mut entry = format!("- {}\n", markdown_safe(subject));
11276 for l in markdown_safe(text).lines() {
11277 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11278 }
11279 if left == 0 {
11280 body.push_str(&format!(
11281 "- ... {} more commit(s)\n",
11282 facts.commits.len() - i
11283 ));
11284 break;
11285 }
11286 if entry.len() > left {
11287 let mut end = left;
11290 while !entry.is_char_boundary(end) {
11291 end -= 1;
11292 }
11293 entry.truncate(end);
11294 entry.push_str("\n ... (truncated)\n");
11295 left = 0;
11296 } else {
11297 left -= entry.len();
11298 }
11299 body.push_str(&entry);
11300 }
11301 if !facts.stat.trim().is_empty() {
11302 body.push_str(&format!(
11303 "\n## Diff stat\n\n```\n{}\n```\n",
11304 markdown_safe(facts.stat.trim())
11305 ));
11306 }
11307 } else {
11308 match &state.reviewed_commits {
11309 Some(subjects) => {
11310 for s in subjects {
11311 body.push_str(&format!("- {}\n", s.trim()));
11312 }
11313 }
11314 None => {
11315 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11318 body.push_str(rest.trim());
11319 body.push('\n');
11320 }
11321 }
11322 }
11323 } else {
11324 let task = state.instruction.trim();
11325 let task = if task.is_empty() {
11326 "(empty task)"
11327 } else {
11328 task
11329 };
11330 body.push_str(&format!(
11331 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11332 task.replace("</details>", "</details>")
11333 ));
11334 }
11335
11336 body.push_str(&format!(
11337 "\n---\nmagi:run/{} magi:candidate-{}\n",
11338 state.id,
11339 winner.to_ascii_lowercase()
11340 ));
11341
11342 PrMessage { title, body }
11343}
11344
11345fn seeded_instruction(state: &RunState) -> String {
11348 let mut inputs = Vec::new();
11349 if let Some(start) = &state.sources.start_from {
11350 inputs.push(format!(
11351 "Start from `{}` at `{}` (including ancestry).",
11352 start.token, start.sha
11353 ));
11354 }
11355 for pick in &state.sources.pick_commits {
11356 inputs.push(format!(
11357 "Pick `{}` at `{}` (single commit).",
11358 pick.token, pick.sha
11359 ));
11360 }
11361 if inputs.is_empty() {
11362 state.instruction.clone()
11363 } else {
11364 format!(
11365 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11366 state.instruction,
11367 inputs.join("\n")
11368 )
11369 }
11370}
11371
11372async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11376 let base = &state.base_branch;
11377 let mut against = base.clone();
11378 if mode == MergeMode::Pr {
11379 let remote = &state.config.merge.remote;
11383 let tracking = format!("{remote}/{base}");
11384 let fetched = git::fetch(repo, remote, base).await;
11385 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11386 against = tracking;
11387 } else {
11388 return false;
11389 }
11390 }
11391 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11392}
11393
11394fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11397 let mut detail = format!(
11398 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11399 nothing to open a pull request for"
11400 );
11401 detail.push_str(
11402 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11403 );
11404 detail
11405}
11406
11407#[derive(Debug, PartialEq, Eq)]
11410enum PrPlan {
11411 Create,
11412 Adopt { url: String, title: String },
11413 Stop(String),
11414}
11415
11416fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11419 match found {
11420 Ok(land::OpenPr::None) => PrPlan::Create,
11421 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11422 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11423 "several open pull requests exist for this branch, not picking one: {}",
11424 urls.join(" ")
11425 )),
11426 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11427 }
11428}
11429
11430pub(crate) async fn gh_pr_create(
11432 cwd: &Path,
11433 base: &str,
11434 head: &str,
11435 title: &str,
11436 body: &str,
11437) -> Result<String> {
11438 let out = tokio::process::Command::new("gh")
11439 .args([
11440 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11441 ])
11442 .current_dir(cwd)
11443 .quiet()
11444 .stdin(std::process::Stdio::null())
11445 .output()
11446 .await
11447 .context("spawn gh")?;
11448 if out.status.success() {
11449 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11450 } else {
11451 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11452 }
11453}
11454
11455async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11457 match std::fs::symlink_metadata(path) {
11458 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11459 git::worktree_remove_clean(repo, path).await?;
11462 return Ok(());
11463 }
11464 Err(e) => return Err(e.into()),
11465 Ok(_) => {}
11466 }
11467 if !git::is_clean(path).await? {
11468 bail!(
11469 "worktree has uncommitted work; refusing removal: {}",
11470 path.display()
11471 );
11472 }
11473 if !git::worktree_remove_clean(repo, path).await? {
11474 bail!("worktree removal refused: {}", path.display());
11475 }
11476 Ok(())
11477}
11478
11479pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11488 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11489 bail!("refusing fold: {reason}");
11490 }
11491 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11492 bail!("candidate preservation incomplete; refusing cleanup");
11493 }
11494 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11498 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11499 let _driver = crate::driver::maintenance(home, &state.id)?;
11500 let mut output_guards = Vec::new();
11501 for entry in std::fs::read_dir(state.worktree_root())
11502 .into_iter()
11503 .flatten()
11504 .flatten()
11505 {
11506 if entry.path().join(".git").is_file() {
11507 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11508 }
11509 }
11510 let repo = state.repo.clone();
11511 let wait = repository_wait(state, None);
11514 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11515 Ok(lease) => lease,
11516 Err(e) => {
11517 state.save_under(home)?;
11518 return Err(e);
11519 }
11520 };
11521 let _source_outputs = if state.status == RunStatus::Merged
11524 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11525 {
11526 verify_merged_fold_sources(state, drop_winner, home).await?
11527 } else {
11528 Vec::new()
11529 };
11530 let root = state.worktree_root();
11531 let winner = state.tally.as_ref().map(|t| t.winner);
11532 let mut removed = Vec::new();
11533
11534 for i in 0..state.candidates.len() {
11535 let c = state.candidates[i].clone();
11536 let is_winner = Some(c.label) == winner;
11537 if is_winner && !drop_winner {
11538 continue;
11539 }
11540 let existed = c.worktree.exists();
11541 remove_preserved_worktree(&repo, &c.worktree).await?;
11544 if existed {
11545 removed.push(c.worktree.to_string_lossy().into_owned());
11546 }
11547 let handed_over = state.released_branches.contains(&c.branch);
11550 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11551 if !git::branch_delete(&repo, &c.branch).await? {
11552 bail!("candidate branch removal refused");
11553 }
11554 removed.push(c.branch.clone());
11555 }
11556 state.candidates[i].folded = true;
11557 }
11558
11559 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11560 let path = name.path();
11561 let keep = !drop_winner
11562 && winner.is_some_and(|w| {
11563 path.file_name()
11564 .is_some_and(|n| n == format!("cand-{w}").as_str())
11565 });
11566 if keep {
11567 continue;
11568 }
11569 remove_preserved_worktree(&repo, &path).await?;
11570 removed.push(path.to_string_lossy().into_owned());
11571 }
11572
11573 remove_if_empty(&root);
11582
11583 if state.enabled_worktree_config && drop_winner {
11584 git::release_worktree_config(&repo).await.ok();
11588 state.enabled_worktree_config = false;
11589 }
11590 state.save_under(home)?;
11591 Ok(removed)
11592}
11593
11594async fn verify_merged_fold_sources(
11595 state: &RunState,
11596 drop_winner: bool,
11597 home: &Path,
11598) -> Result<Vec<std::fs::File>> {
11599 let winner = state.tally.as_ref().map(|t| t.winner);
11600 let mut heads = BTreeSet::new();
11601 for candidate in &state.candidates {
11602 if Some(candidate.label) == winner && !drop_winner {
11603 continue;
11604 }
11605 anyhow::ensure!(
11606 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11607 "candidate recovery source is outside the inventoried run root: {}",
11608 candidate.worktree.display()
11609 );
11610 match std::fs::symlink_metadata(&candidate.worktree) {
11611 Ok(_) => {}
11612 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11613 Err(error) => return Err(error.into()),
11614 }
11615 if !state.released_branches.contains(&candidate.branch)
11616 && git::branch_exists(&state.repo, &candidate.branch).await?
11617 {
11618 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11619 }
11620 }
11621 let mut cargo_guards = Vec::new();
11622 let entries = match std::fs::read_dir(state.worktree_root()) {
11623 Ok(entries) => Some(entries),
11624 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11625 Err(error) => return Err(error.into()),
11626 };
11627 for entry in entries.into_iter().flatten() {
11628 let path = entry?.path();
11629 let metadata = std::fs::symlink_metadata(&path)?;
11630 anyhow::ensure!(
11631 metadata.is_dir() && !metadata.file_type().is_symlink(),
11632 "non-directory or symlink recovery source retained: {}",
11633 path.display()
11634 );
11635 if !drop_winner
11636 && winner.is_some_and(|w| {
11637 path.file_name()
11638 .is_some_and(|name| name == format!("cand-{w}").as_str())
11639 })
11640 {
11641 continue;
11642 }
11643 anyhow::ensure!(
11644 path.join(".git").is_file(),
11645 "unregistered recovery source retained: {}",
11646 path.display()
11647 );
11648 anyhow::ensure!(
11649 git::git(
11650 &path,
11651 &[
11652 "status",
11653 "--porcelain",
11654 "--untracked-files=all",
11655 "--ignored",
11656 "--",
11657 ".",
11658 ":(exclude)target"
11659 ]
11660 )
11661 .await?
11662 .is_empty(),
11663 "source changes or ignored recovery files retained: {}",
11664 path.display()
11665 );
11666 match std::fs::symlink_metadata(path.join("target")) {
11667 Ok(_) => {
11668 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11669 }
11670 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11671 Err(error) => return Err(error.into()),
11672 }
11673 for checkpoint in [
11674 "rebase-merge",
11675 "rebase-apply",
11676 "sequencer",
11677 "CHERRY_PICK_HEAD",
11678 "MERGE_HEAD",
11679 "REBASE_HEAD",
11680 "REVERT_HEAD",
11681 "BISECT_LOG",
11682 "AUTO_MERGE",
11683 "MERGE_MSG",
11684 ] {
11685 let location = git::git(
11686 &path,
11687 &[
11688 "rev-parse",
11689 "--path-format=absolute",
11690 "--git-path",
11691 checkpoint,
11692 ],
11693 )
11694 .await?;
11695 anyhow::ensure!(
11696 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11697 if e.kind() == std::io::ErrorKind::NotFound),
11698 "Git recovery checkpoint retained: {checkpoint}"
11699 );
11700 }
11701 heads.insert(git::rev_parse(&path, "HEAD").await?);
11702 }
11703 crate::fold_evidence::preserve(state, home, &heads).await?;
11704
11705 Ok(cargo_guards)
11706}
11707
11708fn remove_if_empty(dir: &Path) {
11719 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11720 std::fs::remove_dir(dir).ok();
11721 }
11722}
11723
11724pub fn worst_open(state: &RunState) -> Option<Severity> {
11726 state
11727 .reviews
11728 .last()?
11729 .reviews
11730 .iter()
11731 .flat_map(|r| r.findings.iter())
11732 .map(|f| f.severity)
11733 .max()
11734}
11735
11736async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11740 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11741 git::acquire_worktree_config(repo).await?;
11742 state.enabled_worktree_config = true;
11743 }
11744 Ok(())
11745}
11746
11747#[cfg(test)]
11748mod tests {
11749 #[tokio::test]
11750 async fn launcher_base_fetch_requires_remote_success() {
11751 let tmp = tempfile::tempdir().unwrap();
11752 let repo = tmp.path().join("repo");
11753 let remote = tmp.path().join("remote");
11754 std::fs::create_dir(&repo).unwrap();
11755 std::fs::create_dir(&remote).unwrap();
11756 git::git(&remote, &["init", "--bare", "-b", "main"])
11757 .await
11758 .unwrap();
11759 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11760 git::git(
11761 &repo,
11762 &[
11763 "-c",
11764 "user.name=Fixture",
11765 "-c",
11766 "user.email=fixture@example.invalid",
11767 "commit",
11768 "--allow-empty",
11769 "-m",
11770 "fixture",
11771 ],
11772 )
11773 .await
11774 .unwrap();
11775 git::git(
11776 &repo,
11777 &["remote", "add", "origin", remote.to_str().unwrap()],
11778 )
11779 .await
11780 .unwrap();
11781 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11782 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11783 assert_eq!(
11784 resolve_base(&repo, "main", "origin", None)
11785 .await
11786 .unwrap()
11787 .commit,
11788 head
11789 );
11790 #[cfg(unix)]
11791 {
11792 let policy: crate::config::Coordination = toml::from_str(
11793 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11794 ).unwrap();
11795 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11796 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11797 let held = crate::repo_lock::SeatLease::operation_with(
11798 &repo,
11799 Some(&policy),
11800 &crate::repo_lock::WaitPolicy::default(),
11801 )
11802 .await
11803 .unwrap();
11804 assert!(held.is_some());
11805 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11806 tokio::pin!(fetch);
11807 assert!(
11808 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11809 .await
11810 .is_err()
11811 );
11812 assert!(
11813 !repo.join(".git/FETCH_HEAD").exists(),
11814 "no fetch before admission"
11815 );
11816 drop(held);
11817 let fetched = fetch.await.unwrap();
11818 assert_eq!(fetched.commit, head);
11819 assert_eq!(fetched.remote, "origin");
11820 assert_eq!(fetched.branch, "main");
11821 }
11822 git::git(
11824 &repo,
11825 &[
11826 "remote",
11827 "set-url",
11828 "origin",
11829 tmp.path().join("missing").to_str().unwrap(),
11830 ],
11831 )
11832 .await
11833 .unwrap();
11834 let error = resolve_base(&repo, "main", "origin", None)
11835 .await
11836 .unwrap_err()
11837 .to_string();
11838 assert!(error.contains("cannot read origin/main"));
11839 assert!(error.contains("refusing to branch off"));
11840 }
11841
11842 #[test]
11843 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11844 let leaked = vec!["chore(deps): update a crate".to_owned()];
11845 let own = "fix(daemon): apply a chosen action";
11846 assert!(should_retitle("", own, &leaked));
11847 assert!(should_retitle(
11848 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11849 own,
11850 &leaked
11851 ));
11852 assert!(should_retitle(
11853 "chore: land candidate A of run 1",
11854 own,
11855 &leaked
11856 ));
11857 assert!(should_retitle(
11858 "magi: candidate A (uncommitted work)",
11859 own,
11860 &leaked
11861 ));
11862 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11863 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11864 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11865 }
11866
11867 #[test]
11868 fn pr_merge_plan_creates_adopts_or_stops() {
11869 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11870 assert_eq!(
11871 pr_merge_plan(Ok(land::OpenPr::One {
11872 url: "u".into(),
11873 title: "t".into()
11874 })),
11875 PrPlan::Adopt {
11876 url: "u".into(),
11877 title: "t".into()
11878 }
11879 );
11880 let PrPlan::Stop(many) =
11881 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11882 else {
11883 panic!("many must stop");
11884 };
11885 assert!(many.contains('a') && many.contains('b'));
11886 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11887 panic!("a failed lookup must stop");
11888 };
11889 assert!(err.contains("bad token"));
11890 }
11891
11892 use super::*;
11893 use crate::run::GateStatus;
11894 use std::collections::BTreeMap;
11895 use std::time::Duration;
11896
11897 fn conductor() -> AgentSpec {
11898 AgentSpec {
11899 id: "conductor".to_owned(),
11900 kind: crate::config::AgentKind::Command,
11901 model: None,
11902 command: vec!["true".to_owned()],
11903 extra_args: Vec::new(),
11904 env: BTreeMap::new(),
11905 prompt_delivery: None,
11906 max_turn_secs: None,
11907 }
11908 }
11909
11910 fn spec(id: &str) -> AgentSpec {
11911 AgentSpec {
11912 id: id.to_owned(),
11913 kind: crate::config::AgentKind::Command,
11914 model: None,
11915 command: vec!["true".to_owned()],
11916 extra_args: Vec::new(),
11917 env: BTreeMap::new(),
11918 prompt_delivery: None,
11919 max_turn_secs: None,
11920 }
11921 }
11922
11923 fn ids(xs: &[&str]) -> BTreeSet<String> {
11924 xs.iter().map(|s| (*s).to_owned()).collect()
11925 }
11926
11927 #[test]
11928 fn pick_successor_skips_an_agent_another_seat_holds() {
11929 let roster = [spec("a"), spec("b"), spec("c")];
11931 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11932 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11933 }
11934
11935 #[test]
11936 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11937 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11939 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11940 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11941 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11942 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11943 }
11944
11945 #[test]
11946 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11947 let roster = [spec("a"), spec("b")];
11948 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11949 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11950 let carried = ids(&["b"]);
11951 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11952 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11953 }
11954
11955 #[test]
11956 fn pick_successor_returns_none_without_any_untried_successor() {
11957 let roster = [spec("a"), spec("b")];
11958 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11959 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11960 let carried = ids(&["a"]);
11961 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11962 }
11963
11964 #[test]
11965 fn pick_successor_rescue_avoids_another_seats_occupant() {
11966 let roster = [spec("a"), spec("b"), spec("c")];
11968 let carried = ids(&["b", "c"]);
11969 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11970 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11971 }
11972
11973 #[test]
11974 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11975 let roster = [spec("a"), spec("b"), spec("c")];
11976 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11977 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11978 }
11979
11980 #[test]
11981 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11982 let roster = [spec("a"), spec("b"), spec("c")];
11983 let failed = ids(&["a", "b", "c"]);
11984 let mut tried = ids(&["b"]);
11986 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11987 assert_eq!(first.id, "a");
11988 tried.insert(first.id.clone());
11989 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11990 assert_eq!(second.id, "c");
11991 tried.insert(second.id.clone());
11992 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11993 }
11994
11995 #[test]
11996 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11997 let roster = [spec("a"), spec("b")];
11998 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11999 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12000 }
12001
12002 #[test]
12003 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12004 let roster = [spec("a"), spec("b"), spec("c")];
12005 let h = SeatHistory {
12006 failed: ids(&["a"]),
12007 last_ok: Some("b".to_owned()),
12008 last_fail: None,
12009 };
12010 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12011 let gone = SeatHistory {
12013 last_ok: Some("zzz".to_owned()),
12014 ..h.clone()
12015 };
12016 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12017 let failed = SeatHistory {
12018 failed: ids(&["a", "b"]),
12019 last_ok: Some("b".to_owned()),
12020 last_fail: None,
12021 };
12022 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12023 }
12024
12025 #[test]
12026 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12027 let first = SeatState::new("review-1", "alpha", 7);
12028 let next = handover_seat("review-1", "gamma", 7);
12029 assert_ne!(first.claude_session, next.claude_session);
12030 }
12031
12032 #[test]
12033 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12034 let mut state = state_with_summary("x", "y");
12035 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12036 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12037 assert_ne!(a.claude_session, b.claude_session);
12038 }
12039
12040 #[test]
12041 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12042 let roster = [spec("a"), spec("b")];
12043 let h = SeatHistory {
12044 failed: ids(&["a", "b"]),
12045 ..SeatHistory::default()
12046 };
12047 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12048 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12049 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12050 }
12051
12052 #[test]
12059 fn failure_signature_ignores_numbers_and_paths() {
12060 assert_eq!(
12061 failure_signature("exited with Some(2) and no usable output"),
12062 failure_signature("exited with Some(137) and no usable output")
12063 );
12064 assert_eq!(
12065 failure_signature("process exited with no usable output (exit code 2)"),
12066 failure_signature("process exited with no usable output (exit code 137)")
12067 );
12068 for (old, new) in [
12069 (
12070 "exited with Some(2) and no usable output",
12071 "process exited with no usable output (exit code 2)",
12072 ),
12073 (
12074 "exited with None and no usable output",
12075 "process exited with no usable output (exit code unavailable)",
12076 ),
12077 ] {
12078 let previous = FailClass::Other(failure_signature(old));
12079 let current = FailClass::Other(failure_signature(new));
12080 assert_eq!(previous, current, "resumption retains the failure class");
12081 assert!(!should_hand_over(Some(&previous), ¤t));
12082 }
12083 assert_eq!(
12084 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12085 failure_signature("cannot open /var/x.txt: denied")
12086 );
12087 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12088 }
12089
12090 #[test]
12091 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12092 let other = FailClass::Other("x".into());
12093 assert!(should_hand_over(None, &FailClass::Quota));
12094 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12095 assert!(should_hand_over(
12096 Some(&FailClass::Timeout),
12097 &FailClass::Timeout
12098 ));
12099 assert!(should_hand_over(None, &other));
12100 assert!(!should_hand_over(Some(&other), &other));
12101 assert!(should_hand_over(
12102 Some(&other),
12103 &FailClass::Other("y".into())
12104 ));
12105 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12107 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12108 }
12109
12110 #[test]
12111 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12112 let a = SeatState::new("judge-1", "alpha", 7);
12113 let b = handover_seat("judge-1", "beta", 7);
12114 assert_ne!(a.claude_session, b.claude_session);
12115 assert_eq!(b.turns, 0);
12116 }
12117
12118 #[test]
12119 fn a_timeout_is_classified_apart_from_other_failures() {
12120 assert_eq!(
12121 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12122 Some(FailClass::Timeout)
12123 );
12124 assert!(matches!(
12125 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12126 Some(FailClass::Other(_))
12127 ));
12128 }
12129
12130 #[test]
12131 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12132 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12133 let tried = BTreeSet::from(["beta".to_owned()]);
12134 let next = next_untried_in_roster(&roster, 1, &tried);
12137 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12138 }
12139
12140 #[test]
12141 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12142 let roster = vec![spec("alpha"), spec("beta")];
12143 let tried = BTreeSet::from(["beta".to_owned()]);
12144 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12148 }
12149
12150 #[test]
12151 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12152 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12153 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12154 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12158 }
12159
12160 #[test]
12161 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12162 let roster = vec![spec("a"), spec("a"), spec("b")];
12163 let tried = BTreeSet::from(["a".to_owned()]);
12164 let next = next_untried_in_roster(&roster, 0, &tried);
12165 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12166 }
12167
12168 #[test]
12169 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12170 let roster = vec![spec("a"), spec("b")];
12171 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12172 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12173 }
12174
12175 #[test]
12176 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12177 let dir = tempfile::tempdir().unwrap();
12178 let bay = dir.path().join("ffff");
12179
12180 remove_if_empty(&bay);
12182 assert!(!bay.exists());
12183
12184 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12187 remove_if_empty(&bay);
12188 assert!(bay.exists(), "non-empty directory must survive");
12189
12190 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12192 remove_if_empty(&bay);
12193 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12194 }
12195
12196 #[test]
12205 fn a_full_panel_that_found_nothing_is_clean() {
12206 assert!(round_is_clean(
12207 0,
12208 true,
12209 2,
12210 2,
12211 0,
12212 IncompleteReviewPolicy::Block
12213 ));
12214 }
12215
12216 #[test]
12217 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12218 assert!(!round_is_clean(
12219 0,
12220 true,
12221 1,
12222 2,
12223 0,
12224 IncompleteReviewPolicy::Block
12225 ));
12226 }
12227
12228 #[test]
12229 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12230 assert!(!round_is_clean(
12231 1,
12232 true,
12233 1,
12234 2,
12235 0,
12236 IncompleteReviewPolicy::Warn
12237 ));
12238 }
12239
12240 #[test]
12241 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12242 assert!(round_is_clean(
12243 0,
12244 true,
12245 1,
12246 2,
12247 0,
12248 IncompleteReviewPolicy::Warn
12249 ));
12250 }
12251
12252 #[test]
12253 fn a_full_panel_with_an_open_finding_is_not_clean() {
12254 assert!(!round_is_clean(
12255 1,
12256 true,
12257 2,
12258 2,
12259 0,
12260 IncompleteReviewPolicy::Block
12261 ));
12262 }
12263
12264 #[test]
12265 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12266 assert!(!round_is_clean(
12267 0,
12268 false,
12269 2,
12270 2,
12271 0,
12272 IncompleteReviewPolicy::Block
12273 ));
12274 }
12275
12276 #[test]
12283 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12284 assert!(round_is_clean(
12287 0,
12288 true,
12289 1,
12290 2,
12291 1,
12292 IncompleteReviewPolicy::Block
12293 ));
12294 }
12295
12296 #[test]
12297 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12298 assert!(!round_is_clean(
12301 0,
12302 true,
12303 1,
12304 2,
12305 0,
12306 IncompleteReviewPolicy::Block
12307 ));
12308 }
12309
12310 #[test]
12311 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12312 assert!(!round_is_clean(
12313 1,
12314 true,
12315 1,
12316 2,
12317 1,
12318 IncompleteReviewPolicy::Block
12319 ));
12320 assert!(!round_is_clean(
12321 0,
12322 false,
12323 1,
12324 2,
12325 1,
12326 IncompleteReviewPolicy::Block
12327 ));
12328 }
12329
12330 #[test]
12331 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12332 assert!(!round_is_clean(
12336 0,
12337 true,
12338 0,
12339 2,
12340 2,
12341 IncompleteReviewPolicy::Block
12342 ));
12343 }
12344
12345 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12346 CommandOutcome {
12347 command: "test".to_owned(),
12348 code,
12349 output_tail: String::new(),
12350 duration_ms: 0,
12351 resource_blocked,
12352 }
12353 }
12354
12355 #[test]
12356 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12357 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12358 assert!(
12359 !verify_inconclusive(&[outcome(Some(1), false)]),
12360 "an ordinary failure is still evidence about the patch"
12361 );
12362 assert!(verify_inconclusive(&[outcome(None, true)]));
12363 assert!(
12364 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12365 "one inconclusive outcome taints the whole batch"
12366 );
12367 assert!(!verify_inconclusive(&[]));
12368 }
12369
12370 #[tokio::test]
12371 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12372 let calls = std::sync::atomic::AtomicUsize::new(0);
12376 let started = Instant::now();
12377 wait_for_pids_with(
12378 &[123],
12379 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12380 Duration::from_millis(5),
12381 Duration::from_secs(5),
12382 )
12383 .await;
12384 assert!(
12385 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12386 "must keep checking rather than deciding on the first answer"
12387 );
12388 assert!(
12389 started.elapsed() < Duration::from_secs(1),
12390 "must return the moment it is confirmed dead, not wait out the ceiling"
12391 );
12392 }
12393
12394 #[tokio::test]
12395 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12396 let started = Instant::now();
12397 wait_for_pids_with(
12398 &[123],
12399 |_| true, Duration::from_millis(5),
12401 Duration::from_millis(30),
12402 )
12403 .await;
12404 let elapsed = started.elapsed();
12405 assert!(
12406 elapsed >= Duration::from_millis(30),
12407 "must not give up before its own ceiling: {elapsed:?}"
12408 );
12409 assert!(
12410 elapsed < Duration::from_secs(1),
12411 "must not wait past its own ceiling either: {elapsed:?}"
12412 );
12413 }
12414
12415 #[tokio::test]
12416 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12417 let started = Instant::now();
12418 wait_for_pids_with(
12419 &[],
12420 |_| true,
12421 Duration::from_secs(5),
12422 Duration::from_secs(5),
12423 )
12424 .await;
12425 assert!(
12426 started.elapsed() < Duration::from_millis(200),
12427 "an empty pid list has nothing to confirm"
12428 );
12429 }
12430
12431 fn review_round(
12437 clean: bool,
12438 blocking: usize,
12439 answered: usize,
12440 expected: usize,
12441 progressed: bool,
12442 e2e_ok: bool,
12443 ) -> ReviewRound {
12444 ReviewRound {
12445 round: 1,
12446 head: "h".to_owned(),
12447 verified_head: None,
12448 verified_at: None,
12449 reviews: Vec::new(),
12450 e2e: vec![CommandOutcome {
12451 command: "test".to_owned(),
12452 code: Some(if e2e_ok { 0 } else { 1 }),
12453 output_tail: String::new(),
12454 duration_ms: 0,
12455 resource_blocked: false,
12456 }],
12457 verify_retried: false,
12458 e2e_deferred: false,
12459 e2e_defer_reason: None,
12460 fix: None,
12461 blocking,
12462 answered,
12463 expected,
12464 clean,
12465 progressed,
12466 vote_split: false,
12467 reconsideration: Vec::new(),
12468 verdict: None,
12469 }
12470 }
12471
12472 #[test]
12473 fn review_conclusion_is_none_when_nothing_has_run() {
12474 assert_eq!(review_conclusion(&[], 3), None);
12475 }
12476
12477 #[test]
12478 fn review_conclusion_is_none_while_rounds_remain() {
12479 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12480 assert_eq!(review_conclusion(&rounds, 3), None);
12481 }
12482
12483 #[test]
12484 fn review_conclusion_is_gating_once_a_round_is_clean() {
12485 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12486 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12487 }
12488
12489 #[test]
12490 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12491 let rounds = vec![
12492 review_round(false, 1, 2, 2, true, true),
12493 review_round(false, 1, 2, 2, true, true),
12494 ];
12495 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12496 }
12497
12498 #[test]
12499 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12500 let rounds = vec![
12501 review_round(false, 1, 2, 2, true, true),
12502 review_round(false, 1, 2, 2, true, false),
12503 ];
12504 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12505 }
12506
12507 #[test]
12508 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12509 let mut blocked = review_round(false, 1, 2, 2, true, false);
12516 blocked.e2e[0].resource_blocked = true;
12517 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12518 assert_eq!(review_conclusion(&rounds, 2), None);
12519 }
12520
12521 #[test]
12522 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12523 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12525 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12526 }
12527
12528 #[test]
12529 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12530 let rounds = vec![
12531 review_round(false, 1, 2, 2, false, true),
12532 review_round(false, 1, 2, 2, false, true),
12533 ];
12534 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12535 }
12536
12537 fn secs(n: u64) -> Duration {
12538 Duration::from_secs(n)
12539 }
12540
12541 #[tokio::test]
12544 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12545 ask_test_home();
12546 let tmp = tempfile::tempdir().unwrap();
12547 let repo = tmp.path().join("repo");
12548 std::fs::create_dir_all(&repo).unwrap();
12549 init_repo(&repo);
12550 let g = |args: &[&str]| {
12551 let out = std::process::Command::new("git")
12552 .args(args)
12553 .current_dir(&repo)
12554 .quiet()
12555 .output()
12556 .unwrap();
12557 assert!(
12558 out.status.success(),
12559 "{}",
12560 String::from_utf8_lossy(&out.stderr)
12561 );
12562 };
12563 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12564 g(&["checkout", "-b", "historical"]);
12565 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12566 g(&["add", "."]);
12567 g(&["commit", "-m", "old alternative"]);
12568 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12569 g(&["checkout", "-b", "magi/task-selected", &base]);
12570 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12571 g(&["add", "."]);
12572 g(&["commit", "-m", "measurement infrastructure"]);
12573 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12574 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12575 g(&["add", "."]);
12576 g(&["commit", "-m", "dependent implementation"]);
12577 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12578 g(&["checkout", "main"]);
12579 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12580 g(&["add", "."]);
12581 g(&["commit", "-m", "unrelated base change"]);
12582 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12583 let origin = tmp.path().join("origin.git");
12584 g(&[
12585 "clone",
12586 "--bare",
12587 repo.to_str().unwrap(),
12588 origin.to_str().unwrap(),
12589 ]);
12590 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12591 g(&["fetch", "origin"]);
12592 let selection = crate::pr_recovery::Selection {
12593 url: "https://example.test/project/pull/1".into(),
12594 branch: "magi/task-selected".into(),
12595 head: head.clone(),
12596 source_head: None,
12597 source_run: None,
12598 source_proof: None,
12599 disposition: Some(crate::pr_recovery::Disposition::Review),
12600 instruction: "preserve measured behavior".into(),
12601 approved_at: jiff::Timestamp::now(),
12602 };
12603 let mut template = runner_at(RunStatus::Reviewing);
12604 template.state.repo = repo.clone();
12605 template.state.base_commit = current.clone();
12606 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12607 template.state.origin = Some(crate::run::Origin::queue("selected"));
12608 template.state.pr_head = Some(selection.branch.clone());
12609 template.state.pr_recovery = Some(selection.clone());
12610 let mut runner =
12611 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12612 .await
12613 .unwrap();
12614 let wt = runner.state.winner().unwrap().worktree.clone();
12615 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12616 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12617 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12618 assert!(runner.state.seeds.is_empty());
12619 assert_eq!(runner.state.instruction, "preserve measured behavior");
12620 runner.sync_to_base().await.unwrap();
12621 assert_eq!(
12622 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12623 "prerequisite"
12624 );
12625 assert_eq!(
12626 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12627 "measured implementation"
12628 );
12629 assert_eq!(
12630 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12631 "current main change"
12632 );
12633 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12634 assert_eq!(
12635 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12636 head
12637 );
12638
12639 let mut partial = runner_at(RunStatus::Reviewing);
12641 partial.state.repo = repo.clone();
12642 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12643 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12644 run: Some("exit 1".into()),
12645 ..Default::default()
12646 }];
12647 let selection = runner.state.pr_recovery.clone().unwrap();
12648 partial.state.pr_recovery = Some(selection.clone());
12649 partial.state.pr_head = Some(selection.branch.clone());
12650 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12651 let id = partial.state.id.clone();
12652 let error = match Runner::open_selected(
12653 &repo,
12654 partial.state,
12655 partial.roles,
12656 1,
12657 runner.state.base_commit.clone(),
12658 selection,
12659 )
12660 .await
12661 {
12662 Ok(_) => panic!("setup must fail"),
12663 Err(e) => e,
12664 };
12665 assert_eq!(
12666 error
12667 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12668 .unwrap()
12669 .run,
12670 id
12671 );
12672 let failed = RunState::load(&id).unwrap();
12673 assert_eq!(failed.status, RunStatus::Blocked);
12674 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12675 assert!(Runner::resume(&id).is_err());
12676 runner.state.status = RunStatus::Ready;
12677 runner.state.driver_exited = true;
12678 runner.state.save().unwrap();
12679 let mut retry = runner_at(RunStatus::Reviewing);
12680 retry.state.repo = repo.clone();
12681 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12682 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12683 reselected.approved_at = jiff::Timestamp::now();
12684 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12685 retry.state.pr_head = Some(reselected.branch.clone());
12686 retry.state.pr_recovery = Some(reselected.clone());
12687 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12688 .await
12689 .unwrap();
12690 let restarted = Runner::open_selected(
12691 &repo,
12692 retry.state,
12693 retry.roles,
12694 1,
12695 runner.state.base_commit.clone(),
12696 reselected,
12697 )
12698 .await
12699 .unwrap();
12700 assert_eq!(
12701 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12702 .await
12703 .unwrap(),
12704 head
12705 );
12706 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12707 assert_eq!(
12708 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12709 head
12710 );
12711 }
12712
12713 #[tokio::test]
12714 #[cfg(unix)]
12715 async fn fallback_and_interrupted_restoration_require_coordination() {
12716 use std::os::fd::AsRawFd as _;
12717 ask_test_home();
12718 let tmp = tempfile::tempdir().unwrap();
12719 let mut runner = registered_runner(tmp.path()).await;
12720 runner.state.config.graph.timeout_resource_wait = 1;
12721 let winner = runner.state.candidates[0].clone();
12722 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12723 let leftover = winner.worktree.join("leftover");
12724 std::fs::write(&leftover, "untrusted edits").unwrap();
12725 let next = tmp.path().join("next-fixer");
12726 let mut failed = spec("first");
12727 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12728 let mut successor = spec("second");
12729 successor.command = vec![
12730 "sh".into(),
12731 "-c".into(),
12732 format!("touch '{}'; echo done", next.display()),
12733 ];
12734 let attempts = vec![
12735 (failed.clone(), "first".into()),
12736 (successor.clone(), "second".into()),
12737 ];
12738 let lock_path = runner.state.repo.join(".git/updater.lock");
12739 let external = std::fs::File::create(&lock_path).unwrap();
12740 assert_eq!(
12741 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12742 0
12743 );
12744 let artifacts = tmp.path().join("artifacts");
12745 let build = |spec, seat| SeatJob {
12746 spec,
12747 seat,
12748 cwd: winner.worktree.clone(),
12749 prompt: "fixture".into(),
12750 timeout: Duration::from_secs(10),
12751 allow_write: true,
12752 sessions: false,
12753 artifacts: artifacts.clone(),
12754 stem: "fix".into(),
12755 handover: None,
12756 };
12757 let (_, _, out) = runner
12758 .ask_fixer_among(
12759 attempts.clone(),
12760 "graft",
12761 None,
12762 Some((&winner.worktree, &head)),
12763 build,
12764 )
12765 .await;
12766 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12767 assert!(leftover.exists(), "contention must prevent clean");
12768 assert!(!next.exists());
12769 assert!(runner.state.handovers.is_empty());
12770 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12772 from_run: "earlier".into(),
12773 generation: 1,
12774 mode: crate::recovery::Mode::VerifyRepair,
12775 reasons: Vec::new(),
12776 reviewed_head: None,
12777 rounds: Vec::new(),
12778 open_findings: Vec::new(),
12779 evidence: None,
12780 criteria_inherited: true,
12781 repair: crate::recovery::Repair::Started {
12782 before: head.clone(),
12783 },
12784 });
12785 assert!(
12786 runner
12787 .recovery_repair(&winner)
12788 .await
12789 .unwrap_err()
12790 .is::<crate::repo_lock::ResourceBlocked>()
12791 );
12792 assert!(matches!(
12793 runner.state.recovery.as_ref().unwrap().repair,
12794 crate::recovery::Repair::Started { .. }
12795 ));
12796 assert!(leftover.exists());
12797 drop(external);
12798 runner.recovery_repair(&winner).await.unwrap();
12799 assert!(!leftover.exists());
12800 assert!(matches!(
12801 runner.state.recovery.as_ref().unwrap().repair,
12802 crate::recovery::Repair::Done { .. }
12803 ));
12804 std::fs::remove_file(&lock_path).unwrap();
12806 std::fs::create_dir(&lock_path).unwrap();
12807 std::fs::write(&leftover, "untrusted edits").unwrap();
12808 let (_, _, out) = runner
12809 .ask_fixer_among(
12810 attempts,
12811 "graft",
12812 None,
12813 Some((&winner.worktree, &head)),
12814 build,
12815 )
12816 .await;
12817 assert!(matches!(out, AgentOutcome::Failed(_)));
12818 assert!(leftover.exists());
12819 assert!(!next.exists());
12820 assert_eq!(
12821 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12822 head
12823 );
12824 }
12825
12826 #[cfg(unix)]
12827 async fn registered_runner(root: &Path) -> Runner {
12828 ask_test_home();
12829 let repo = root.join("repo");
12830 std::fs::create_dir_all(&repo).unwrap();
12831 init_repo(&repo);
12832 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12833 let mut runner = runner_at(RunStatus::Prep);
12834 runner.state.repo = repo.clone();
12835 runner.state.config.coordination = crate::config::test_coordination();
12836 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12837 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12838 runner.state.config.graph.advise = false;
12839 runner.roles.implementers = vec![spec("implement")];
12840 runner.roles.judges = vec![spec("judge")];
12841 runner.prep().await.unwrap();
12842 runner
12843 }
12844
12845 #[tokio::test]
12846 #[cfg(unix)]
12847 async fn parked_implementer_persists_completed_work() {
12848 for registered in [false, true] {
12849 let tmp = tempfile::tempdir().unwrap();
12850 let mut runner = registered_runner(tmp.path()).await;
12851 if !registered {
12852 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12853 }
12854 let marker = tmp.path().join("started");
12855 let release = tmp.path().join("release");
12856 let mut implementer = spec("implement");
12857 implementer.command = vec![
12858 "sh".into(),
12859 "-c".into(),
12860 format!(
12861 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12862 marker.display(),
12863 release.display()
12864 ),
12865 ];
12866 runner.roles.implementers = vec![implementer.clone()];
12867 runner.state.config.agents = vec![implementer];
12868 let pause = runner.pause.clone();
12869 let (result, ()) = tokio::join!(runner.implement(), async {
12870 tokio::time::timeout(Duration::from_secs(10), async {
12871 while !marker.exists() {
12872 tokio::time::sleep(Duration::from_millis(10)).await;
12873 }
12874 })
12875 .await
12876 .unwrap();
12877 pause.park();
12878 std::fs::write(&release, "go").unwrap();
12879 });
12880 result.unwrap();
12881 let candidate = &runner.state.candidates[0];
12882 assert!(candidate.commits > 0, "{candidate:?}");
12883 assert!(candidate.summary.contains("saved-result"));
12884 assert_eq!(candidate.files, 1);
12885 let before = candidate.commits;
12886 runner.implement().await.unwrap();
12887 assert_eq!(runner.state.candidates[0].commits, before);
12888 }
12889 }
12890
12891 #[tokio::test]
12892 #[cfg(unix)]
12893 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12894 let tmp = tempfile::tempdir().unwrap();
12895 let mut runner = registered_runner(tmp.path()).await;
12896 let winner = runner.state.candidates[0].clone();
12897 let cache = tmp.path().join("cache");
12898 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12899 runner.state.config.graph.timeout_resource_wait = 0;
12900 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12901 let crate::cache::AcquireOutcome::Acquired(held) =
12902 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12903 else {
12904 panic!("cache must be free")
12905 };
12906 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12907 from_run: "earlier".into(),
12908 generation: 1,
12909 mode: crate::recovery::Mode::VerifyRepair,
12910 reasons: Vec::new(),
12911 reviewed_head: None,
12912 rounds: Vec::new(),
12913 open_findings: Vec::new(),
12914 evidence: None,
12915 criteria_inherited: true,
12916 repair: crate::recovery::Repair::NotStarted,
12917 });
12918 runner.recovery_repair(&winner).await.unwrap();
12919 assert!(matches!(
12920 runner.state.recovery.as_ref().unwrap().repair,
12921 crate::recovery::Repair::WaitingForRestore { .. }
12922 ));
12923 drop(held);
12924 runner.state.resource_block = None;
12925 runner.state.config.graph.timeout_resource_wait = 1;
12926 runner.recovery_repair(&winner).await.unwrap();
12927 assert!(matches!(
12928 runner.state.recovery.as_ref().unwrap().repair,
12929 crate::recovery::Repair::Done { .. }
12930 ));
12931 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12932 }
12933
12934 #[tokio::test]
12935 #[cfg(unix)]
12936 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12937 use std::os::fd::AsRawFd as _;
12938 for graft in [false, true] {
12939 let tmp = tempfile::tempdir().unwrap();
12940 let mut runner = registered_runner(tmp.path()).await;
12941 runner.state.config.graph.timeout_resource_wait = 1;
12942 let winner = runner.state.candidates[0].clone();
12943 let marker = tmp.path().join("fixer-started");
12944 let mut fixer = spec("fixer");
12945 fixer.command = vec![
12946 "sh".into(),
12947 "-c".into(),
12948 format!("echo started >> '{}'; echo done", marker.display()),
12949 ];
12950 runner.roles.fixer = Some(vec![fixer]);
12951 if graft {
12952 let mut loser = winner.clone();
12953 loser.label = 'B';
12954 loser.index = 1;
12955 runner.state.candidates.push(loser);
12956 runner.state.config.graph.graft_runner_up = true;
12957 runner.state.tally = Some(Tally {
12958 first_choice: BTreeMap::new(),
12959 borda: BTreeMap::new(),
12960 winner: winner.label,
12961 rankings: 1,
12962 unanimous_initial: true,
12963 deliberated: false,
12964 changed_votes: 0,
12965 unanimous_final: true,
12966 tie_break: None,
12967 judges: 1,
12968 present: 1,
12969 quorum: 1,
12970 met_quorum: true,
12971 uncontested: None,
12972 });
12973 runner.state.judgements.push(crate::run::Judgement {
12974 judge: 1,
12975 seat: "judge-1".into(),
12976 agent: "judge".into(),
12977 ranking: vec![winner.label, 'B'],
12978 reasons: BTreeMap::new(),
12979 confidence: None,
12980 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12981 order: vec![0, 1],
12982 failed: None,
12983 duration_ms: 0,
12984 });
12985 } else {
12986 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12987 from_run: "earlier".into(),
12988 generation: 1,
12989 mode: crate::recovery::Mode::VerifyRepair,
12990 reasons: Vec::new(),
12991 reviewed_head: None,
12992 rounds: Vec::new(),
12993 open_findings: Vec::new(),
12994 evidence: None,
12995 criteria_inherited: true,
12996 repair: crate::recovery::Repair::NotStarted,
12997 });
12998 }
12999 let reader = crate::repo_lock::SeatLease::for_checkout(
13000 &winner.worktree,
13001 false,
13002 crate::config::test_coordination().as_ref(),
13003 )
13004 .await
13005 .unwrap();
13006 let pause = runner.pause.clone();
13007 let (result, ()) = tokio::join!(
13008 async {
13009 if graft {
13010 runner.graft().await
13011 } else {
13012 runner.recovery_repair(&winner).await
13013 }
13014 },
13015 async {
13016 tokio::time::sleep(Duration::from_millis(100)).await;
13017 pause.park();
13018 }
13019 );
13020 result.unwrap();
13021 assert!(!marker.exists(), "a blocked seat must not spawn");
13022 runner.state = RunState::load(&runner.state.id).unwrap();
13024 if graft {
13025 assert_eq!(
13026 runner.state.graft.as_ref().unwrap().outcome,
13027 GraftOutcome::WaitingForRestore
13028 );
13029 } else {
13030 assert!(matches!(
13031 runner.state.recovery.as_ref().unwrap().repair,
13032 crate::recovery::Repair::WaitingForRestore { .. }
13033 ));
13034 }
13035 drop(reader);
13036 runner.pause = Pause::new();
13037 runner.state.resource_block = None;
13038 let external =
13040 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13041 assert_eq!(
13042 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13043 0
13044 );
13045 let leftover = winner.worktree.join("leftover");
13046 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13047 let result = if graft {
13048 runner.graft().await
13049 } else {
13050 runner.recovery_repair(&winner).await
13051 };
13052 assert!(
13053 result
13054 .unwrap_err()
13055 .is::<crate::repo_lock::ResourceBlocked>()
13056 );
13057 assert!(leftover.exists());
13058 assert!(!marker.exists());
13059 drop(external);
13060 runner.state.resource_block = None;
13061 let result = if graft {
13062 runner.graft().await
13063 } else {
13064 runner.recovery_repair(&winner).await
13065 };
13066 result.unwrap();
13067 assert!(!leftover.exists(), "restore must precede retry");
13068 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13069 if graft {
13070 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13071 } else {
13072 assert!(matches!(
13073 runner.state.recovery.as_ref().unwrap().repair,
13074 crate::recovery::Repair::Done { .. }
13075 ));
13076 }
13077 }
13078 }
13079
13080 #[tokio::test]
13081 #[cfg(unix)]
13082 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13083 use std::os::fd::AsRawFd as _;
13084 let tmp = tempfile::tempdir().unwrap();
13085 let mut runner = registered_runner(tmp.path()).await;
13086 runner.state.config.graph.timeout_resource_wait = 1;
13087 let winner = runner.state.candidates[0].clone();
13088 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13089 runner.state.tally = Some(Tally {
13090 first_choice: BTreeMap::new(),
13091 borda: BTreeMap::new(),
13092 winner: winner.label,
13093 rankings: 0,
13094 unanimous_initial: false,
13095 deliberated: false,
13096 changed_votes: 0,
13097 unanimous_final: false,
13098 tie_break: None,
13099 judges: 0,
13100 present: 0,
13101 quorum: 0,
13102 met_quorum: true,
13103 uncontested: Some("fixture".into()),
13104 });
13105 runner.state.graft = Some(GraftRecord {
13106 pre: pre.clone(),
13107 post: None,
13108 lines: None,
13109 limit: None,
13110 agent: None,
13111 outcome: GraftOutcome::Running,
13112 requested: Vec::new(),
13113 adoption: None,
13114 adoption_error: None,
13115 });
13116 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13117 git::commit_all(&winner.worktree, "unfinished graft")
13118 .await
13119 .unwrap();
13120 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13121 let leftover = winner.worktree.join("leftover");
13122 std::fs::write(&leftover, "untrusted edits").unwrap();
13123 let path = runner.state.repo.join(".git/updater.lock");
13124 let external = std::fs::File::create(&path).unwrap();
13125 assert_eq!(
13126 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13127 0
13128 );
13129 assert!(
13130 runner
13131 .graft()
13132 .await
13133 .unwrap_err()
13134 .is::<crate::repo_lock::ResourceBlocked>()
13135 );
13136 assert!(matches!(
13137 runner.state.graft.as_ref().unwrap().outcome,
13138 GraftOutcome::Running
13139 ));
13140 assert_eq!(
13141 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13142 unfinished
13143 );
13144 assert!(leftover.exists());
13145 drop(external);
13146 std::fs::remove_file(&path).unwrap();
13147 std::fs::create_dir(&path).unwrap();
13148 assert!(runner.graft().await.is_err());
13149 assert!(matches!(
13150 runner.state.graft.as_ref().unwrap().outcome,
13151 GraftOutcome::Running
13152 ));
13153 assert_eq!(
13154 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13155 unfinished
13156 );
13157 assert!(leftover.exists());
13158 std::fs::remove_dir(&path).unwrap();
13159 runner.graft().await.unwrap();
13160 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13161 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13162 assert!(!leftover.exists());
13163 }
13164
13165 #[tokio::test]
13166 #[cfg(unix)]
13167 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13168 use std::os::fd::AsRawFd as _;
13169 let tmp = tempfile::tempdir().unwrap();
13170 let mut runner = registered_runner(tmp.path()).await;
13171 runner.state.prep_complete = Some(false);
13172 runner.state.config.graph.timeout_resource_wait = 1;
13173 runner.state.save().unwrap();
13174 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13175 assert_eq!(
13176 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13177 0
13178 );
13179 runner.execute().await.unwrap();
13180 let disk = RunState::load(&runner.state.id).unwrap();
13181 assert!(disk.driver_exited);
13182 assert_eq!(disk.driver_pid, None);
13183 assert_eq!(runner.state.status, RunStatus::Blocked);
13184 assert_eq!(
13185 runner.state.resource_block.as_ref().unwrap().kind,
13186 "repository coordination"
13187 );
13188 assert!(runner.state.prep_failure.is_none());
13189 assert!(runner.state.seats.is_empty());
13190 runner.pause.park();
13191 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13194 .await
13195 .unwrap()
13196 .unwrap();
13197 assert!(runner.state.parked);
13198 assert!(runner.state.prep_failure.is_none());
13199 let disk = RunState::load(&runner.state.id).unwrap();
13200 assert!(disk.driver_exited);
13201 assert_eq!(disk.driver_pid, None);
13202 }
13203
13204 #[tokio::test]
13205 #[cfg(unix)]
13206 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13207 use std::os::fd::AsRawFd as _;
13208 let tmp = tempfile::tempdir().unwrap();
13209 let mut runner = registered_runner(tmp.path()).await;
13210 runner.state.prep_complete = Some(false);
13211 runner.state.config.graph.timeout_resource_wait = 30;
13212 runner.state.save().unwrap();
13213 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13214 assert_eq!(
13215 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13216 0
13217 );
13218 assert!(
13219 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13220 .await
13221 .is_err()
13222 );
13223 let disk = RunState::load(&runner.state.id).unwrap();
13224 assert!(disk.driver_exited);
13225 assert_eq!(disk.driver_pid, None);
13226 assert!(disk.status.resumable());
13227 assert!(runner.state.driver_exited);
13228 assert_eq!(runner.state.driver_pid, None);
13229 runner.state.save().unwrap();
13230 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13231 }
13232
13233 #[tokio::test]
13234 #[cfg(unix)]
13235 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13236 let tmp = tempfile::tempdir().unwrap();
13237 let mut runner = registered_runner(tmp.path()).await;
13238 let winner = runner.state.candidates[0].clone();
13239 let cache = tmp.path().join("cache");
13240 let started = tmp.path().join("pre-gate-started");
13241 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13242 runner.state.config.verify.gate =
13243 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13244 let home = crate::run::home();
13245 let owner =
13246 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13247 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13248 crate::cache::AcquireOutcome::Acquired(g) => g,
13249 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13250 };
13251 let mut gate = Box::pin(runner.run_gate(&winner));
13252 assert!(
13253 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13254 .await
13255 .is_err()
13256 );
13257 assert!(!started.exists(), "pre-gate must wait for the cache too");
13258 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13262 let editor = crate::repo_lock::SeatLease::checkout_with(
13263 &winner.worktree,
13264 true,
13265 crate::config::test_coordination().as_ref(),
13266 &wait,
13267 )
13268 .await
13269 .unwrap();
13270 drop(editor);
13271 guard.release();
13272 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13273 .await
13274 .unwrap()
13275 .unwrap();
13276 drop(gate);
13277 assert!(outcomes.iter().all(CommandOutcome::ok));
13278 assert!(started.exists());
13279 }
13280
13281 #[tokio::test]
13282 #[cfg(unix)]
13283 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13284 use std::os::fd::AsRawFd as _;
13285
13286 ask_test_home();
13287 let tmp = tempfile::tempdir().unwrap();
13288 let repo = tmp.path().join("repo");
13289 std::fs::create_dir_all(&repo).unwrap();
13290 init_repo(&repo);
13291 let common = repo.join(".git");
13292 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13293 let mut runner = runner_at(RunStatus::Prep);
13294 runner.state.repo = repo.clone();
13295 runner.state.config.coordination = crate::config::test_coordination();
13296 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13297 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13298 runner.state.config.graph.advise = false;
13299 runner.roles.implementers = vec![spec("implement")];
13300 runner.roles.judges = vec![spec("judge")];
13301 runner.prep().await.unwrap();
13302 let winner = runner.state.candidates[0].clone();
13303 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13306 let completed = tmp.path().join("seat-completed");
13307 runner.roles.implementers[0].command = vec![
13308 "sh".into(),
13309 "-c".into(),
13310 format!("touch '{}'; echo completed", completed.display()),
13311 ];
13312 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13313 assert_eq!(
13314 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13315 0
13316 );
13317 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13318 let mut implement = Box::pin(runner.implement());
13319 tokio::select! {
13320 result = &mut implement => panic!("rescue must wait: {result:?}"),
13321 result = tokio::time::timeout(Duration::from_secs(10), async {
13322 while !completed.exists() {
13323 tokio::time::sleep(Duration::from_millis(20)).await;
13324 }
13325 }) => result.unwrap(),
13326 }
13327 assert!(
13328 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13329 .await
13330 .is_err()
13331 );
13332 assert_eq!(
13333 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13334 before
13335 );
13336 assert_eq!(
13337 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13338 0
13339 );
13340 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13341 .await
13342 .unwrap()
13343 .unwrap();
13344 drop(implement);
13345 assert_ne!(
13346 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13347 before
13348 );
13349 assert!(git::is_clean(&winner.worktree).await.unwrap());
13350
13351 let started = tmp.path().join("pre-gate-started");
13352 let proceed = tmp.path().join("pre-gate-proceed");
13353 let checked = tmp.path().join("gate-ran");
13354 runner.state.config.verify.pre_gate = vec![format!(
13355 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13356 started.display(),
13357 proceed.display()
13358 )];
13359 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13360 assert_eq!(
13361 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13362 0
13363 );
13364 let mut gate = Box::pin(runner.run_gate(&winner));
13365 assert!(
13366 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13367 .await
13368 .is_err()
13369 );
13370 assert!(!started.exists());
13371 assert!(!checked.exists());
13372 assert!(git::is_clean(&winner.worktree).await.unwrap());
13373 assert_eq!(
13374 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13375 0
13376 );
13377 tokio::select! {
13378 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13379 result = tokio::time::timeout(Duration::from_secs(10), async {
13380 while !started.exists() {
13381 tokio::time::sleep(Duration::from_millis(20)).await;
13382 }
13383 }) => result.unwrap(),
13384 }
13385 assert_eq!(
13386 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13387 -1,
13388 "mechanical edits must hold the real lock"
13389 );
13390 std::fs::write(proceed, "ready").unwrap();
13391 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13392 .await
13393 .unwrap()
13394 .unwrap();
13395 drop(gate);
13396 assert!(outcomes.iter().all(CommandOutcome::ok));
13397 assert!(checked.exists());
13398 assert!(runner.state.pre_gate_commit.is_some());
13399 assert!(git::is_clean(&winner.worktree).await.unwrap());
13400 assert_eq!(
13401 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13402 0,
13403 "gate completion must release coordination"
13404 );
13405 drop(outsider);
13406 std::fs::remove_file(common.join("updater.lock")).unwrap();
13407 std::fs::create_dir(common.join("updater.lock")).unwrap();
13408 std::fs::remove_file(&started).unwrap();
13409 std::fs::remove_file(&checked).unwrap();
13410 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13411 let error = runner.run_gate(&winner).await.unwrap_err();
13412 assert!(
13413 error
13414 .to_string()
13415 .contains("acquire mandatory repository lock")
13416 );
13417 assert!(
13418 !started.exists(),
13419 "failed acquisition must prevent pre-gate"
13420 );
13421 assert!(!checked.exists(), "failed acquisition must prevent gate");
13422 assert!(git::is_clean(&winner.worktree).await.unwrap());
13423 assert_eq!(
13424 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13425 head
13426 );
13427 }
13428
13429 #[tokio::test]
13430 #[cfg(unix)]
13431 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13432 use std::os::fd::AsRawFd as _;
13433 use std::time::Duration;
13434
13435 ask_test_home();
13436 let tmp = tempfile::tempdir().unwrap();
13437 let repo = tmp.path().join("repo");
13438 std::fs::create_dir_all(&repo).unwrap();
13439 init_repo(&repo);
13440 let common = repo.join(".git");
13441 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13442 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13443 assert_eq!(
13444 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13445 0
13446 );
13447 let mut runner = runner_at(RunStatus::Prep);
13448 runner.state.repo = repo.clone();
13449 runner.state.config.coordination = crate::config::test_coordination();
13450 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13451 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13452 runner.state.config.graph.advise = true;
13453 runner.state.config.graph.advisors = 1;
13454 runner.roles.implementers = vec![spec("implement")];
13455 runner.roles.judges = vec![spec("judge")];
13456 let root = runner.state.worktree_root();
13457 let started = tmp.path().join("setup-started");
13458 let proceed = tmp.path().join("setup-proceed");
13459 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13460 run: Some(format!(
13461 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13462 started.display(),
13463 proceed.display()
13464 )),
13465 ..Default::default()
13466 }];
13467 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13468 let config = git::git(&repo, &["config", "--local", "--list"])
13469 .await
13470 .unwrap();
13471 let mut prep = Box::pin(runner.prep());
13472 assert!(
13473 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13474 .await
13475 .is_err()
13476 );
13477 assert!(
13478 !root.exists(),
13479 "contention must prevent even checkout creation"
13480 );
13481 assert!(!started.exists(), "contention must prevent setup");
13482 assert_eq!(
13483 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13484 branches
13485 );
13486 assert_eq!(
13487 git::git(&repo, &["config", "--local", "--list"])
13488 .await
13489 .unwrap(),
13490 config
13491 );
13492 drop(lock);
13493 tokio::select! {
13494 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13495 result = tokio::time::timeout(Duration::from_secs(10), async {
13496 while !started.exists() {
13497 tokio::time::sleep(Duration::from_millis(20)).await;
13498 }
13499 }) => result.unwrap(),
13500 }
13501 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13502 assert_eq!(
13503 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13504 -1
13505 );
13506 assert!(
13507 tokio::time::timeout(
13508 Duration::from_millis(100),
13509 crate::repo_lock::SeatLease::for_checkout(
13510 &repo,
13511 true,
13512 crate::config::test_coordination().as_ref()
13513 )
13514 )
13515 .await
13516 .is_err(),
13517 "preparation must also exclude local editors"
13518 );
13519 std::fs::write(proceed, "ready").unwrap();
13520 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13521 .await
13522 .unwrap()
13523 .unwrap();
13524 drop(prep);
13525 assert_eq!(runner.state.prep_complete, Some(true));
13526 assert!(root.join("judge-1").exists());
13527 assert!(root.join("advisor-1").exists());
13528 assert_eq!(
13529 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13530 0
13531 );
13532 }
13533
13534 #[tokio::test]
13535 #[cfg(unix)]
13536 async fn preparation_lock_failure_prevents_repository_changes() {
13537 ask_test_home();
13538 let tmp = tempfile::tempdir().unwrap();
13539 let repo = tmp.path().join("repo");
13540 std::fs::create_dir_all(&repo).unwrap();
13541 init_repo(&repo);
13542 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13543 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13546 let mut runner = runner_at(RunStatus::Prep);
13547 runner.state.repo = repo.clone();
13548 runner.state.config.coordination = crate::config::test_coordination();
13549 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13550 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13551 runner.roles.implementers = vec![spec("implement")];
13552 let error = runner.prep().await.unwrap_err();
13553 assert!(
13554 error
13555 .to_string()
13556 .contains("acquire mandatory repository lock")
13557 );
13558 assert!(!runner.state.worktree_root().exists());
13559 assert!(runner.state.candidates.is_empty());
13560 assert!(!repo.join(".git/config.worktree").exists());
13561 assert_eq!(
13562 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13563 "* main"
13564 );
13565 }
13566
13567 #[tokio::test]
13568 async fn preparation_records_creation_intent_before_git_can_fail() {
13569 ask_test_home();
13570 let tmp = tempfile::tempdir().unwrap();
13571 let repo = tmp.path().join("repo");
13572 std::fs::create_dir_all(&repo).unwrap();
13573 init_repo(&repo);
13574 let mut runner = runner_at(RunStatus::Prep);
13575 runner.state.repo = repo.clone();
13576 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13577 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13578 runner.state.config.blind.commit_msg_hook = false;
13579 runner.roles.implementers = vec![spec("one")];
13580 let label = blind::assign_labels(1, runner.state.seed)[0];
13581 let branch = runner.state.branch_for(label);
13582 git::git(&repo, &["branch", &branch]).await.unwrap();
13585 assert!(runner.prep().await.is_err());
13586 let saved = RunState::load(&runner.state.id).unwrap();
13587 assert!(saved.candidates.is_empty());
13588 let event = saved
13589 .events
13590 .iter()
13591 .find(|e| e.node == "prep-intent")
13592 .unwrap();
13593 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13594 assert_eq!(intent["branch"], branch);
13595 assert_eq!(intent["start"], runner.state.base_commit);
13596 assert_eq!(
13597 intent["worktree"],
13598 runner
13599 .state
13600 .worktree_root()
13601 .join(format!("cand-{label}"))
13602 .to_string_lossy()
13603 .as_ref()
13604 );
13605 assert_eq!(intent["picks"], serde_json::json!([]));
13606 }
13607
13608 #[tokio::test]
13609 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13610 ask_test_home();
13611 let tmp = tempfile::tempdir().unwrap();
13612 let repo = tmp.path().join("repo");
13613 std::fs::create_dir_all(&repo).unwrap();
13614 init_repo(&repo);
13615 let mut runner = runner_at(RunStatus::Prep);
13616 runner.state.repo = repo.clone();
13617 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13618 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13619 runner.state.sources.pick_commits.push(refs::Input {
13620 token: "missing".into(),
13621 sha: "0".repeat(40),
13622 provenance: refs::Provenance::ExplicitCli,
13623 });
13624 assert!(runner.prep().await.is_err());
13625 assert!(runner.state.candidates.is_empty());
13626 assert!(!runner.state.worktree_root().exists());
13627 }
13628
13629 #[tokio::test]
13630 async fn prose_and_legacy_inference_never_import_into_candidates() {
13631 ask_test_home();
13632 let tmp = tempfile::tempdir().unwrap();
13633 let repo = tmp.path().join("repo");
13634 std::fs::create_dir_all(&repo).unwrap();
13635 init_repo(&repo);
13636 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13637 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13638 .await
13639 .unwrap();
13640 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13641 git::git(&repo, &["add", "."]).await.unwrap();
13642 git::git(&repo, &["commit", "-m", "historical implementation"])
13643 .await
13644 .unwrap();
13645 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13646 git::git(&repo, &["checkout", "main"]).await.unwrap();
13647 for instruction in [
13648 format!("Example: magi/history/A at {source}"),
13649 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13650 format!("Never import magi/history/A or {source}"),
13651 format!("Conversation answer: I mentioned {source} only as background"),
13652 ] {
13653 let mut runner = runner_at(RunStatus::Prep);
13654 runner.state.repo = repo.clone();
13655 runner.state.base_commit = base.clone();
13656 runner.state.instruction = instruction.clone();
13657 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13658 runner.state.config.blind.commit_msg_hook = false;
13659 runner.state.seeds = vec![refs::Seed {
13660 token: "magi/history/A".into(),
13661 sha: source.clone(),
13662 kind: refs::SeedKind::Unmerged,
13663 branch: true,
13664 detail: String::new(),
13665 }];
13666 runner.roles.implementers = vec![spec("one")];
13667 assert_eq!(seeded_instruction(&runner.state), instruction);
13668 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13670 legacy.as_object_mut().unwrap().remove("sources");
13671 runner.state = serde_json::from_value(legacy).unwrap();
13672 assert_eq!(runner.state.sources, refs::Sources::default());
13673 runner.state.save().unwrap();
13674 runner.state = RunState::load(&runner.state.id).unwrap();
13675 runner.prep().await.unwrap();
13676 for candidate in &runner.state.candidates {
13677 assert_eq!(
13678 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13679 base
13680 );
13681 assert!(!candidate.worktree.join("unwanted").exists());
13682 }
13683 }
13684 }
13685
13686 #[tokio::test]
13687 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13688 ask_test_home();
13689 let tmp = tempfile::tempdir().unwrap();
13690 let repo = tmp.path().join("repo");
13691 std::fs::create_dir_all(&repo).unwrap();
13692 init_repo(&repo);
13693 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13694 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13695 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13696 git::git(&repo, &["add", "."]).await.unwrap();
13697 git::git(&repo, &["commit", "-m", "required seed"])
13698 .await
13699 .unwrap();
13700 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13701 git::git(&repo, &["checkout", "main"]).await.unwrap();
13702 let mut runner = runner_at(RunStatus::Prep);
13703 runner.state.repo = repo.clone();
13704 runner.state.base_commit = base.clone();
13705 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13706 runner.state.config.graph.advise = false;
13707 runner.state.config.blind.commit_msg_hook = false;
13708 runner.state.sources =
13709 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13710 .await
13711 .unwrap();
13712 runner.roles.implementers = vec![spec("one"), spec("two")];
13713 runner.roles.judges = vec![spec("judge")];
13714 let label = blind::assign_labels(2, runner.state.seed)[0];
13715 let branch = runner.state.branch_for(label);
13716 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13717 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13718 .await
13719 .unwrap();
13720 runner.state.candidates.push(Candidate {
13721 index: 0,
13722 label,
13723 agent: "one".into(),
13724 branch,
13725 worktree,
13726 summary: String::new(),
13727 stat: String::new(),
13728 files: 0,
13729 commits: 0,
13730 empty: false,
13731 failed: None,
13732 verified_noop: None,
13733 duration_ms: 0,
13734 folded: false,
13735 });
13736 runner.state.save().unwrap();
13737 runner.state = RunState::load(&runner.state.id).unwrap();
13739 assert_eq!(runner.state.prep_complete, Some(false));
13740 runner.prep().await.unwrap();
13741 assert_eq!(runner.state.prep_complete, Some(true));
13742 assert_eq!(runner.state.candidates.len(), 2);
13743 for candidate in &runner.state.candidates {
13744 assert_eq!(
13745 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13746 git::tree_of(&repo, &seed).await.unwrap()
13747 );
13748 assert_eq!(
13749 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13750 "required commit"
13751 );
13752 }
13753 assert!(runner.state.worktree_root().join("judge-1").exists());
13754 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13755 runner.prep().await.unwrap();
13756 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13757
13758 runner.state.prep_complete = Some(false);
13759 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13760 run: Some(format!(
13761 "test -f '{}'",
13762 tmp.path().join("dependency-ready").display()
13763 )),
13764 ..Default::default()
13765 }];
13766 assert!(runner.prep().await.is_err());
13767 assert!(runner.state.prep_failure.is_some());
13768 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13769 runner.state = RunState::load(&runner.state.id).unwrap();
13770 assert!(
13771 runner.prep().await.is_err(),
13772 "correcting the cause alone must not auto-retry"
13773 );
13774 let failed_at = runner.state.events.last().unwrap().at;
13775 assert!(
13776 runner.retry_preparation(failed_at).is_err(),
13777 "old approvals cannot unlock the hold"
13778 );
13779 let mut task = crate::queue::Task::new(
13780 "retry".into(),
13781 "retry setup".into(),
13782 repo.clone(),
13783 crate::queue::Source::Human,
13784 );
13785 task.start(runner.state.id.clone());
13786 task.hold_machine(Some("setup failed".into()));
13787 task.release_operator("cli", None);
13788 runner
13789 .retry_preparation(task.resumption.as_ref().unwrap().at)
13790 .unwrap();
13791 runner.prep().await.unwrap();
13792 assert!(runner.state.prep_failure.is_none());
13793 assert_eq!(runner.state.prep_complete, Some(true));
13794 assert_eq!(runner.state.candidates.len(), 2);
13795 assert!(runner.state.worktree_root().join("judge-1").exists());
13796 for candidate in &runner.state.candidates {
13797 assert_eq!(
13798 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13799 git::tree_of(&repo, &seed).await.unwrap()
13800 );
13801 }
13802 }
13803
13804 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13805 let mut heads = Vec::new();
13806 for candidate in candidates {
13807 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13808 }
13809 heads
13810 }
13811
13812 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13813 ask_test_home();
13814 let tmp = tempfile::tempdir().unwrap();
13815 let repo = tmp.path().join("repo");
13816 std::fs::create_dir(&repo).unwrap();
13817 init_repo(&repo);
13818 let mut runner = runner_at(RunStatus::Reviewing);
13819 runner.state.repo = repo.clone();
13820 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13821 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13822 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13823 let branch = format!("candidate-{label}");
13824 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13825 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13826 .await
13827 .unwrap();
13828 if label == 'B' {
13829 std::fs::write(worktree.join("change"), "winner").unwrap();
13830 git::rescue_commit(&worktree, "candidate change")
13831 .await
13832 .unwrap();
13833 }
13834 runner.state.candidates.push(Candidate {
13835 index,
13836 label,
13837 agent: "alpha".into(),
13838 branch,
13839 worktree,
13840 summary: "completed reply".into(),
13841 stat: String::new(),
13842 files: usize::from(label == 'B'),
13843 commits: usize::from(label == 'B'),
13844 empty: label == 'A',
13845 failed: None,
13846 verified_noop: None,
13847 duration_ms: 1,
13848 folded: false,
13849 });
13850 }
13851 runner.state.judge_skipped = true;
13852 runner.state.tally = Some(Tally {
13853 first_choice: BTreeMap::new(),
13854 borda: BTreeMap::new(),
13855 winner: 'B',
13856 rankings: 0,
13857 unanimous_initial: false,
13858 deliberated: false,
13859 changed_votes: 0,
13860 unanimous_final: false,
13861 tie_break: None,
13862 judges: 0,
13863 present: 0,
13864 quorum: 0,
13865 met_quorum: true,
13866 uncontested: Some("only B produced a change".into()),
13867 });
13868 runner.state.save().unwrap();
13869 (tmp, runner)
13870 }
13871
13872 #[tokio::test]
13873 #[cfg(unix)]
13874 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13875 for (solo, failed_checkpoint) in
13876 [(true, false), (false, false), (true, true), (false, true)]
13877 {
13878 let (_tmp, mut runner) = preservation_resume_fixture().await;
13879 if solo {
13880 runner.state.candidates.retain(|c| c.label == 'A');
13881 }
13882 runner.state.candidates[0].empty = false;
13883 runner.state.candidates[0].summary.clear();
13884 runner.state.tally = None;
13885 runner.state.judge_skipped = false;
13886 runner.state.status = RunStatus::Implementing;
13887 runner.state.config.disk.min_free_bytes = u64::MAX;
13888 let mut implementer = spec("alpha");
13889 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13890 runner.roles.implementers = vec![implementer];
13891 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13892 if failed_checkpoint {
13893 std::fs::create_dir_all(&patch).unwrap();
13894 }
13895 runner.state.save().unwrap();
13896 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13897 runner.state = RunState::load(&runner.state.id).unwrap();
13898 assert_eq!(runner.state.status, RunStatus::Blocked);
13899 assert!(runner.state.interrupted_implementations.contains(&'A'));
13900 assert_eq!(runner.state.candidates[0].commits, 0);
13901 assert!(!runner.state.candidates[0].empty);
13902 assert!(runner.state.candidates[0].failed.is_none());
13903 if failed_checkpoint {
13904 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13905 std::fs::remove_dir(patch).unwrap();
13906 }
13907 runner.state.resource_block = None;
13908 runner.state.config.disk.min_free_bytes = 0;
13909 runner.roles.implementers[0].command = vec![
13910 "sh".into(),
13911 "-c".into(),
13912 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13913 ];
13914 runner.implement().await.unwrap();
13915 assert!(runner.state.interrupted_implementations.is_empty());
13916 assert!(runner.state.candidate_preservation.is_empty());
13917 assert_eq!(runner.state.candidates[0].commits, 1);
13918 assert!(!runner.state.candidates[0].empty);
13919 assert!(
13920 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13921 .unwrap()
13922 .contains("retried empty interrupted seat")
13923 );
13924 if !solo {
13925 assert_eq!(runner.state.candidates[1].commits, 1);
13926 }
13927 }
13928 }
13929
13930 #[tokio::test]
13931 #[cfg(unix)]
13932 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13933 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13934 let (_tmp, mut runner) = preservation_resume_fixture().await;
13935 runner.state.candidates.retain(|c| c.label == 'A');
13936 runner.state.candidates[0].empty = false;
13937 runner.state.candidates[0].summary.clear();
13938 runner.state.tally = None;
13939 runner.state.judge_skipped = false;
13940 runner.state.status = RunStatus::Implementing;
13941 runner.state.config.disk.min_free_bytes = u64::MAX;
13944 let wt = runner.state.candidates[0].worktree.clone();
13945 std::fs::write(
13946 wt.join("interrupted-source"),
13947 "retain implementation work\n",
13948 )
13949 .unwrap();
13950 if committed {
13951 git::rescue_commit(&wt, "implementation checkpoint")
13952 .await
13953 .unwrap();
13954 }
13955 let lock = git::git(
13956 &wt,
13957 &[
13958 "rev-parse",
13959 "--path-format=absolute",
13960 "--git-path",
13961 "index.lock",
13962 ],
13963 )
13964 .await
13965 .unwrap();
13966 if block_rescue {
13967 std::fs::write(&lock, "fixture-owned lock").unwrap();
13968 }
13969 let mut implementer = spec("alpha");
13970 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13971 runner.roles.implementers = vec![implementer];
13972 runner.state.save().unwrap();
13973 let result = runner.implement().await;
13974 assert_eq!(result.is_err(), block_rescue);
13975 assert_eq!(
13976 runner.state.resource_block.as_ref().unwrap().kind,
13977 "disk pressure"
13978 );
13979 assert!(
13980 runner
13981 .state
13982 .events
13983 .iter()
13984 .any(|e| e.message.contains("agent execution interrupted"))
13985 );
13986 assert!(
13987 runner
13988 .state
13989 .dir()
13990 .join("artifacts/candidate-A-recovery/staged.patch")
13991 .exists()
13992 );
13993 if block_rescue {
13994 let saved = RunState::load(&runner.state.id).unwrap();
13995 assert!(saved.candidate_preservation.contains_key(&'A'));
13996 assert!(!saved.candidates[0].empty);
13997 assert_eq!(
13998 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13999 "retain implementation work\n"
14000 );
14001 std::fs::remove_file(lock).unwrap();
14002 runner.state = saved;
14003 } else {
14004 assert_eq!(runner.state.status, RunStatus::Blocked);
14005 assert_eq!(runner.state.candidates[0].commits, 1);
14006 }
14007 runner.roles.implementers.clear();
14010 runner.state.config.disk.min_free_bytes = 0;
14011 runner.state.resource_block = None;
14012 runner.implement().await.unwrap();
14013 assert!(runner.state.candidate_preservation.is_empty());
14014 assert_eq!(runner.state.candidates[0].commits, 1);
14015 assert!(!runner.state.candidates[0].empty);
14016 assert!(git::is_clean(&wt).await.unwrap());
14017 assert!(
14018 crate::run::read_artifact(&runner.state, "cand-A.patch")
14019 .unwrap()
14020 .contains("retain implementation work")
14021 );
14022 }
14023 }
14024
14025 #[tokio::test]
14026 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14027 let (_tmp, mut runner) = preservation_resume_fixture().await;
14028 runner.implement().await.unwrap();
14030 assert!(runner.state.candidate_preservation.is_empty());
14031 assert!(
14032 !runner
14033 .state
14034 .dir()
14035 .join("artifacts/candidate-A-recovery")
14036 .exists()
14037 );
14038 runner.fold_losers().await.unwrap();
14039 assert!(!runner.state.candidates[0].worktree.exists());
14040 for schema in [14, crate::run::SCHEMA] {
14041 runner.state.schema = schema;
14042 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14043 runner.state.save().unwrap();
14044 runner.state = RunState::load(&runner.state.id).unwrap();
14045 runner.implement().await.unwrap();
14046 assert!(runner.state.candidate_preservation.is_empty());
14047 assert!(runner.state.candidates[0].folded);
14048 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14049 }
14050 }
14051
14052 #[tokio::test]
14053 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14054 let (_tmp, mut runner) = preservation_resume_fixture().await;
14055 let wt = runner.state.candidates[0].worktree.clone();
14056 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14057 runner.state.schema = 14;
14058 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14059 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14060 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14061 for _ in 0..2 {
14062 assert!(runner.implement().await.is_err());
14063 assert_eq!(runner.state.status, RunStatus::Blocked);
14064 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14065 assert!(
14066 runner.state.candidates[0].empty,
14067 "old verdict must remain unusable"
14068 );
14069 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14070 assert!(runner.state.judge_skipped);
14071 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14072 assert!(
14073 crate::run::read_artifact(&runner.state, "cand-A.patch")
14074 .unwrap()
14075 .contains("owned edits")
14076 );
14077 assert!(wt.exists());
14078 assert_eq!(
14079 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14080 .await
14081 .unwrap(),
14082 1
14083 );
14084 runner.state = RunState::load(&runner.state.id).unwrap();
14085 }
14086 }
14087
14088 #[tokio::test]
14089 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14090 let (_tmp, mut runner) = preservation_resume_fixture().await;
14091 let wt = runner.state.candidates[0].worktree.clone();
14092 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14093 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14094 let index_lock = PathBuf::from(
14095 git::git(
14096 &wt,
14097 &[
14098 "rev-parse",
14099 "--path-format=absolute",
14100 "--git-path",
14101 "index.lock",
14102 ],
14103 )
14104 .await
14105 .unwrap(),
14106 );
14107 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14108 runner.state.schema = 14;
14109 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14110 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14111 assert!(runner.implement().await.is_err());
14112 assert!(runner.after_implement().is_err());
14113 assert!(runner.fold_losers().await.is_err());
14114 assert_eq!(
14115 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14116 "failed seat edits"
14117 );
14118 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14119 runner.state = RunState::load(&runner.state.id).unwrap();
14120 std::fs::remove_file(index_lock).unwrap();
14121 assert!(runner.implement().await.is_err());
14123 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14124 assert!(runner.state.candidates[0].empty);
14125 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14126 assert!(
14127 crate::run::read_artifact(&runner.state, "cand-A.patch")
14128 .unwrap()
14129 .contains("failed seat edits")
14130 );
14131 assert!(wt.exists());
14132 }
14133
14134 #[tokio::test]
14135 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14136 let (_tmp, mut runner) = preservation_resume_fixture().await;
14137 let repo = runner.state.repo.clone();
14138 let common = git::git(
14139 &repo,
14140 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14141 )
14142 .await
14143 .unwrap();
14144 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14145 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14146 assert!(runner.fold_losers().await.is_err());
14147 assert!(!runner.state.candidates[0].worktree.exists());
14148 assert!(!runner.state.candidates[0].folded);
14149 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14150 runner.state = RunState::load(&runner.state.id).unwrap();
14151 std::fs::remove_file(branch_lock).unwrap();
14152 let judge = runner.state.worktree_root().join("judge-1");
14153 git::worktree_add_detached(&repo, &judge, "main")
14154 .await
14155 .unwrap();
14156 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14157 runner.roles.judges = vec![spec("judge")];
14158 assert!(runner.fold_losers().await.is_err());
14159 runner.state = RunState::load(&runner.state.id).unwrap();
14160 assert!(
14161 runner.state.candidates[0].folded,
14162 "candidate progress saved before judge failure"
14163 );
14164 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14165 assert!(judge.join("uncommitted").exists());
14166 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14167 runner.fold_losers().await.unwrap();
14168 assert!(!judge.exists());
14169 runner.fold_losers().await.unwrap();
14170 }
14171
14172 #[tokio::test]
14173 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14174 let (_tmp, mut runner) = preservation_resume_fixture().await;
14175 runner.state.candidates.retain(|c| c.label == 'B');
14176 runner.state.tally = None;
14177 runner.state.judge_skipped = false;
14178 runner.state.prep_complete = Some(true);
14179 runner.state.config.graph.advise = false;
14180 runner.state.config.graph.selected_implementer = Some("alpha".into());
14181 let wt = runner.state.candidates[0].worktree.clone();
14182 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14183 runner
14184 .state
14185 .candidate_preservation
14186 .insert('B', "awaiting persistence".into());
14187 let index_lock = git::git(
14188 &wt,
14189 &[
14190 "rev-parse",
14191 "--path-format=absolute",
14192 "--git-path",
14193 "index.lock",
14194 ],
14195 )
14196 .await
14197 .unwrap();
14198 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14199 runner.state.save().unwrap();
14200 assert!(runner.execute_graph().await.is_err());
14201 assert_eq!(runner.state.status, RunStatus::Blocked);
14202 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14203 std::fs::remove_file(index_lock).unwrap();
14204 for _ in 0..2 {
14207 runner.state = RunState::load(&runner.state.id).unwrap();
14208 runner.pause.park();
14209 runner.execute_graph().await.unwrap();
14210 assert_eq!(runner.state.status, RunStatus::Implementing);
14211 assert!(runner.state.candidate_preservation.is_empty());
14212 assert_eq!(runner.state.candidates[0].commits, 2);
14213 assert!(runner.state.tally.is_none());
14214 }
14215 runner.implement().await.unwrap();
14216 assert_eq!(runner.state.status, RunStatus::Judging);
14217 runner.state.status = RunStatus::Blocked;
14219 runner.state.candidates[0].failed = Some("implementation failed".into());
14220 runner.state.save().unwrap();
14221 let error = runner.execute_graph().await.unwrap_err();
14222 assert!(
14223 error
14224 .to_string()
14225 .contains("selected implementation is held")
14226 );
14227 }
14228
14229 #[tokio::test]
14230 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14231 let (_tmp, mut runner) = preservation_resume_fixture().await;
14232 let repo = runner.state.repo.clone();
14233 let candidate = runner.state.candidates[0].clone();
14234 let registered = git::worktree_holding(&repo, &candidate.branch)
14237 .await
14238 .unwrap()
14239 .unwrap();
14240 assert_eq!(
14241 registered.canonicalize().unwrap(),
14242 candidate.worktree.canonicalize().unwrap()
14243 );
14244 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14245 assert_eq!(
14246 git::worktree_holding(&repo, &candidate.branch)
14247 .await
14248 .unwrap(),
14249 Some(registered)
14250 );
14251 assert!(
14252 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14253 "the stale registration initially keeps the branch checked out"
14254 );
14255 runner.fold_losers().await.unwrap();
14256 assert!(runner.state.candidates[0].folded);
14257 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14258 assert_eq!(
14259 git::worktree_holding(&repo, &candidate.branch)
14260 .await
14261 .unwrap(),
14262 None
14263 );
14264 runner.state = RunState::load(&runner.state.id).unwrap();
14265 runner.fold_losers().await.unwrap();
14266 }
14267
14268 #[tokio::test]
14269 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14270 let (_tmp, mut runner) = preservation_resume_fixture().await;
14271 let repo = runner.state.repo.clone();
14272 let candidate = runner.state.candidates[0].clone();
14273 let winner = runner.state.candidates[1].clone();
14274 let registered = git::worktree_holding(&repo, &candidate.branch)
14275 .await
14276 .unwrap()
14277 .unwrap();
14278 assert_eq!(
14279 registered.canonicalize().unwrap(),
14280 candidate.worktree.canonicalize().unwrap()
14281 );
14282 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14283 assert_eq!(
14284 git::worktree_holding(&repo, &candidate.branch)
14285 .await
14286 .unwrap(),
14287 Some(registered)
14288 );
14289 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14290
14291 for _ in 0..2 {
14292 fold_run(&mut runner.state, false, &crate::run::home())
14293 .await
14294 .unwrap();
14295 runner.state = RunState::load(&runner.state.id).unwrap();
14296 assert!(runner.state.candidates[0].folded);
14297 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14298 assert_eq!(
14299 git::worktree_holding(&repo, &candidate.branch)
14300 .await
14301 .unwrap(),
14302 None
14303 );
14304 assert!(!runner.state.candidates[1].folded);
14305 assert!(winner.worktree.exists());
14306 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14307 }
14308 }
14309
14310 #[tokio::test]
14311 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14312 ask_test_home();
14313 let tmp = tempfile::tempdir().unwrap();
14314 let repo = tmp.path().join("repo");
14315 std::fs::create_dir(&repo).unwrap();
14316 init_repo(&repo);
14317 let mut runner = runner_at(RunStatus::Implementing);
14318 runner.state.repo = repo.clone();
14319 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14320 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14321 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14322 let branch = format!("candidate-{label}");
14323 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14324 git::worktree_add_branch(&repo, &wt, &branch, "main")
14325 .await
14326 .unwrap();
14327 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14328 git::git(&wt, &["add", "README.md"]).await.unwrap();
14329 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14330 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14331 runner.state.candidates.push(Candidate {
14332 index,
14333 label,
14334 agent: "alpha".to_owned(),
14335 branch,
14336 worktree: wt,
14337 summary: "completed implementation".to_owned(),
14338 stat: String::new(),
14339 files: 0,
14340 commits: 0,
14341 empty: false,
14342 failed: None,
14343 verified_noop: None,
14344 duration_ms: 42,
14345 folded: false,
14346 });
14347 runner
14348 .state
14349 .candidate_preservation
14350 .insert(label, "awaiting persistence".to_owned());
14351 }
14352 let wt = runner.state.candidates[0].worktree.clone();
14353 let index_lock = git::git(
14354 &wt,
14355 &[
14356 "rev-parse",
14357 "--path-format=absolute",
14358 "--git-path",
14359 "index.lock",
14360 ],
14361 )
14362 .await
14363 .unwrap();
14364 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14365 runner.state.save().unwrap();
14366 assert!(runner.persist_candidates().await.is_err());
14367 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14368 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14369 assert_eq!(
14370 runner.state.candidates[1].commits, 1,
14371 "other results are still collected"
14372 );
14373 assert!(
14374 !runner.state.candidates[0].empty,
14375 "failed rescue is not no change"
14376 );
14377 assert_eq!(
14378 runner.state.candidates[0].summary,
14379 "completed implementation"
14380 );
14381 assert!(runner.after_implement().is_err());
14382 let home = tmp.path().join("fold-home");
14383 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14384 assert!(wt.join("new-file").exists());
14385 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14386 assert!(recovery.join("staged.patch").exists());
14387 assert!(recovery.join("unstaged.patch").exists());
14388 assert_eq!(
14389 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14390 "untracked edits\n"
14391 );
14392 runner.state.begin_pass();
14393 runner.state.end_pass_parked(ParkCause::Upgrade {
14394 from: "1.0".into(),
14395 to: Some("1.1".into()),
14396 });
14397 runner.state.save().unwrap();
14398 runner.state = RunState::load(&runner.state.id).unwrap();
14399 assert!(matches!(
14400 runner.state.passes.last().unwrap().end,
14401 Some(crate::run::PassEnd::Parked {
14402 cause: ParkCause::Upgrade { .. }
14403 })
14404 ));
14405 std::fs::remove_file(index_lock).unwrap();
14406 runner.implement().await.unwrap();
14408 assert!(runner.state.candidate_preservation.is_empty());
14409 assert!(
14410 runner
14411 .state
14412 .candidates
14413 .iter()
14414 .all(|c| c.commits == 1 && !c.empty)
14415 );
14416 assert!(git::is_clean(&wt).await.unwrap());
14417 assert!(
14418 crate::run::read_artifact(&runner.state, "cand-A.patch")
14419 .unwrap()
14420 .contains("later tracked edits")
14421 );
14422 }
14423
14424 #[tokio::test]
14425 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14426 ask_test_home();
14427 let tmp = tempfile::tempdir().unwrap();
14428 let repo = tmp.path().join("repo");
14429 std::fs::create_dir(&repo).unwrap();
14430 init_repo(&repo);
14431 let wt = tmp.path().join("candidate");
14432 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14433 .await
14434 .unwrap();
14435 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14436 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14437 assert!(wt.join("uncommitted").exists());
14438 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14439 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14440 let mut runner = runner_at(RunStatus::Implementing);
14441 runner.state.repo = repo.clone();
14442 runner.state.base_commit = "missing-base".to_owned();
14443 runner.state.candidates.push(Candidate {
14444 index: 0,
14445 label: 'A',
14446 agent: "alpha".into(),
14447 branch: "candidate".into(),
14448 worktree: wt.clone(),
14449 summary: "completed".into(),
14450 stat: String::new(),
14451 files: 0,
14452 commits: 0,
14453 empty: false,
14454 failed: None,
14455 verified_noop: None,
14456 duration_ms: 1,
14457 folded: false,
14458 });
14459 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14460 assert!(
14461 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14462 .await
14463 .is_err()
14464 );
14465 assert!(!runner.state.candidates[0].folded);
14466 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14467 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14468 runner
14469 .state
14470 .candidate_preservation
14471 .insert('A', "pending".into());
14472 assert!(runner.persist_candidates().await.is_err());
14473 assert!(!runner.state.candidates[0].empty);
14474 assert!(!runner.state.candidates[0].folded);
14475 assert!(wt.exists());
14476 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14477 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14478 std::fs::create_dir_all(&patch_path).unwrap();
14479 assert!(
14480 runner.persist_candidates().await.is_err(),
14481 "artifact failure stays pending"
14482 );
14483 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14484 assert!(!runner.state.candidates[0].empty);
14485 std::fs::remove_dir(patch_path).unwrap();
14486 runner.persist_candidates().await.unwrap();
14487 assert!(
14488 runner.state.candidates[0].empty,
14489 "only successful inspection confirms no change"
14490 );
14491 }
14492
14493 fn init_repo(dir: &Path) {
14494 let run = |args: &[&str]| {
14495 let out = std::process::Command::new("git")
14496 .args(args)
14497 .current_dir(dir)
14498 .quiet()
14499 .output()
14500 .expect("spawn git");
14501 assert!(
14502 out.status.success(),
14503 "git {args:?} failed: {}",
14504 String::from_utf8_lossy(&out.stderr)
14505 );
14506 };
14507 run(&["init", "-b", "main"]);
14508 run(&["config", "user.name", "magi test"]);
14509 run(&["config", "user.email", "magi@example.com"]);
14510 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14511 run(&["add", "-A"]);
14512 run(&["commit", "-m", "init"]);
14513 }
14514
14515 fn ask_test_home() {
14523 crate::run::pin_test_home();
14524 }
14525
14526 fn runner_at(status: RunStatus) -> Runner {
14529 let mut state = RunState::new(
14530 PathBuf::from("/nonexistent/repo"),
14531 "main".to_owned(),
14532 "deadbeef".to_owned(),
14533 "task".to_owned(),
14534 Config::default(),
14535 );
14536 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14539 state.id = format!(
14540 "20000101-000000-{:04x}",
14541 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14542 );
14543 state.status = status;
14544 Runner {
14545 state,
14546 roles: ResolvedRoles {
14547 implementers: Vec::new(),
14548 judges: Vec::new(),
14549 reviewers: Vec::new(),
14550 fixer: None,
14551 conductor: conductor(),
14552 implementer_roster: Vec::new(),
14553 judge_roster: Vec::new(),
14554 reviewer_roster: Vec::new(),
14555 },
14556 sem: Arc::new(Semaphore::new(1)),
14557 pause: Pause::new(),
14558 interrupt: Pause::new(),
14559 }
14560 }
14561
14562 #[tokio::test]
14563 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14564 crate::run::pin_test_home();
14565 let mut runner = runner_at(RunStatus::Gating);
14566 runner.state.save().unwrap();
14567 let id = runner.state.id.clone();
14568 let passes = runner.state.passes.len();
14569 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14570 let mut claiming = Box::pin(runner.claim_driver());
14571 std::future::poll_fn(|cx| {
14572 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14573 std::task::Poll::Ready(())
14574 })
14575 .await;
14576 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14577 drop(owner);
14578 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14579 .await
14580 .unwrap()
14581 .unwrap();
14582 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14583 assert!(execution.runner.state.resource_block.is_none());
14584 assert_eq!(
14585 execution.runner.state.liveness(false),
14586 crate::run::Liveness::Live
14587 );
14588 drop(execution);
14589 assert!(RunState::load(&id).unwrap().driver_exited);
14590 }
14591
14592 #[tokio::test]
14593 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14594 crate::run::pin_test_home();
14595 let mut runner = runner_at(RunStatus::Gating);
14596 runner.state.save().unwrap();
14597 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14600 assert!(runner.claim_driver().await.is_err());
14601 assert!(runner.state.driver_exited);
14602 assert_eq!(runner.state.driver_pid, None);
14603 let disk = RunState::load(&runner.state.id).unwrap();
14604 assert!(disk.driver_exited);
14605 assert_eq!(disk.driver_pid, None);
14606 assert_eq!(disk.status, RunStatus::Gating);
14607 }
14608
14609 #[tokio::test]
14610 async fn an_early_graph_error_releases_its_driver() {
14611 crate::run::pin_test_home();
14612 let mut runner = runner_at(RunStatus::Blocked);
14613 runner.state.config.graph.selected_implementer = Some("selected".into());
14614 runner.state.save().unwrap();
14615 assert!(runner.execute().await.is_err());
14616 let disk = RunState::load(&runner.state.id).unwrap();
14617 assert!(disk.driver_exited);
14618 assert_eq!(disk.driver_pid, None);
14619 assert_eq!(disk.status, RunStatus::Blocked);
14620 }
14621
14622 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14623 crate::run::MergeRepair {
14624 question: "repair-question".into(),
14625 head: "old-head".into(),
14626 pr: "https://github.com/o/r/pull/42".into(),
14627 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14628 gate: vec![outcome(Some(0), false)],
14629 gate_head: Some("old-head".into()),
14630 gate_fixes: Vec::new(),
14631 pre_gate: Vec::new(),
14632 phase,
14633 failure: None,
14634 }
14635 }
14636
14637 #[tokio::test]
14638 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14639 crate::run::pin_test_home();
14640 let mut runner = runner_at(RunStatus::Landing);
14641 runner.state.merge_repairs.push(merge_repair_request(
14642 crate::run::MergeRepairPhase::Requested,
14643 ));
14644 runner.state.save().unwrap();
14645 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14646 assert!(
14647 runner
14648 .execute()
14649 .await
14650 .unwrap_err()
14651 .to_string()
14652 .contains("already running")
14653 );
14654 assert_eq!(
14655 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14656 crate::run::MergeRepairPhase::Requested
14657 );
14658 drop(claim);
14659 let mut disk = runner.state.clone();
14660 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14661 disk.save().unwrap();
14662 assert!(
14663 runner
14664 .execute()
14665 .await
14666 .unwrap_err()
14667 .to_string()
14668 .contains("decision changed")
14669 );
14670 assert_eq!(
14671 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14672 crate::run::MergeRepairPhase::Complete
14673 );
14674 }
14675
14676 #[test]
14677 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14678 crate::run::pin_test_home();
14679 let mut runner = runner_at(RunStatus::Landing);
14680 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14681 runner.state.reviews = request.reviews.clone();
14682 runner.state.gate = request.gate.clone();
14683 runner.state.gate_ran = true;
14684 runner.state.land_approval = Some(crate::run::LandApproval {
14685 question: "old".into(),
14686 head: "old-head".into(),
14687 });
14688 runner.state.merge_repairs.push(request);
14689 runner.begin_merge_repair_review(0).unwrap();
14690 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14691 assert!(!runner.state.gate_status().ok());
14692 assert!(runner.state.land_approval.is_none());
14693 let disk = RunState::load(&runner.state.id).unwrap();
14694 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14695 assert!(disk.merge_repairs[0].gate[0].ok());
14696 assert_eq!(
14697 disk.merge_repairs[0].phase,
14698 crate::run::MergeRepairPhase::Reviewing
14699 );
14700 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14701 assert!(disk.reviews.is_empty());
14702 }
14703
14704 #[tokio::test]
14705 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14706 crate::run::pin_test_home();
14707 for phase in [
14708 crate::run::MergeRepairPhase::Fixing,
14709 crate::run::MergeRepairPhase::Requested,
14710 ] {
14711 let mut runner = runner_at(RunStatus::Landing);
14712 runner.state.config.graph.land_rounds = 0;
14713 runner.state.merge_repairs.push(merge_repair_request(phase));
14714 runner.state.save().unwrap();
14715 assert!(!runner.repair_merge_review().await.unwrap());
14716 assert_eq!(runner.state.status, RunStatus::Blocked);
14717 assert_eq!(
14718 runner.state.merge_repairs[0].phase,
14719 crate::run::MergeRepairPhase::Failed
14720 );
14721 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14722 assert!(
14723 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14724 "interrupted"
14725 } else {
14726 "budget exhausted"
14727 })
14728 );
14729 assert!(runner.state.merge.is_none());
14730 }
14731 }
14732
14733 #[test]
14737 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14738 crate::run::pin_test_home();
14739 let mut runner = runner_at(RunStatus::Implementing);
14740 let interrupt = Pause::new();
14741 runner.watch_interrupt(interrupt.clone());
14742
14743 interrupt.park_because("task a1b2 asked to run first");
14744
14745 assert!(runner.park_here().expect("park_here"));
14746 assert!(runner.state.parked);
14747 let last = runner.state.events.last().expect("a park event");
14748 assert_eq!(last.node, "park");
14749 assert!(
14750 last.message.contains("task a1b2 asked to run first"),
14751 "expected the interrupt reason in {:?}",
14752 last.message
14753 );
14754 }
14755
14756 #[test]
14759 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14760 crate::run::pin_test_home();
14761 let mut runner = runner_at(RunStatus::Implementing);
14762 runner.state.begin_pass();
14763 let shutdown = Pause::new();
14764 runner.on_pause(shutdown.clone());
14765 let upgrade = ParkCause::Upgrade {
14766 from: "0.121.0".to_owned(),
14767 to: Some("0.121.1".to_owned()),
14768 };
14769 shutdown.park_for(upgrade.clone());
14770 shutdown.park_for(ParkCause::Operator);
14771
14772 assert!(runner.park_here().expect("park_here"));
14773 assert_eq!(
14774 runner.state.passes[0].end,
14775 Some(crate::run::PassEnd::Parked { cause: upgrade })
14776 );
14777 let last = runner.state.events.last().expect("a park event");
14778 assert!(last.message.contains("0.121.0"), "{}", last.message);
14779 }
14780
14781 #[test]
14789 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14790 crate::run::pin_test_home();
14791 let mut runner = runner_at(RunStatus::Implementing);
14792 let shutdown = Pause::new();
14793 runner.on_pause(shutdown.clone());
14794 let interrupt = Pause::new();
14795 runner.watch_interrupt(interrupt.clone());
14796
14797 assert!(!runner.park_here().expect("park_here"));
14799 assert!(!runner.state.parked);
14800
14801 interrupt.park_because("test");
14803 assert!(!shutdown.parked());
14804 assert!(runner.park_here().expect("park_here"));
14805 }
14806
14807 #[tokio::test]
14821 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14822 crate::run::pin_test_home();
14823 let mut runner = runner_at(RunStatus::Implementing);
14824 let interrupt = Pause::new();
14825 runner.watch_interrupt(interrupt.clone());
14826
14827 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14828 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14829
14830 let node = async move {
14834 started_tx.send(()).expect("send started");
14835 finish_rx.await.expect("recv finish");
14836 "node finished"
14837 };
14838
14839 let interrupter = async move {
14840 started_rx.await.expect("recv started");
14841 interrupt.park_because("higher-priority task waiting");
14843 tokio::task::yield_now().await;
14847 finish_tx.send(()).expect("send finish");
14848 };
14849
14850 let (node_result, ()) = tokio::join!(node, interrupter);
14851 assert_eq!(
14852 node_result, "node finished",
14853 "the in-flight call ran to completion"
14854 );
14855
14856 assert!(runner.park_here().expect("park_here"));
14859 assert!(runner.state.parked);
14860 }
14861
14862 #[test]
14868 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14869 crate::run::pin_test_home();
14870 let mut runner = runner_at(RunStatus::Judging);
14871 runner.state.config.agents = vec![conductor()];
14875 runner.state.candidates = vec![Candidate {
14876 index: 0,
14877 label: 'A',
14878 agent: "alpha".to_owned(),
14879 branch: "magi/x/A".to_owned(),
14880 worktree: PathBuf::from("/nonexistent/worktree"),
14881 summary: "did the thing".to_owned(),
14882 stat: "1 file changed".to_owned(),
14883 files: 1,
14884 commits: 1,
14885 empty: false,
14886 failed: None,
14887 verified_noop: None,
14888 duration_ms: 1234,
14889 folded: false,
14890 }];
14891 let run_id = runner.state.id.clone();
14892
14893 let interrupt = Pause::new();
14894 runner.watch_interrupt(interrupt.clone());
14895 interrupt.park_because("task c3d4 asked to run first");
14896 assert!(runner.park_here().expect("park_here"));
14897
14898 let resumed = Runner::resume(&run_id).expect("resume");
14899 assert_eq!(resumed.state.candidates.len(), 1);
14900 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14901 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14902 assert_eq!(resumed.state.status, runner.state.status);
14903 assert!(
14904 resumed.state.parked,
14905 "still parked until `execute` actually walks the graph again"
14906 );
14907 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14908 }
14909
14910 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14912 let mut q = ask::Question::new(
14913 run.to_owned(),
14914 "implement".to_owned(),
14915 "impl-A".to_owned(),
14916 "Which storage backend should the cache use?".to_owned(),
14917 String::new(),
14918 vec!["SQLite".to_owned(), "Redis".to_owned()],
14919 );
14920 store.put(&mut q).unwrap();
14921 q
14922 }
14923
14924 #[test]
14925 fn a_failed_runs_open_question_is_abandoned() {
14926 ask_test_home();
14927 let store = ask::Questions::open();
14928 let mut runner = runner_at(RunStatus::Failed);
14929 let run = runner.state.id.clone();
14930 let q = ask_open_question(&store, &run);
14931
14932 runner.settle_questions();
14933
14934 let back = store.get(&q.id).unwrap();
14935 assert!(
14936 !back.status.open(),
14937 "the seat that asked died with the run; nobody is left to read an answer"
14938 );
14939 assert!(
14940 back.detail.contains(&run) && back.detail.contains("failed"),
14941 "the reason names what the run became, not just that it is gone: {}",
14942 back.detail
14943 );
14944 }
14945
14946 #[test]
14947 fn a_merged_runs_open_question_is_abandoned_too() {
14948 ask_test_home();
14949 let store = ask::Questions::open();
14950 for status in [RunStatus::Merged, RunStatus::Ready] {
14953 let mut runner = runner_at(status);
14954 let run = runner.state.id.clone();
14955 let q = ask_open_question(&store, &run);
14956
14957 runner.settle_questions();
14958
14959 let back = store.get(&q.id).unwrap();
14960 assert!(
14961 !back.status.open(),
14962 "{status:?} run's question must not outlive the run"
14963 );
14964 }
14965 }
14966
14967 #[test]
14968 fn a_still_resumable_runs_open_question_is_left_alone() {
14969 ask_test_home();
14970 let store = ask::Questions::open();
14971 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14977 let mut runner = runner_at(status);
14978 let run = runner.state.id.clone();
14979 let q = ask_open_question(&store, &run);
14980
14981 runner.settle_questions();
14982
14983 let back = store.get(&q.id).unwrap();
14984 assert!(
14985 back.status.open(),
14986 "{status:?} is still alive; the question must still be waiting"
14987 );
14988 }
14989 }
14990
14991 #[test]
14992 fn settle_questions_never_touches_an_already_answered_question() {
14993 ask_test_home();
14994 let store = ask::Questions::open();
14995 let mut runner = runner_at(RunStatus::Failed);
14996 let run = runner.state.id.clone();
14997 let mut q = ask_open_question(&store, &run);
14998 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14999 .unwrap();
15000 store.put(&mut q).unwrap();
15001
15002 runner.settle_questions();
15007 runner.settle_questions();
15008
15009 let back = store.get(&q.id).unwrap();
15010 assert_eq!(
15011 back.status,
15012 ask::QuestionStatus::Answered,
15013 "a real answer is a decision on record, never overwritten by a sweep"
15014 );
15015 }
15016
15017 #[tokio::test]
15028 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15029 crate::run::pin_test_home();
15030 let tmp = tempfile::tempdir().expect("tempdir");
15031 let repo = tmp.path().join("repo");
15032 std::fs::create_dir_all(&repo).unwrap();
15033 init_repo(&repo);
15034
15035 let mut config = Config::default();
15036 config.graph.worktree_root = Some(tmp.path().join("wt"));
15037
15038 let mut state = RunState::new(
15039 repo.clone(),
15040 "main".to_owned(),
15041 "deadbeef".to_owned(),
15042 "task".to_owned(),
15043 config,
15044 );
15045 let root = state.worktree_root();
15046 let wt_a = root.join("cand-A");
15047 let wt_b = root.join("cand-B");
15048 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15049 .await
15050 .expect("worktree A");
15051 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15052 .await
15053 .expect("worktree B");
15054
15055 state.candidates = vec![
15056 Candidate {
15057 index: 0,
15058 label: 'A',
15059 agent: "alpha".to_owned(),
15060 branch: "magi/x/A".to_owned(),
15061 worktree: wt_a.clone(),
15062 summary: String::new(),
15063 stat: String::new(),
15064 files: 0,
15065 commits: 0,
15066 empty: false,
15067 failed: None,
15068 verified_noop: None,
15069 duration_ms: 0,
15070 folded: false,
15071 },
15072 Candidate {
15073 index: 1,
15074 label: 'B',
15075 agent: "beta".to_owned(),
15076 branch: "magi/x/B".to_owned(),
15077 worktree: wt_b.clone(),
15078 summary: String::new(),
15079 stat: String::new(),
15080 files: 0,
15081 commits: 0,
15082 empty: false,
15083 failed: None,
15084 verified_noop: None,
15085 duration_ms: 0,
15086 folded: false,
15087 },
15088 ];
15089 state.tally = Some(Tally {
15090 first_choice: BTreeMap::from([('A', 1)]),
15091 borda: BTreeMap::new(),
15092 winner: 'A',
15093 rankings: 1,
15094 unanimous_initial: true,
15095 deliberated: false,
15096 changed_votes: 0,
15097 unanimous_final: true,
15098 tie_break: None,
15099 judges: 1,
15100 present: 1,
15101 quorum: 1,
15102 met_quorum: true,
15103 uncontested: None,
15104 });
15105 state.status = RunStatus::Ready;
15106
15107 fold_run(&mut state, false, &crate::run::home())
15108 .await
15109 .expect("fold_run");
15110
15111 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15112 assert!(
15113 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15114 "the unmerged winner's branch survives"
15115 );
15116 assert!(
15117 !state.candidates[0].folded,
15118 "the winner is not marked folded"
15119 );
15120
15121 assert!(!wt_b.exists(), "the loser's worktree is removed");
15122 assert!(
15123 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15124 "the loser's branch is removed"
15125 );
15126 assert!(state.candidates[1].folded, "the loser is marked folded");
15127 }
15128
15129 #[tokio::test]
15130 #[cfg(unix)]
15131 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15132 use std::os::fd::AsRawFd as _;
15133 use std::time::Duration;
15134
15135 let tmp = tempfile::tempdir().unwrap();
15136 let repo = tmp.path().join("repo");
15137 std::fs::create_dir_all(&repo).unwrap();
15138 init_repo(&repo);
15139 let common = repo.join(".git");
15140 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15141 let home = tmp.path().join("home");
15142 let mut config = Config::default();
15143 config.coordination = crate::config::test_coordination();
15144 config.graph.worktree_root = Some(tmp.path().join("wt"));
15145 config.graph.timeout_resource_wait = 1;
15146 let mut state = RunState::new(
15147 repo.clone(),
15148 "main".into(),
15149 "deadbeef".into(),
15150 "task".into(),
15151 config,
15152 );
15153 let root = state.worktree_root();
15154 let candidate = root.join("cand-A");
15155 let reviewer = root.join("review-1");
15156 let branch = state.branch_for('A');
15157 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15158 .await
15159 .unwrap();
15160 git::worktree_add_detached(&repo, &reviewer, "main")
15161 .await
15162 .unwrap();
15163 git::acquire_worktree_config(&repo).await.unwrap();
15164 state.enabled_worktree_config = true;
15165 state.status = RunStatus::Ready;
15166 state.candidates.push(Candidate {
15167 index: 0,
15168 label: 'A',
15169 agent: "implement".into(),
15170 branch: branch.clone(),
15171 worktree: candidate.clone(),
15172 summary: String::new(),
15173 stat: String::new(),
15174 files: 0,
15175 commits: 0,
15176 empty: false,
15177 failed: None,
15178 verified_noop: None,
15179 duration_ms: 0,
15180 folded: false,
15181 });
15182 state.save_under(&home).unwrap();
15183 let config_before = std::fs::read(common.join("config")).unwrap();
15184 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15185 .await
15186 .unwrap();
15187 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15190 assert_eq!(
15191 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15192 0
15193 );
15194 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15195 assert!(
15196 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15197 .await
15198 .is_err()
15199 );
15200 assert!(candidate.exists());
15201 assert!(reviewer.exists());
15202 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15203 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15204 assert_eq!(
15205 git::git(&repo, &["worktree", "list", "--porcelain"])
15206 .await
15207 .unwrap(),
15208 worktrees_before
15209 );
15210 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15211 .await
15212 .unwrap()
15213 .unwrap_err();
15214 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15215 drop(fold);
15216 assert!(candidate.exists() && reviewer.exists());
15217 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15218 assert_eq!(
15219 state.resource_block.as_ref().unwrap().kind,
15220 "repository coordination"
15221 );
15222 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15223 drop(external);
15224 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15225 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15226 .await
15227 .unwrap()
15228 .unwrap();
15229 drop(fold);
15230 assert!(removed.contains(&branch));
15231 assert!(!root.exists());
15232 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15233 assert!(state.candidates[0].folded);
15234 assert!(!state.enabled_worktree_config);
15235 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15236 let enabled = git::git_raw(
15237 &repo,
15238 &["config", "--local", "--get", "extensions.worktreeConfig"],
15239 )
15240 .await
15241 .unwrap();
15242 assert!(!enabled.ok(), "the last config reference was released");
15243 }
15244
15245 #[tokio::test]
15248 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15249 let tmp = tempfile::tempdir().expect("tempdir");
15250 let repo = tmp.path().join("repo");
15251 std::fs::create_dir_all(&repo).unwrap();
15252 init_repo(&repo);
15253 let home = tmp.path().join("home");
15254
15255 let mut config = Config::default();
15256 config.graph.worktree_root = Some(tmp.path().join("wt"));
15257 let mut state = RunState::new(
15258 repo.clone(),
15259 "main".to_owned(),
15260 "deadbeef".to_owned(),
15261 "task".to_owned(),
15262 config,
15263 );
15264 git::git(&repo, &["branch", "magi/x/A", "main"])
15266 .await
15267 .expect("branch");
15268 state.candidates = vec![Candidate {
15269 index: 0,
15270 label: 'A',
15271 agent: "alpha".to_owned(),
15272 branch: "magi/x/A".to_owned(),
15273 worktree: state.worktree_root().join("cand-A"),
15274 summary: String::new(),
15275 stat: String::new(),
15276 files: 0,
15277 commits: 0,
15278 empty: false,
15279 failed: None,
15280 verified_noop: None,
15281 duration_ms: 0,
15282 folded: true,
15283 }];
15284 state.released_to = Some("20260901-000000-new1".to_owned());
15285 state.released_branches = vec!["magi/x/A".to_owned()];
15286
15287 fold_run(&mut state, true, &home).await.expect("fold_run");
15288 state.status = RunStatus::Merged;
15289 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15290 state.merge = Some(crate::run::MergeOutcome {
15291 mode: MergeMode::Pr,
15292 ok: true,
15293 detail: "confirmed merged".into(),
15294 empty: false,
15295 });
15296 fold_run(&mut state, true, &home)
15297 .await
15298 .expect("merged fold");
15299
15300 assert!(
15301 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15302 "the handed-over branch survives a fold"
15303 );
15304 }
15305
15306 #[tokio::test]
15310 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15311 let tmp = tempfile::tempdir().expect("tempdir");
15312 let repo = tmp.path().join("repo");
15313 std::fs::create_dir_all(&repo).unwrap();
15314 init_repo(&repo);
15315 let run = |args: &[&str]| {
15316 let out = std::process::Command::new("git")
15317 .quiet()
15318 .args(args)
15319 .current_dir(&repo)
15320 .output()
15321 .expect("spawn git");
15322 assert!(out.status.success(), "git {args:?}");
15323 };
15324 run(&["branch", "magi/x/A"]);
15325 run(&["checkout", "-q", "-b", "magi/x/B"]);
15326 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15327 run(&["add", "-A"]);
15328 run(&["commit", "-q", "-m", "work"]);
15329 run(&["checkout", "-q", "main"]);
15330
15331 let probe = RunState::new(
15335 repo.clone(),
15336 "main".to_owned(),
15337 "deadbeef".to_owned(),
15338 "task".to_owned(),
15339 Config::default(),
15340 );
15341 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15342 let bare = tmp.path().join("origin.git");
15343 let out = std::process::Command::new("git")
15344 .quiet()
15345 .args(["init", "-q", "--bare"])
15346 .arg(&bare)
15347 .output()
15348 .expect("spawn git");
15349 assert!(out.status.success(), "git init --bare");
15350 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15351 run(&["push", "-q", "origin", "main"]);
15352
15353 let mut state = RunState::new(
15354 repo.clone(),
15355 "main".to_owned(),
15356 "deadbeef".to_owned(),
15357 "task".to_owned(),
15358 Config::default(),
15359 );
15360 state.seeds = vec![refs::Seed {
15361 token: "magi/27b2/A".to_owned(),
15362 kind: refs::SeedKind::Unresolved,
15363 sha: String::new(),
15364 branch: true,
15365 detail: "no branch or commit named magi/27b2/A".to_owned(),
15366 }];
15367
15368 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15369 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15370 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15371 let detail = empty_candidate_detail(&state, "main");
15372 assert!(detail.starts_with("empty candidate"), "{detail}");
15373 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15374 assert!(detail.contains("--start-from"), "{detail}");
15375 }
15376
15377 #[tokio::test]
15386 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15387 let tmp = tempfile::tempdir().expect("tempdir");
15388 let repo = tmp.path().join("repo");
15389 std::fs::create_dir_all(&repo).unwrap();
15390 init_repo(&repo);
15391
15392 let mut config = Config::default();
15393 config.merge.mode = MergeMode::Local;
15394
15395 let mut state = RunState::new(
15396 repo.clone(),
15397 "main".to_owned(),
15398 "deadbeef".to_owned(),
15399 "task".to_owned(),
15400 config,
15401 );
15402 state.candidates = vec![Candidate {
15403 index: 0,
15404 label: 'A',
15405 agent: "alpha".to_owned(),
15406 branch: "does-not-exist".to_owned(),
15407 worktree: repo.clone(),
15408 summary: String::new(),
15409 stat: String::new(),
15410 files: 0,
15411 commits: 0,
15412 empty: false,
15413 failed: None,
15414 verified_noop: None,
15415 duration_ms: 0,
15416 folded: false,
15417 }];
15418 state.tally = Some(Tally {
15419 first_choice: BTreeMap::from([('A', 1)]),
15420 borda: BTreeMap::new(),
15421 winner: 'A',
15422 rankings: 1,
15423 unanimous_initial: true,
15424 deliberated: false,
15425 changed_votes: 0,
15426 unanimous_final: true,
15427 tie_break: None,
15428 judges: 0,
15429 present: 0,
15430 quorum: 0,
15431 met_quorum: true,
15432 uncontested: Some("only candidate A produced a change".to_owned()),
15433 });
15434 state.reviews = vec![ReviewRound {
15435 round: 1,
15436 head: "deadbeef".to_owned(),
15437 verified_head: None,
15438 verified_at: None,
15439 reviews: Vec::new(),
15440 e2e: Vec::new(),
15441 fix: None,
15442 blocking: 0,
15443 answered: 0,
15444 expected: 0,
15445 clean: true,
15446 verify_retried: false,
15447 e2e_deferred: false,
15448 e2e_defer_reason: None,
15449 progressed: false,
15450 vote_split: false,
15451 reconsideration: Vec::new(),
15452 verdict: None,
15453 }];
15454 state.gate = vec![CommandOutcome {
15455 command: "test".to_owned(),
15456 code: Some(0),
15457 output_tail: String::new(),
15458 duration_ms: 0,
15459 resource_blocked: false,
15460 }];
15461 state.gate_ran = true;
15462 state.status = RunStatus::Ready;
15467 state.merge = Some(MergeOutcome {
15468 mode: MergeMode::Local,
15469 ok: false,
15470 detail: "already concluded".to_owned(),
15471 empty: false,
15472 });
15473
15474 let mut runner = Runner {
15475 state,
15476 roles: ResolvedRoles {
15477 implementers: Vec::new(),
15478 judges: Vec::new(),
15479 reviewers: Vec::new(),
15480 fixer: None,
15481 conductor: conductor(),
15482 implementer_roster: Vec::new(),
15483 judge_roster: Vec::new(),
15484 reviewer_roster: Vec::new(),
15485 },
15486 sem: Arc::new(Semaphore::new(1)),
15487 pause: Pause::new(),
15488 interrupt: Pause::new(),
15489 };
15490
15491 runner.merge().await.expect("merge");
15492
15493 assert_eq!(
15494 runner.state.status,
15495 RunStatus::Ready,
15496 "a concluded run's status must not change on reentry"
15497 );
15498 assert_eq!(
15499 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15500 Some("already concluded"),
15501 "merge must not run again once the node already recorded an outcome"
15502 );
15503 }
15504
15505 #[tokio::test]
15514 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15515 let tmp = tempfile::tempdir().expect("tempdir");
15516 let repo = tmp.path().join("repo");
15517 std::fs::create_dir_all(&repo).unwrap();
15518 init_repo(&repo);
15519
15520 let mut config = Config::default();
15521 config.merge.mode = MergeMode::Local;
15522
15523 let mut state = RunState::new(
15524 repo.clone(),
15525 "main".to_owned(),
15526 "deadbeef".to_owned(),
15527 "task".to_owned(),
15528 config,
15529 );
15530 state.candidates = vec![Candidate {
15531 index: 0,
15532 label: 'A',
15533 agent: "alpha".to_owned(),
15534 branch: "does-not-exist".to_owned(),
15535 worktree: repo.clone(),
15536 summary: String::new(),
15537 stat: String::new(),
15538 files: 0,
15539 commits: 0,
15540 empty: false,
15541 failed: None,
15542 verified_noop: None,
15543 duration_ms: 0,
15544 folded: false,
15545 }];
15546 state.tally = Some(Tally {
15547 first_choice: BTreeMap::from([('A', 1)]),
15548 borda: BTreeMap::new(),
15549 winner: 'A',
15550 rankings: 1,
15551 unanimous_initial: true,
15552 deliberated: false,
15553 changed_votes: 0,
15554 unanimous_final: true,
15555 tie_break: None,
15556 judges: 0,
15557 present: 0,
15558 quorum: 0,
15559 met_quorum: true,
15560 uncontested: Some("only candidate A produced a change".to_owned()),
15561 });
15562 state.reviews = vec![ReviewRound {
15563 round: 1,
15564 head: "deadbeef".to_owned(),
15565 verified_head: None,
15566 verified_at: None,
15567 reviews: Vec::new(),
15568 e2e: Vec::new(),
15569 fix: None,
15570 blocking: 0,
15571 answered: 0,
15572 expected: 0,
15573 clean: true,
15574 verify_retried: false,
15575 e2e_deferred: false,
15576 e2e_defer_reason: None,
15577 progressed: false,
15578 vote_split: false,
15579 reconsideration: Vec::new(),
15580 verdict: None,
15581 }];
15582 state.gate = Vec::new();
15584 state.gate_ran = false;
15585 state.status = RunStatus::Gating;
15586
15587 let mut runner = Runner {
15588 state,
15589 roles: ResolvedRoles {
15590 implementers: Vec::new(),
15591 judges: Vec::new(),
15592 reviewers: Vec::new(),
15593 fixer: None,
15594 conductor: conductor(),
15595 implementer_roster: Vec::new(),
15596 judge_roster: Vec::new(),
15597 reviewer_roster: Vec::new(),
15598 },
15599 sem: Arc::new(Semaphore::new(1)),
15600 pause: Pause::new(),
15601 interrupt: Pause::new(),
15602 };
15603
15604 runner.merge().await.expect("merge");
15605
15606 assert!(
15607 runner.state.merge.is_none(),
15608 "an empty gate must never be read as a passing one: {:?}",
15609 runner.state.merge
15610 );
15611 }
15612
15613 #[tokio::test]
15620 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15621 let tmp = tempfile::tempdir().expect("tempdir");
15622 let repo = tmp.path().join("repo");
15623 std::fs::create_dir_all(&repo).unwrap();
15624 init_repo(&repo);
15625
15626 let config = Config::default();
15628
15629 let mut state = RunState::new(
15630 repo.clone(),
15631 "main".to_owned(),
15632 "deadbeef".to_owned(),
15633 "task".to_owned(),
15634 config,
15635 );
15636 state.candidates = vec![Candidate {
15637 index: 0,
15638 label: 'A',
15639 agent: "alpha".to_owned(),
15640 branch: "does-not-exist".to_owned(),
15641 worktree: repo.clone(),
15642 summary: String::new(),
15643 stat: String::new(),
15644 files: 0,
15645 commits: 0,
15646 empty: false,
15647 failed: None,
15648 verified_noop: None,
15649 duration_ms: 0,
15650 folded: false,
15651 }];
15652 state.tally = Some(Tally {
15653 first_choice: BTreeMap::from([('A', 1)]),
15654 borda: BTreeMap::new(),
15655 winner: 'A',
15656 rankings: 1,
15657 unanimous_initial: true,
15658 deliberated: false,
15659 changed_votes: 0,
15660 unanimous_final: true,
15661 tie_break: None,
15662 judges: 0,
15663 present: 0,
15664 quorum: 0,
15665 met_quorum: true,
15666 uncontested: Some("only candidate A produced a change".to_owned()),
15667 });
15668 state.reviews = vec![ReviewRound {
15669 round: 1,
15670 head: "deadbeef".to_owned(),
15671 verified_head: None,
15672 verified_at: None,
15673 reviews: Vec::new(),
15674 e2e: Vec::new(),
15675 fix: None,
15676 blocking: 0,
15677 answered: 0,
15678 expected: 0,
15679 clean: true,
15680 verify_retried: false,
15681 e2e_deferred: false,
15682 e2e_defer_reason: None,
15683 progressed: false,
15684 vote_split: false,
15685 reconsideration: Vec::new(),
15686 verdict: None,
15687 }];
15688
15689 let mut runner = Runner {
15690 state,
15691 roles: ResolvedRoles {
15692 implementers: Vec::new(),
15693 judges: Vec::new(),
15694 reviewers: Vec::new(),
15695 fixer: None,
15696 conductor: conductor(),
15697 implementer_roster: Vec::new(),
15698 judge_roster: Vec::new(),
15699 reviewer_roster: Vec::new(),
15700 },
15701 sem: Arc::new(Semaphore::new(1)),
15702 pause: Pause::new(),
15703 interrupt: Pause::new(),
15704 };
15705
15706 runner.gate().await.expect("gate");
15707 assert!(
15708 runner.state.gate_ran,
15709 "zero configured commands is still a real attempt, not an unrun gate"
15710 );
15711 assert!(runner.state.gate.is_empty());
15712 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15713 assert_ne!(
15714 runner.state.status,
15715 RunStatus::Blocked,
15716 "a gate with nothing to check must not read as failed"
15717 );
15718
15719 runner.merge().await.expect("merge");
15720 assert_eq!(
15721 runner.state.status,
15722 RunStatus::Ready,
15723 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15724 );
15725 }
15726
15727 #[tokio::test]
15738 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15739 crate::run::pin_test_home();
15740 let home = crate::run::home();
15741
15742 let tmp = tempfile::tempdir().expect("tempdir");
15743 let repo = tmp.path().join("repo");
15744 std::fs::create_dir_all(&repo).unwrap();
15745 init_repo(&repo);
15746 let cache_dir = tmp.path().join("target");
15749
15750 let mut config = Config::default();
15751 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15752 config.graph.timeout_verify = Some(2);
15755 config.graph.timeout_resource_wait = 2;
15756
15757 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15758 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15759 .expect("no io error acquiring directly")
15760 {
15761 crate::cache::AcquireOutcome::Acquired(g) => g,
15762 crate::cache::AcquireOutcome::Busy(b) => {
15763 panic!("expected the direct acquire to win the lease first: {b:?}")
15764 }
15765 };
15766
15767 let mut state = RunState::new(
15768 repo.clone(),
15769 "main".to_owned(),
15770 "deadbeef".to_owned(),
15771 "task".to_owned(),
15772 config,
15773 );
15774 state.candidates = vec![Candidate {
15775 index: 0,
15776 label: 'A',
15777 agent: "alpha".to_owned(),
15778 branch: "does-not-exist".to_owned(),
15779 worktree: repo.clone(),
15780 summary: String::new(),
15781 stat: String::new(),
15782 files: 0,
15783 commits: 0,
15784 empty: false,
15785 failed: None,
15786 verified_noop: None,
15787 duration_ms: 0,
15788 folded: false,
15789 }];
15790 state.tally = Some(Tally {
15791 first_choice: BTreeMap::from([('A', 1)]),
15792 borda: BTreeMap::new(),
15793 winner: 'A',
15794 rankings: 1,
15795 unanimous_initial: true,
15796 deliberated: false,
15797 changed_votes: 0,
15798 unanimous_final: true,
15799 tie_break: None,
15800 judges: 0,
15801 present: 0,
15802 quorum: 0,
15803 met_quorum: true,
15804 uncontested: Some("only candidate A produced a change".to_owned()),
15805 });
15806 state.reviews = vec![ReviewRound {
15807 round: 1,
15808 head: "deadbeef".to_owned(),
15809 verified_head: None,
15810 verified_at: None,
15811 reviews: Vec::new(),
15812 e2e: Vec::new(),
15813 fix: None,
15814 blocking: 0,
15815 answered: 0,
15816 expected: 0,
15817 clean: true,
15818 verify_retried: false,
15819 e2e_deferred: false,
15820 e2e_defer_reason: None,
15821 progressed: false,
15822 vote_split: false,
15823 reconsideration: Vec::new(),
15824 verdict: None,
15825 }];
15826
15827 let mut runner = Runner {
15828 state,
15829 roles: ResolvedRoles {
15830 implementers: Vec::new(),
15831 judges: Vec::new(),
15832 reviewers: Vec::new(),
15833 fixer: None,
15834 conductor: conductor(),
15835 implementer_roster: Vec::new(),
15836 judge_roster: Vec::new(),
15837 reviewer_roster: Vec::new(),
15838 },
15839 sem: Arc::new(Semaphore::new(1)),
15840 pause: Pause::new(),
15841 interrupt: Pause::new(),
15842 };
15843
15844 let started = std::time::Instant::now();
15845 runner.gate().await.expect("gate");
15846 assert!(
15847 started.elapsed() < Duration::from_secs(1),
15848 "a gate with nothing to run must never wait on a lease it never needed"
15849 );
15850 assert!(
15851 runner.state.gate_ran,
15852 "zero commands is still a real, immediate attempt"
15853 );
15854 assert!(runner.state.gate.is_empty());
15855 assert_ne!(
15856 runner.state.status,
15857 RunStatus::Blocked,
15858 "must not read as resource-blocked on a lease it never asked for"
15859 );
15860 }
15861
15862 #[cfg(unix)]
15863 #[tokio::test]
15864 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15865 crate::run::pin_test_home();
15866 let tmp = tempfile::tempdir().unwrap();
15867 let mut config = Config::default();
15868 config.disk.min_free_bytes = 0;
15869 let mut state = RunState::new(
15870 tmp.path().to_owned(),
15871 "main".into(),
15872 "base".into(),
15873 "verification evidence".into(),
15874 config,
15875 );
15876 let (outcomes, pids) = run_commands(
15877 &mut state,
15878 "gate",
15879 "gate",
15880 1,
15881 &["sh".into(), "-c".into()],
15882 &[
15883 "printf before-timeout; exec sleep 30".into(),
15884 "kill -TERM $$".into(),
15885 "exit 7".into(),
15886 "printf success".into(),
15887 ],
15888 tmp.path(),
15889 Duration::from_secs(1),
15890 )
15891 .await;
15892 wait_for_timed_out_children_to_die(&pids).await;
15893 assert_eq!(
15894 pids.len(),
15895 1,
15896 "unleased timeout retains its death-check PID"
15897 );
15898 assert_eq!(outcomes.len(), 4);
15899 assert_eq!(outcomes[0].code, None);
15900 assert!(outcomes[0].output_tail.contains("before-timeout"));
15901 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15902 assert_eq!(outcomes[1].code, None);
15903 assert!(outcomes[1].output_tail.contains("command terminated:"));
15904 assert!(!outcomes[1].output_tail.contains("timed out"));
15905 assert_eq!(outcomes[2].code, Some(7));
15906 assert!(!outcomes[2].output_tail.contains("timed out"));
15907 assert_eq!(outcomes[3].code, Some(0));
15908 assert_eq!(outcomes[3].output_tail, "success");
15909 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15910 }
15911
15912 #[tokio::test]
15922 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15923 crate::run::pin_test_home();
15924
15925 let tmp = tempfile::tempdir().expect("tempdir");
15926 let repo = tmp.path().join("repo");
15927 std::fs::create_dir_all(&repo).unwrap();
15928 init_repo(&repo);
15929
15930 let mut config = Config::default();
15931 config.verify.gate = vec![
15932 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15933 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15934 .to_owned(),
15935 ];
15936
15937 let mut state = RunState::new(
15938 repo.clone(),
15939 "main".to_owned(),
15940 "deadbeef".to_owned(),
15941 "task".to_owned(),
15942 config,
15943 );
15944 let run_id = state.id.clone();
15945 state.candidates = vec![Candidate {
15946 index: 0,
15947 label: 'A',
15948 agent: "alpha".to_owned(),
15949 branch: "does-not-exist".to_owned(),
15950 worktree: repo.clone(),
15951 summary: String::new(),
15952 stat: String::new(),
15953 files: 0,
15954 commits: 0,
15955 empty: false,
15956 failed: None,
15957 verified_noop: None,
15958 duration_ms: 0,
15959 folded: false,
15960 }];
15961 state.tally = Some(Tally {
15962 first_choice: BTreeMap::from([('A', 1)]),
15963 borda: BTreeMap::new(),
15964 winner: 'A',
15965 rankings: 1,
15966 unanimous_initial: true,
15967 deliberated: false,
15968 changed_votes: 0,
15969 unanimous_final: true,
15970 tie_break: None,
15971 judges: 0,
15972 present: 0,
15973 quorum: 0,
15974 met_quorum: true,
15975 uncontested: Some("only candidate A produced a change".to_owned()),
15976 });
15977 state.reviews = vec![ReviewRound {
15978 round: 1,
15979 head: "deadbeef".to_owned(),
15980 verified_head: None,
15981 verified_at: None,
15982 reviews: Vec::new(),
15983 e2e: Vec::new(),
15984 fix: None,
15985 blocking: 0,
15986 answered: 0,
15987 expected: 0,
15988 clean: true,
15989 verify_retried: false,
15990 e2e_deferred: false,
15991 e2e_defer_reason: None,
15992 progressed: false,
15993 vote_split: false,
15994 reconsideration: Vec::new(),
15995 verdict: None,
15996 }];
15997
15998 let mut runner = Runner {
15999 state,
16000 roles: ResolvedRoles {
16001 implementers: Vec::new(),
16002 judges: Vec::new(),
16003 reviewers: Vec::new(),
16004 fixer: None,
16005 conductor: conductor(),
16006 implementer_roster: Vec::new(),
16007 judge_roster: Vec::new(),
16008 reviewer_roster: Vec::new(),
16009 },
16010 sem: Arc::new(Semaphore::new(1)),
16011 pause: Pause::new(),
16012 interrupt: Pause::new(),
16013 };
16014
16015 let started_marker = repo.join("started.marker");
16016 let release_marker = repo.join("release.marker");
16017 let poller = tokio::spawn(async move {
16018 for _ in 0..100 {
16023 if started_marker.exists()
16024 && let Ok(s) = crate::run::RunState::load(&run_id)
16025 && let Some(a) = s.active.get("gate")
16026 {
16027 std::fs::write(&release_marker, b"go").expect("release marker");
16028 return Some(a.clone());
16029 }
16030 tokio::time::sleep(Duration::from_millis(50)).await;
16031 }
16032 None
16033 });
16034
16035 runner.gate().await.expect("gate");
16036 let captured = poller.await.expect("poller task");
16037 let captured = captured.expect(
16038 "the poller never saw a `gate` task entry in run.json while the command was \
16039 still blocked on its own release marker",
16040 );
16041
16042 assert_eq!(captured.task.as_deref(), Some("gate"));
16043 assert_eq!(captured.node, "gate");
16044 assert_eq!(captured.index, Some(1));
16045 assert_eq!(captured.total, Some(1));
16046 assert!(
16047 captured
16048 .command
16049 .as_deref()
16050 .is_some_and(|c| c.contains("started.marker")),
16051 "{captured:?}"
16052 );
16053
16054 assert!(
16055 runner.state.active.is_empty(),
16056 "the entry must be cleared once the command actually finished: {:?}",
16057 runner.state.active
16058 );
16059 assert!(runner.state.gate_ran);
16060 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16061 }
16062
16063 #[tokio::test]
16066 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16067 use crate::verdict::{Finding, ReviewVote, Severity};
16068 crate::run::pin_test_home();
16069 let tmp = tempfile::tempdir().expect("tempdir");
16070 let repo = tmp.path().join("repo");
16071 std::fs::create_dir_all(&repo).unwrap();
16072 init_repo(&repo);
16073
16074 for (severity, vote, expect) in [
16075 (Severity::Major, ReviewVote::Reject, true),
16076 (Severity::Minor, ReviewVote::Reject, false),
16077 (Severity::Major, ReviewVote::Approve, false),
16078 ] {
16079 let mut round = review_round(false, 1, 1, 1, false, true);
16080 round.reviews = vec![ReviewRecord {
16081 reviewer: 1,
16082 agent: "alpha".to_owned(),
16083 summary: String::new(),
16084 findings: vec![Finding {
16085 id: "R1-1-1".to_owned(),
16086 severity,
16087 file: None,
16088 line: None,
16089 title: "t".to_owned(),
16090 detail: String::new(),
16091 }],
16092 vote: Some(vote),
16093 failed: None,
16094 duration_ms: 0,
16095 attempts: 0,
16096 }];
16097 let mut state = RunState::new(
16098 repo.clone(),
16099 "main".to_owned(),
16100 "deadbeef".to_owned(),
16101 "task".to_owned(),
16102 Config::default(),
16103 );
16104 state.reviews = vec![round];
16105 let mut runner = Runner {
16106 state,
16107 roles: ResolvedRoles {
16108 implementers: Vec::new(),
16109 judges: Vec::new(),
16110 reviewers: Vec::new(),
16111 fixer: None,
16112 conductor: conductor(),
16113 implementer_roster: Vec::new(),
16114 judge_roster: Vec::new(),
16115 reviewer_roster: Vec::new(),
16116 },
16117 sem: Arc::new(Semaphore::new(1)),
16118 pause: Pause::new(),
16119 interrupt: Pause::new(),
16120 };
16121 let shell = runner.state.config.shell();
16122 runner
16123 .stop_reviewing("round budget spent", &shell, &repo)
16124 .await
16125 .expect("stop_reviewing");
16126 assert_eq!(runner.state.status, RunStatus::Gating);
16127 assert_eq!(
16128 runner.state.contested_handoff.is_some(),
16129 expect,
16130 "{severity:?} + {vote:?}"
16131 );
16132 }
16133 }
16134
16135 #[tokio::test]
16148 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16149 crate::run::pin_test_home();
16150 let home = crate::run::home();
16151
16152 let tmp = tempfile::tempdir().expect("tempdir");
16153 let repo = tmp.path().join("repo");
16154 std::fs::create_dir_all(&repo).unwrap();
16155 init_repo(&repo);
16156 let head = crate::git::rev_parse(&repo, "HEAD")
16157 .await
16158 .expect("rev-parse");
16159 let cache_dir = tmp.path().join("target");
16162
16163 let mut config = Config::default();
16164 config.verify.e2e = vec![format!(
16165 "CARGO_TARGET_DIR='{}' test -f README.md",
16166 cache_dir.display()
16167 )];
16168 config.graph.review_rounds = 1;
16169 config.graph.timeout_verify = Some(2);
16172 config.graph.timeout_resource_wait = 2;
16173
16174 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16175 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16176 .expect("no io error acquiring directly")
16177 {
16178 crate::cache::AcquireOutcome::Acquired(g) => g,
16179 crate::cache::AcquireOutcome::Busy(b) => {
16180 panic!("expected the direct acquire to win the lease first: {b:?}")
16181 }
16182 };
16183
16184 let mut state = RunState::new(
16185 repo.clone(),
16186 "main".to_owned(),
16187 head.clone(),
16188 "task".to_owned(),
16189 config,
16190 );
16191 state.candidates = vec![Candidate {
16192 index: 0,
16193 label: 'A',
16194 agent: "alpha".to_owned(),
16195 branch: "does-not-exist".to_owned(),
16196 worktree: repo.clone(),
16197 summary: String::new(),
16198 stat: String::new(),
16199 files: 0,
16200 commits: 0,
16201 empty: false,
16202 failed: None,
16203 verified_noop: None,
16204 duration_ms: 0,
16205 folded: false,
16206 }];
16207 state.tally = Some(Tally {
16208 first_choice: BTreeMap::from([('A', 1)]),
16209 borda: BTreeMap::new(),
16210 winner: 'A',
16211 rankings: 1,
16212 unanimous_initial: true,
16213 deliberated: false,
16214 changed_votes: 0,
16215 unanimous_final: true,
16216 tie_break: None,
16217 judges: 0,
16218 present: 0,
16219 quorum: 0,
16220 met_quorum: true,
16221 uncontested: Some("only candidate A produced a change".to_owned()),
16222 });
16223 state.reviews = vec![ReviewRound {
16227 round: 1,
16228 head: head.clone(),
16229 verified_head: None,
16230 verified_at: None,
16231 reviews: Vec::new(),
16232 e2e: Vec::new(),
16233 fix: None,
16234 blocking: 1,
16235 answered: 1,
16236 expected: 1,
16237 clean: false,
16238 verify_retried: false,
16239 e2e_deferred: true,
16240 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16241 progressed: false,
16242 vote_split: false,
16243 reconsideration: Vec::new(),
16244 verdict: None,
16245 }];
16246
16247 let mut runner = Runner {
16248 state,
16249 roles: ResolvedRoles {
16250 implementers: Vec::new(),
16251 judges: Vec::new(),
16252 reviewers: Vec::new(),
16253 fixer: None,
16254 conductor: conductor(),
16255 implementer_roster: Vec::new(),
16256 judge_roster: Vec::new(),
16257 reviewer_roster: Vec::new(),
16258 },
16259 sem: Arc::new(Semaphore::new(1)),
16260 pause: Pause::new(),
16261 interrupt: Pause::new(),
16262 };
16263
16264 let shell = runner.state.config.shell();
16265 runner
16266 .stop_reviewing("round budget spent", &shell, &repo)
16267 .await
16268 .expect("stop_reviewing");
16269
16270 let last = runner.state.reviews.last().expect("round record");
16271 assert_eq!(
16272 last.e2e_status(),
16273 E2eStatus::ResourceBlocked,
16274 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16275 failed: {last:?}"
16276 );
16277 assert_eq!(
16278 last.verified_head.as_deref(),
16279 Some(head.as_str()),
16280 "which commit this attempt targeted is known even though nothing finished checking \
16281 it"
16282 );
16283 let first_attempt_at = last
16284 .verified_at
16285 .expect("when this attempt ran is known too");
16286 assert_ne!(
16287 runner.state.status,
16288 RunStatus::Blocked,
16289 "contention is evidence about the machine, not the patch — it must not settle the \
16290 run as blocked: {:?}",
16291 runner.state.status
16292 );
16293 assert!(
16294 !runner
16295 .state
16296 .events
16297 .iter()
16298 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16299 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16300 runner.state.events
16301 );
16302
16303 runner
16308 .stop_reviewing("round budget spent", &shell, &repo)
16309 .await
16310 .expect("stop_reviewing retry");
16311 assert_eq!(
16312 runner.state.reviews.len(),
16313 1,
16314 "no new round was started: {:?}",
16315 runner.state.reviews
16316 );
16317 let last = runner.state.reviews.last().expect("round record");
16318 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16319 assert!(
16320 last.verified_at.expect("still known") > first_attempt_at,
16321 "a second reentry must be a fresh attempt, not a stale copy of the first"
16322 );
16323 assert_ne!(runner.state.status, RunStatus::Blocked);
16324
16325 held.release();
16326 }
16327
16328 #[tokio::test]
16340 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16341 crate::run::pin_test_home();
16342 let home = crate::run::home();
16343
16344 let tmp = tempfile::tempdir().expect("tempdir");
16345 let repo = tmp.path().join("repo");
16346 std::fs::create_dir_all(&repo).unwrap();
16347 init_repo(&repo);
16348 let head = crate::git::rev_parse(&repo, "HEAD")
16349 .await
16350 .expect("rev-parse");
16351 let cache_dir = tmp.path().join("target");
16352
16353 let mut config = Config::default();
16354 config.verify.e2e = vec![format!(
16355 "CARGO_TARGET_DIR='{}' test -f README.md",
16356 cache_dir.display()
16357 )];
16358 config.graph.review_rounds = 1;
16359 config.graph.timeout_verify = Some(2);
16360 config.graph.timeout_resource_wait = 2;
16361
16362 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16363 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16364 .expect("no io error acquiring directly")
16365 {
16366 crate::cache::AcquireOutcome::Acquired(g) => g,
16367 crate::cache::AcquireOutcome::Busy(b) => {
16368 panic!("expected the direct acquire to win the lease first: {b:?}")
16369 }
16370 };
16371
16372 let mut state = RunState::new(
16373 repo.clone(),
16374 "main".to_owned(),
16375 head.clone(),
16376 "task".to_owned(),
16377 config,
16378 );
16379 state.candidates = vec![Candidate {
16380 index: 0,
16381 label: 'A',
16382 agent: "alpha".to_owned(),
16383 branch: "does-not-exist".to_owned(),
16384 worktree: repo.clone(),
16385 summary: String::new(),
16386 stat: String::new(),
16387 files: 0,
16388 commits: 0,
16389 empty: false,
16390 failed: None,
16391 verified_noop: None,
16392 duration_ms: 0,
16393 folded: false,
16394 }];
16395 state.tally = Some(Tally {
16396 first_choice: BTreeMap::from([('A', 1)]),
16397 borda: BTreeMap::new(),
16398 winner: 'A',
16399 rankings: 1,
16400 unanimous_initial: true,
16401 deliberated: false,
16402 changed_votes: 0,
16403 unanimous_final: true,
16404 tie_break: None,
16405 judges: 0,
16406 present: 0,
16407 quorum: 0,
16408 met_quorum: true,
16409 uncontested: Some("only candidate A produced a change".to_owned()),
16410 });
16411 state.reviews = vec![ReviewRound {
16415 round: 1,
16416 head: head.clone(),
16417 verified_head: Some(head.clone()),
16418 verified_at: Some(jiff::Timestamp::now()),
16419 reviews: Vec::new(),
16420 e2e: vec![CommandOutcome {
16421 command: format!(
16422 "CARGO_TARGET_DIR='{}' test -f README.md",
16423 cache_dir.display()
16424 ),
16425 code: None,
16426 output_tail: "waiting for the shared build cache".to_owned(),
16427 duration_ms: 0,
16428 resource_blocked: true,
16429 }],
16430 fix: None,
16431 blocking: 1,
16432 answered: 1,
16433 expected: 1,
16434 clean: false,
16435 verify_retried: false,
16436 e2e_deferred: false,
16437 e2e_defer_reason: None,
16438 progressed: false,
16439 vote_split: false,
16440 reconsideration: Vec::new(),
16441 verdict: None,
16442 }];
16443
16444 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16445 let mut runner = Runner {
16446 state,
16447 roles: ResolvedRoles {
16448 implementers: Vec::new(),
16449 judges: Vec::new(),
16450 reviewers: Vec::new(),
16451 fixer: None,
16452 conductor: conductor(),
16453 implementer_roster: Vec::new(),
16454 judge_roster: Vec::new(),
16455 reviewer_roster: Vec::new(),
16456 },
16457 sem: Arc::new(Semaphore::new(1)),
16458 pause: Pause::new(),
16459 interrupt: Pause::new(),
16460 };
16461
16462 runner.review_loop().await.expect("review_loop");
16467
16468 assert_eq!(
16469 runner.state.reviews.len(),
16470 1,
16471 "no new round was started on top of the unresolved one: {:?}",
16472 runner.state.reviews
16473 );
16474 let last = &runner.state.reviews[0];
16475 assert_eq!(
16476 last.e2e_status(),
16477 E2eStatus::ResourceBlocked,
16478 "still contended: {last:?}"
16479 );
16480 assert!(
16481 last.verified_at.expect("still known") > first_attempt_at,
16482 "review_loop must have actually retried the check, not left it exactly as found"
16483 );
16484 assert_ne!(
16485 runner.state.status,
16486 RunStatus::Blocked,
16487 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16488 runner.state.status
16489 );
16490
16491 held.release();
16492 }
16493
16494 #[tokio::test]
16495 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16496 crate::run::pin_test_home();
16497 for status in [RunStatus::Blocked, RunStatus::Gating] {
16498 let mut runner = runner_at(status);
16499 runner.state.config.graph.land = true;
16500 runner.state.gate_ran = true;
16501 runner.state.merge = Some(MergeOutcome {
16502 mode: MergeMode::Pr,
16503 ok: false,
16504 detail: "the fixer produced no commit".into(),
16505 empty: false,
16506 });
16507 let url = "https://example.invalid/o/r/pull/1";
16508 runner.state.pr = Some(crate::run::PrRecord {
16509 url: url.into(),
16510 number: 1,
16511 state: "open".into(),
16512 checks: "red".into(),
16513 round: 1,
16514 rounds: 1,
16515 red_at_merge: Vec::new(),
16516 });
16517 runner.state.save().unwrap();
16518 runner.execute_graph().await.unwrap();
16522 assert_eq!(runner.state.status, RunStatus::Blocked);
16523 assert!(runner.state.land_sync_pending);
16524 assert!(runner.state.events.iter().any(|event|
16525 event.node == "land" && event.message == format!("watching {url}")));
16526 assert_eq!(
16527 runner.state.merge.as_ref().unwrap().detail,
16528 "the fixer produced no commit"
16529 );
16530 }
16531 }
16532
16533 #[tokio::test]
16534 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16535 crate::run::pin_test_home();
16536 let tmp = tempfile::tempdir().expect("tempdir");
16537 let repo = tmp.path().join("repo");
16538 std::fs::create_dir_all(&repo).unwrap();
16539 init_repo(&repo);
16540
16541 let mut config = Config::default();
16542 config.merge.mode = MergeMode::Pr;
16543 config.graph.land = true;
16544 config.graph.land_approval = false;
16545
16546 let mut state = RunState::new(
16547 repo.clone(),
16548 "main".to_owned(),
16549 "deadbeef".to_owned(),
16550 "task".to_owned(),
16551 config,
16552 );
16553 state.candidates = vec![Candidate {
16554 index: 0,
16555 label: 'A',
16556 agent: "alpha".to_owned(),
16557 branch: "does-not-exist".to_owned(),
16558 worktree: repo.clone(),
16559 summary: String::new(),
16560 stat: String::new(),
16561 files: 0,
16562 commits: 0,
16563 empty: false,
16564 failed: None,
16565 verified_noop: None,
16566 duration_ms: 0,
16567 folded: false,
16568 }];
16569 state.tally = Some(Tally {
16570 first_choice: BTreeMap::from([('A', 1)]),
16571 borda: BTreeMap::new(),
16572 winner: 'A',
16573 rankings: 1,
16574 unanimous_initial: true,
16575 deliberated: false,
16576 changed_votes: 0,
16577 unanimous_final: true,
16578 tie_break: None,
16579 judges: 0,
16580 present: 0,
16581 quorum: 0,
16582 met_quorum: true,
16583 uncontested: Some("only candidate A produced a change".to_owned()),
16584 });
16585 state.reviews = vec![ReviewRound {
16586 round: 1,
16587 head: "deadbeef".to_owned(),
16588 verified_head: None,
16589 verified_at: None,
16590 reviews: Vec::new(),
16591 e2e: Vec::new(),
16592 fix: None,
16593 blocking: 0,
16594 answered: 0,
16595 expected: 0,
16596 clean: true,
16597 verify_retried: false,
16598 e2e_deferred: false,
16599 e2e_defer_reason: None,
16600 progressed: false,
16601 vote_split: false,
16602 reconsideration: Vec::new(),
16603 verdict: None,
16604 }];
16605 state.gate = vec![CommandOutcome {
16606 command: "test".to_owned(),
16607 code: Some(0),
16608 output_tail: String::new(),
16609 duration_ms: 0,
16610 resource_blocked: false,
16611 }];
16612 state.gate_ran = true;
16613 state.status = RunStatus::Landing;
16617 state.merge = Some(MergeOutcome {
16618 mode: MergeMode::Pr,
16619 ok: true,
16620 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16621 empty: false,
16622 });
16623
16624 ask_test_home();
16628 state.save().expect("save landing run");
16631 let store = ask::Questions::open();
16632 let q = ask_open_question(&store, &state.id);
16633
16634 let mut runner = Runner {
16635 state,
16636 roles: ResolvedRoles {
16637 implementers: Vec::new(),
16638 judges: Vec::new(),
16639 reviewers: Vec::new(),
16640 fixer: None,
16641 conductor: conductor(),
16642 implementer_roster: Vec::new(),
16643 judge_roster: Vec::new(),
16644 reviewer_roster: Vec::new(),
16645 },
16646 sem: Arc::new(Semaphore::new(1)),
16647 pause: Pause::new(),
16648 interrupt: Pause::new(),
16649 };
16650
16651 runner.execute().await.expect("execute");
16656
16657 assert_eq!(
16658 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16659 Some("https://example.invalid/x/y/pull/1"),
16660 "reentry must not push again or open a second pull request over the \
16661 one `land` is already watching"
16662 );
16663 assert_ne!(
16664 runner.state.status,
16665 RunStatus::Landing,
16666 "land could not actually reach the fake pull request, so it must \
16667 have given up rather than left the run silently parked forever"
16668 );
16669 assert_eq!(runner.state.status, RunStatus::Blocked);
16673 assert!(
16674 store.get(&q.id).unwrap().status.open(),
16675 "Blocked is still alive; settle_questions must have been a no-op here"
16676 );
16677 }
16678
16679 fn state_with_round(round: ReviewRound) -> RunState {
16680 let mut s = RunState::new(
16681 PathBuf::from("/repo"),
16682 "main".to_owned(),
16683 "abc1234".to_owned(),
16684 "add retries".to_owned(),
16685 Config::default(),
16686 );
16687 s.reviews = vec![round];
16688 s
16689 }
16690
16691 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16692 crate::verdict::Finding {
16693 id: id.to_owned(),
16694 severity,
16695 file: None,
16696 line: None,
16697 title: title.to_owned(),
16698 detail: String::new(),
16699 }
16700 }
16701
16702 #[test]
16703 fn pr_body_names_open_findings_and_declined_ones() {
16704 let round = ReviewRound {
16705 round: 2,
16706 head: "deadbee".to_owned(),
16707 verified_head: None,
16708 verified_at: None,
16709 reviews: vec![ReviewRecord {
16710 attempts: 0,
16711 reviewer: 1,
16712 agent: "alpha".to_owned(),
16713 summary: String::new(),
16714 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16715 vote: None,
16716 failed: None,
16717 duration_ms: 0,
16718 }],
16719 e2e: vec![CommandOutcome {
16720 command: "cargo test".to_owned(),
16721 code: Some(0),
16722 output_tail: String::new(),
16723 duration_ms: 0,
16724 resource_blocked: false,
16725 }],
16726 verify_retried: false,
16727 e2e_deferred: false,
16728 e2e_defer_reason: None,
16729 fix: Some(FixRecord {
16730 agent: "alpha".to_owned(),
16731 addressed: Vec::new(),
16732 rejected: vec![crate::verdict::Rejection {
16733 id: "R1-1-1".to_owned(),
16734 why: "not reachable from any caller".to_owned(),
16735 }],
16736 notes: String::new(),
16737 committed: true,
16738 failed: None,
16739 duration_ms: 0,
16740 continuation: None,
16741 }),
16742 blocking: 0,
16743 answered: 1,
16744 expected: 1,
16745 clean: false,
16746 progressed: true,
16747 vote_split: false,
16748 reconsideration: Vec::new(),
16749 verdict: None,
16750 };
16751 let state = state_with_round(round);
16752 let body = pr_message(&state, 'A').body;
16753
16754 assert!(body.contains("add retries"), "the task must still be there");
16755 assert!(body.contains("R2-1-1"), "{body}");
16756 assert!(!body.contains("unused import"), "{body}");
16757 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16758 assert!(
16759 !body.contains("not reachable from any caller"),
16760 "the reason it was declined: {body}"
16761 );
16762 }
16763
16764 #[test]
16765 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16766 let round = ReviewRound {
16767 round: 1,
16768 head: "deadbee".to_owned(),
16769 verified_head: None,
16770 verified_at: None,
16771 reviews: vec![ReviewRecord {
16772 attempts: 0,
16773 reviewer: 1,
16774 agent: "alpha".to_owned(),
16775 summary: String::new(),
16776 findings: Vec::new(),
16777 vote: None,
16778 failed: None,
16779 duration_ms: 0,
16780 }],
16781 e2e: Vec::new(),
16782 verify_retried: false,
16783 e2e_deferred: false,
16784 e2e_defer_reason: None,
16785 fix: None,
16786 blocking: 0,
16787 answered: 1,
16788 expected: 1,
16789 clean: true,
16790 progressed: false,
16791 vote_split: false,
16792 reconsideration: Vec::new(),
16793 verdict: None,
16794 };
16795 let state = state_with_round(round);
16796 let body = pr_message(&state, 'A').body;
16797 assert!(!body.contains("Open review findings"), "{body}");
16798 assert!(!body.contains("Declined"), "{body}");
16799 }
16800
16801 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16802 let mut state = RunState::new(
16803 PathBuf::from("/repo"),
16804 "main".to_owned(),
16805 "abc1234".to_owned(),
16806 instruction.to_owned(),
16807 Config::default(),
16808 );
16809 state.candidates.push(Candidate {
16810 index: 0,
16811 label: 'A',
16812 agent: "alpha".to_owned(),
16813 branch: "magi/x/A".to_owned(),
16814 worktree: PathBuf::from("/wt"),
16815 summary: summary.to_owned(),
16816 stat: String::new(),
16817 files: 1,
16818 commits: 1,
16819 empty: false,
16820 failed: None,
16821 verified_noop: None,
16822 folded: false,
16823 duration_ms: 0,
16824 });
16825 state
16826 }
16827
16828 fn review_state(subjects: &[&str]) -> RunState {
16829 let mut state = state_with_summary(
16830 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16831 "",
16832 );
16833 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16834 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16835 state
16836 }
16837
16838 #[tokio::test]
16841 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16842 ask_test_home();
16843 let tmp = tempfile::tempdir().unwrap();
16844 let repo = tmp.path().join("repo");
16845 std::fs::create_dir_all(&repo).unwrap();
16846 init_repo(&repo);
16847 let origin = tmp.path().join("origin.git");
16848 let g = |dir: &Path, args: &[&str]| {
16849 let out = std::process::Command::new("git")
16850 .args(args)
16851 .current_dir(dir)
16852 .quiet()
16853 .output()
16854 .expect("spawn git");
16855 assert!(
16856 out.status.success(),
16857 "git {args:?}: {}",
16858 String::from_utf8_lossy(&out.stderr)
16859 );
16860 };
16861 g(
16862 tmp.path(),
16863 &[
16864 "clone",
16865 "--bare",
16866 "-q",
16867 repo.to_str().unwrap(),
16868 origin.to_str().unwrap(),
16869 ],
16870 );
16871 g(
16872 &repo,
16873 &["remote", "add", "origin", origin.to_str().unwrap()],
16874 );
16875 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16876
16877 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16879 g(&repo, &["add", "-A"]);
16880 g(
16881 &repo,
16882 &["commit", "-q", "-m", "chore(deps): update a crate"],
16883 );
16884 g(&repo, &["push", "-q", "origin", "main"]);
16885 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16886 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16887 g(&repo, &["add", "-A"]);
16888 g(
16889 &repo,
16890 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16891 );
16892 g(&repo, &["checkout", "-q", "main"]);
16893
16894 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16895 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16896 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16898 assert_eq!(fallback, c1);
16899
16900 let mut state = review_state(&[
16901 "chore(deps): update a crate",
16902 "fix(daemon): apply a chosen action",
16903 ]);
16904 state.repo = repo.clone();
16905 state.base_branch = "main".to_owned();
16906 state.base_commit = c1;
16907 refresh_reviewed_commits(&mut state, "feat/own").await;
16908 assert_eq!(
16909 state.reviewed_commits,
16910 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16911 );
16912 assert_eq!(
16913 review_title(&state).as_deref(),
16914 Some("fix(daemon): apply a chosen action")
16915 );
16916 assert_eq!(
16917 leaked_subjects(&state, "feat/own").await,
16918 Some(vec!["chore(deps): update a crate".to_owned()])
16919 );
16920 state.config.merge.remote = "nowhere".to_owned();
16923 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16924 }
16925
16926 struct SummaryFixture {
16930 _tmp: tempfile::TempDir,
16931 repo: PathBuf,
16932 reply: PathBuf,
16933 runner: Runner,
16934 }
16935
16936 fn summary_fixture() -> SummaryFixture {
16937 ask_test_home();
16938 let tmp = tempfile::tempdir().unwrap();
16939 let repo = tmp.path().join("repo");
16940 std::fs::create_dir_all(&repo).unwrap();
16941 init_repo(&repo);
16942 let origin = tmp.path().join("origin.git");
16943 let g = |args: &[&str]| {
16944 let out = std::process::Command::new("git")
16945 .args(args)
16946 .current_dir(&repo)
16947 .quiet()
16948 .output()
16949 .expect("spawn git");
16950 assert!(
16951 out.status.success(),
16952 "git {args:?}: {}",
16953 String::from_utf8_lossy(&out.stderr)
16954 );
16955 };
16956 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16957 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16958 g(&["push", "-q", "origin", "main"]);
16959 g(&["checkout", "-q", "-b", "feat/own"]);
16960 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16961 g(&["add", "-A"]);
16962 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16963 let reply = tmp.path().join("reply.txt");
16964 std::fs::write(
16965 &reply,
16966 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16967 )
16968 .unwrap();
16969 let mut runner = runner_at(RunStatus::Gating);
16970 let mut state = review_state(&["feat(own): add the own file"]);
16971 state.repo = repo.clone();
16972 state.base_branch = "main".to_owned();
16973 state.candidates[0].branch = "feat/own".to_owned();
16974 state.candidates[0].worktree = repo.clone();
16975 let mut author = spec("alpha");
16976 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16977 author
16978 .env
16979 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16980 state.config.agents = vec![author];
16981 runner.state = state;
16982 SummaryFixture {
16983 _tmp: tmp,
16984 repo,
16985 reply,
16986 runner,
16987 }
16988 }
16989
16990 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16991 let winner = f.runner.state.candidates[0].clone();
16992 f.runner
16993 .guarded_pr_message(&winner, None, true)
16994 .await
16995 .unwrap()
16996 .expect("an English description never parks")
16997 }
16998
16999 fn clean_round_with_minor_findings() -> ReviewRound {
17000 let mut round = passing_round(2, None);
17001 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17002 round.reviews = vec![ReviewRecord {
17003 attempts: 0,
17004 reviewer: 1,
17005 agent: "alpha".into(),
17006 summary: "内部レビュー".into(),
17007 findings: vec![
17008 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17009 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17010 ],
17011 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17012 failed: None,
17013 duration_ms: 0,
17014 }];
17015 round
17016 }
17017
17018 #[tokio::test]
17019 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17020 for review_only in [false, true] {
17021 let mut f = summary_fixture();
17022 if !review_only {
17023 f.runner.state.candidates[0].agent = "alpha".into();
17024 }
17025 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17026 assert!(f.runner.state.reviews[0].clean);
17027 assert!(f.runner.state.reviews[0].fix.is_none());
17028 assert!(f.runner.state.open_findings().is_empty());
17029 assert_eq!(
17030 crate::pr_outcome::required_finding_ids(&f.runner.state),
17031 ["R2-1-1", "R2-1-2"]
17032 );
17033 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17034 let evidence = crate::pr_outcome::summary_evidence(
17035 &f.runner.state,
17036 crate::pr_outcome::FinalHead::At(&head),
17037 );
17038 assert!(
17039 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17040 );
17041 let omitted = posted(&mut f).await;
17043 assert!(omitted.body.contains("No description"));
17044 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17045 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17046 assert!(f.runner.state.change_summary.is_none());
17047 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17048 let described = posted(&mut f).await;
17049 assert!(described.body.contains("improve documentation for callers"));
17050 assert!(described.body.contains("remaining edge case"));
17051 assert!(described.body.contains("## Open review findings"));
17052 assert!(!described.body.contains("未修正"));
17053 }
17054 }
17055
17056 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17057 let mut f = summary_fixture();
17058 if !review_only {
17059 f.runner.state.candidates[0].agent = "alpha".into();
17060 }
17061 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17062 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17063 fixed
17064 .fix
17065 .as_mut()
17066 .unwrap()
17067 .rejected
17068 .push(crate::verdict::Rejection {
17069 id: "R1-1-1".into(),
17070 why: "呼び出されないコードのため辞退".into(),
17071 });
17072 let mut clean = clean_round_with_minor_findings();
17073 clean.verified_head = Some(head.clone());
17074 f.runner.state.reviews = vec![fixed, clean];
17075 ok_gate(&mut f.runner.state, Some(&head));
17076 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17077 f.runner.state.config.agents.push(scripted("beta", reply));
17078 f.runner.state.config.roles.language_judge =
17079 Some(crate::config::AgentChoice::One("beta".into()));
17080 f
17081 }
17082
17083 #[tokio::test]
17084 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17085 for review_only in [false, true] {
17086 let mut f = rewrite_fixture_with_findings(review_only,
17087 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17088 let m = posted(&mut f).await;
17089 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17090 let expected =
17091 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17092 assert!(m.body.contains(&expected));
17093 assert!(m.body.contains("## Review rounds"));
17094 assert!(
17095 m.body
17096 .contains("Gate (`verify.gate`): passed on the final commit")
17097 );
17098 assert!(
17099 m.body
17100 .contains("Tests (`verify.e2e`): passed on the final commit")
17101 );
17102 let c = f.runner.state.change_summary.as_ref().unwrap();
17103 assert!(
17104 m.body
17105 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17106 );
17107 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17108 assert!(crate::pr_outcome::covers_required_findings(
17109 &f.runner.state,
17110 &m.body
17111 ));
17112 assert!(m.body.contains("documentation remains incomplete"));
17113 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17114 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17115 assert!(f.runner.state.github_text.is_none());
17116 }
17117 }
17118
17119 #[tokio::test]
17120 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17121 for review_only in [false, true] {
17122 for reply in [
17123 "TITLE: fix: public description\nA short English description.",
17124 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17125 ] {
17126 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17127 let winner = f.runner.state.candidates[0].clone();
17128 let posted = f
17129 .runner
17130 .guarded_pr_message(&winner, None, true)
17131 .await
17132 .unwrap();
17133 assert!(posted.is_none());
17134 assert!(f.runner.state.parked);
17135 let withheld = f.runner.state.github_text.as_ref().unwrap();
17136 assert!(
17137 withheld.rewrite.is_none(),
17138 "an incomplete rewrite must not be selectable"
17139 );
17140 let q = ask::Questions::open()
17141 .get(withheld.question.as_ref().unwrap())
17142 .unwrap();
17143 assert!(
17144 !q.choices
17145 .contains(&crate::github_text::USE_REWRITE.to_owned())
17146 );
17147 assert!(
17148 q.detail.contains("R1-1-1")
17149 && q.detail.contains("R2-1-1")
17150 && q.detail.contains("R2-1-2")
17151 );
17152 }
17153 }
17154 }
17155
17156 #[tokio::test]
17157 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17158 for review_only in [false, true] {
17159 let mut f = summary_fixture();
17160 if !review_only {
17161 f.runner.state.candidates[0].agent = "alpha".into();
17162 }
17163 f.runner.state.instruction = "内部タスクの説明".into();
17164 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17165 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17166 f.runner.state.reviews = vec![
17167 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17168 round_with_notes(2, Some("二回目の内部修正報告です。")),
17169 passing_round(3, Some(&head)),
17170 ];
17171 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17172 ok_gate(&mut f.runner.state, Some(&head));
17173 let captured = f.repo.join("public-prompt.txt");
17174 let author = &mut f.runner.state.config.agents[0];
17175 author.command = vec![
17176 "sh".into(),
17177 "-c".into(),
17178 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17179 ];
17180 author.env.insert(
17181 "PROMPT_FILE".into(),
17182 captured.to_string_lossy().into_owned(),
17183 );
17184 let m = posted(&mut f).await;
17185 assert!(m.body.contains("Adds `own.txt`"));
17186 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17187 assert!(m.body.contains("R1-1-1"));
17188 assert!(
17189 m.body
17190 .contains("Gate (`verify.gate`): passed on the final commit")
17191 );
17192 assert!(
17193 m.body
17194 .contains("Tests (`verify.e2e`): passed on the final commit")
17195 );
17196 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17197 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17198 let prompt = std::fs::read_to_string(captured).unwrap();
17199 assert!(prompt.contains(&format!(
17200 "git diff {} {head}",
17201 f.runner.state.change_summary.as_ref().unwrap().base
17202 )));
17203 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17204 assert_eq!(
17205 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17206 "二回目の内部修正報告です。"
17207 );
17208 }
17209 }
17210
17211 #[tokio::test]
17212 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17213 for review_only in [false, true] {
17214 let mut f = summary_fixture();
17215 if !review_only {
17216 f.runner.state.candidates[0].agent = "alpha".into();
17217 }
17218 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17219 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17220 std::fs::write(&f.reply, "no usable description").unwrap();
17221 let failed = posted(&mut f).await;
17222 assert!(
17223 failed
17224 .body
17225 .contains("No description of what this change does could be generated")
17226 );
17227 assert!(failed.body.is_ascii());
17228 assert!(f.runner.state.github_text.is_none());
17229 assert!(f.runner.state.change_summary.is_none());
17230 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17231 let evidence = crate::pr_outcome::summary_evidence(
17232 &f.runner.state,
17233 crate::pr_outcome::FinalHead::At(&head),
17234 );
17235 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17236 assert!(
17237 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17238 .exists()
17239 );
17240 f.runner.state =
17241 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17242 std::fs::write(
17243 &f.reply,
17244 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17245 )
17246 .unwrap();
17247 let retried = posted(&mut f).await;
17248 assert!(retried.body.contains("Adds the committed own file."));
17249 assert_eq!(
17250 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17251 "日本語の修正報告"
17252 );
17253 }
17254 }
17255
17256 #[tokio::test]
17257 async fn same_head_verification_and_findings_invalidate_public_prose() {
17258 let mut f = summary_fixture();
17259 posted(&mut f).await;
17260 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17261 let original = f
17262 .runner
17263 .state
17264 .change_summary
17265 .as_ref()
17266 .unwrap()
17267 .evidence
17268 .clone();
17269 ok_gate(&mut f.runner.state, Some(&head));
17270 std::fs::write(
17271 &f.reply,
17272 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17273 )
17274 .unwrap();
17275 let verified = posted(&mut f).await;
17276 assert!(verified.body.contains("now verified"));
17277 assert_ne!(
17278 f.runner.state.change_summary.as_ref().unwrap().evidence,
17279 original
17280 );
17281 let mut round = round_with_notes(1, Some("内部報告"));
17282 round.reviews.push(ReviewRecord {
17283 attempts: 0,
17284 reviewer: 1,
17285 agent: "alpha".into(),
17286 summary: "内部レビュー".into(),
17287 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17288 vote: None,
17289 failed: None,
17290 duration_ms: 0,
17291 });
17292 round
17293 .fix
17294 .as_mut()
17295 .unwrap()
17296 .rejected
17297 .push(crate::verdict::Rejection {
17298 id: "R1-1-1".into(),
17299 why: "呼び出し元から到達できない".into(),
17300 });
17301 f.runner.state.reviews = vec![round];
17302 let omitted = posted(&mut f).await;
17304 assert!(omitted.body.contains("No description"));
17305 assert!(omitted.body.contains("R1-1-1"));
17306 assert!(!omitted.body.contains("到達"));
17307 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17308 let described = posted(&mut f).await;
17309 assert!(described.body.contains("unreachable from callers"));
17310 assert!(described.body.contains("R1-1-1"));
17311 assert!(!described.body.contains("到達"));
17312 let evidence = f
17313 .runner
17314 .state
17315 .change_summary
17316 .as_ref()
17317 .unwrap()
17318 .evidence
17319 .clone();
17320 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17321 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17322 assert!(posted(&mut f).await.body.contains("rationale updated"));
17323 assert_ne!(
17324 f.runner.state.change_summary.as_ref().unwrap().evidence,
17325 evidence
17326 );
17327 f.runner
17329 .state
17330 .change_summary
17331 .as_mut()
17332 .unwrap()
17333 .evidence
17334 .clear();
17335 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17336 assert!(posted(&mut f).await.body.contains("current specification"));
17337 }
17338
17339 #[tokio::test]
17340 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17341 let mut f = summary_fixture();
17342 let m = posted(&mut f).await;
17343 assert_eq!(m.title, "feat(own): add the own file");
17344 assert!(
17345 m.body
17346 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17347 "{}",
17348 m.body
17349 );
17350 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17351 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17352 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17353 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17354 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17355 let kept = f.runner.state.change_summary.as_ref().unwrap();
17356 assert_eq!(
17357 (kept.head.as_str(), kept.base.as_str()),
17358 (head.as_str(), base.as_str())
17359 );
17360 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17361 }
17362
17363 #[tokio::test]
17364 async fn review_only_summary_receives_the_configured_github_language() {
17365 let mut f = summary_fixture();
17366 f.runner.state.config.graph.github_language = "ja".into();
17367 let captured = f.repo.join("summary-prompt.txt");
17368 let author = &mut f.runner.state.config.agents[0];
17369 author.command = vec![
17370 "sh".into(),
17371 "-c".into(),
17372 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17373 ];
17374 author.env.insert(
17375 "PROMPT_FILE".into(),
17376 captured.to_string_lossy().into_owned(),
17377 );
17378 std::fs::write(
17379 &f.reply,
17380 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17381 )
17382 .unwrap();
17383 let m = posted(&mut f).await;
17384 let prompt = std::fs::read_to_string(captured).unwrap();
17385 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17386 assert!(!prompt.contains("in English"));
17387 assert_eq!(m.title, "feat(own): ファイルを追加");
17388 assert!(m.body.contains("own.txt を追加します。"));
17389 }
17390
17391 #[tokio::test]
17392 async fn review_fixes_come_after_the_whole_change_summary() {
17393 let mut f = summary_fixture();
17394 f.runner.state.reviews = vec![
17395 round_with_notes(1, Some("handled the PATH case")),
17396 round_with_notes(2, None),
17397 ];
17398 let m = posted(&mut f).await;
17399 let summary = m.body.find("## Summary").expect("summary");
17400 let fixes = m.body.find("## Review rounds").expect("review rounds");
17401 assert!(summary < fixes, "{}", m.body);
17402 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17403 assert!(m.body.contains("## Final verification"), "{}", m.body);
17404 }
17405
17406 #[tokio::test]
17407 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17408 let mut f = summary_fixture();
17409 let first = posted(&mut f).await;
17410 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17411 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17413 let again = posted(&mut f).await;
17414 assert_eq!((again.title, again.body), (first.title, first.body));
17415
17416 let g = |args: &[&str]| {
17417 let out = std::process::Command::new("git")
17418 .args(args)
17419 .current_dir(&f.repo)
17420 .quiet()
17421 .output()
17422 .expect("spawn git");
17423 assert!(out.status.success(), "git {args:?}");
17424 };
17425 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17426 g(&["add", "-A"]);
17427 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17428 let second = posted(&mut f).await;
17429 let kept = f.runner.state.change_summary.clone().unwrap();
17430 assert_ne!(kept.head, first_head);
17431 assert!(second.body.contains("Not read yet."), "{}", second.body);
17432 assert!(!second.body.contains("so the first summary is read"));
17433 assert_eq!(second.title, "feat(own): other");
17434 }
17435
17436 #[tokio::test]
17437 async fn an_unavailable_summary_is_stated_and_never_invented() {
17438 let mut f = summary_fixture();
17439 std::fs::write(&f.reply, "no title line here").unwrap();
17440 let m = posted(&mut f).await;
17441 assert!(
17442 m.body
17443 .contains("No description of what this change does could be generated"),
17444 "{}",
17445 m.body
17446 );
17447 assert!(m.body.contains("internal reports remain in local records"));
17448 assert!(f.runner.state.change_summary.is_none());
17449 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17450 let winner = f.runner.state.candidates[0].clone();
17452 let preview = f
17453 .runner
17454 .guarded_pr_message(&winner, None, false)
17455 .await
17456 .unwrap()
17457 .unwrap();
17458 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17459 }
17460
17461 #[test]
17462 fn pr_message_review_single_commit_uses_its_subject() {
17463 let state = review_state(&["feat(nats): per-role user"]);
17464 let m = pr_message(&state, 'A');
17465 assert_eq!(m.title, "feat(nats): per-role user");
17466 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17467 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17468 }
17469
17470 #[test]
17471 fn pr_message_review_multi_commit_takes_the_oldest() {
17472 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17473 let m = pr_message(&state, 'A');
17474 assert_eq!(m.title, "feat: the change");
17475 for s in ["feat: the change", "fix: typo", "fix: again"] {
17476 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17477 }
17478 }
17479
17480 #[test]
17481 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17482 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17483 let mut state = review_state(&[first, "fix: later fixup"]);
17484 state.candidates[0].branch = "機能/ログイン".to_owned();
17485 let m = pr_message(&state, 'A');
17486 assert!(
17487 m.title.starts_with("chore: land candidate A of run"),
17488 "{}",
17489 m.title
17490 );
17491 }
17492 }
17493
17494 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17495 BranchFacts {
17496 commits: commits
17497 .iter()
17498 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17499 .collect(),
17500 stat: stat.to_owned(),
17501 }
17502 }
17503
17504 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17505 let mut r = review_round(true, 0, 1, 1, true, true);
17506 r.round = round;
17507 r.fix = notes.map(|n| FixRecord {
17508 agent: "fixer".to_owned(),
17509 addressed: Vec::new(),
17510 rejected: Vec::new(),
17511 notes: n.to_owned(),
17512 committed: true,
17513 failed: None,
17514 duration_ms: 0,
17515 continuation: None,
17516 });
17517 r
17518 }
17519
17520 #[test]
17521 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17522 let state = review_state(&["ignored"]);
17523 let f = facts(
17524 &[
17525 (
17526 "fix(login): resolve PATH on macOS",
17527 "Login shells skip rc files.",
17528 ),
17529 ("fix: address review", ""),
17530 ],
17531 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17532 );
17533 let m = pr_message_with(&state, 'A', Some(&f));
17534 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17535 assert!(
17536 m.body
17537 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17538 "{}",
17539 m.body
17540 );
17541 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17542 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17543 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17544 for banned in [
17545 "Review the work already",
17546 "no task statement",
17547 "Original task",
17548 ] {
17549 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17550 }
17551 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17552 }
17553
17554 #[test]
17555 fn pr_message_review_truncates_a_huge_first_commit_body() {
17556 let state = review_state(&["ignored"]);
17557 let f = facts(
17558 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17559 "s",
17560 );
17561 let m = pr_message_with(&state, 'A', Some(&f));
17562 assert!(m.body.len() < 30_000, "{}", m.body.len());
17563 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17564 assert!(
17565 m.body.contains("- ... 1 more commit(s)"),
17566 "{}",
17567 m.body.len()
17568 );
17569 assert!(m.body.ends_with("magi:candidate-a\n"));
17570 }
17571
17572 #[test]
17573 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17574 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17575 assert_eq!(m.title, "feat: x");
17576 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17577 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17578 assert!(!m.body.contains("no task statement"), "{}", m.body);
17579 }
17580
17581 #[test]
17582 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17583 let mut state = review_state(&["日本語の件名"]);
17584 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17585 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17586 state.candidates[0].branch = "機能/ログイン".to_owned();
17587 assert!(
17588 pr_message(&state, 'A')
17589 .title
17590 .starts_with("chore: land candidate A")
17591 );
17592 }
17593
17594 #[test]
17595 fn pr_message_review_fixes_survive_a_clean_final_round() {
17596 let mut state = review_state(&["feat: x"]);
17597 state.reviews = vec![
17598 round_with_notes(1, Some("handled the PATH case")),
17599 round_with_notes(2, None),
17600 ];
17601 let body = pr_message(&state, 'A').body;
17602 assert!(
17603 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17604 "{body}"
17605 );
17606 assert!(body.contains("### Round 1"), "{body}");
17607
17608 state.reviews = vec![
17609 round_with_notes(1, Some("first fix")),
17610 round_with_notes(2, Some("")),
17611 round_with_notes(3, Some("second fix")),
17612 round_with_notes(4, None),
17613 ];
17614 let body = pr_message(&state, 'A').body;
17615 assert!(
17616 body.contains("### Round 1") && !body.contains("first fix"),
17617 "{body}"
17618 );
17619 assert!(
17620 body.contains("### Round 3") && !body.contains("second fix"),
17621 "{body}"
17622 );
17623 assert!(body.contains("### Round 2"), "{body}");
17624 }
17625
17626 const FINAL: &str = "abcdef1234567890";
17627
17628 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17629 let mut r = round_with_notes(round, None);
17630 r.verified_head = verified.map(str::to_owned);
17631 r
17632 }
17633
17634 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17635 state.gate = vec![CommandOutcome {
17636 command: "gate".to_owned(),
17637 code: Some(0),
17638 output_tail: String::new(),
17639 duration_ms: 0,
17640 resource_blocked: false,
17641 }];
17642 state.gate_ran = true;
17643 state.gate_head = head.map(str::to_owned);
17644 }
17645
17646 fn body_at(state: &RunState) -> String {
17647 pr_message_raw(
17648 state,
17649 'A',
17650 None,
17651 crate::pr_outcome::FinalHead::At(FINAL),
17652 ChangeNote::NotAsked,
17653 )
17654 .body
17655 }
17656
17657 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17658
17659 #[test]
17660 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17661 let mut state = review_state(&["feat: x"]);
17662 state.reviews = vec![
17663 round_with_notes(1, Some(SANDBOXED)),
17664 passing_round(2, Some(FINAL)),
17665 ];
17666 ok_gate(&mut state, Some(FINAL));
17667 let body = body_at(&state);
17668 assert!(!body.contains("Handled the PATH case."), "{body}");
17669 assert!(!body.contains("could not commit"), "{body}");
17670 assert!(!body.contains("could not run"), "{body}");
17671 assert!(
17672 body.contains("The fix is committed on the branch"),
17673 "{body}"
17674 );
17675 assert!(!body.contains("sandbox"), "{body}");
17676 assert!(
17677 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17678 "{body}"
17679 );
17680 assert!(
17681 body.contains("Gate (`verify.gate`): passed on the final commit"),
17682 "{body}"
17683 );
17684 assert!(
17686 state.reviews[0]
17687 .fix
17688 .as_ref()
17689 .unwrap()
17690 .notes
17691 .contains("could not commit")
17692 );
17693 }
17694
17695 #[test]
17696 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17697 let mut state = review_state(&["feat: x"]);
17698 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17699 r1.fix.as_mut().unwrap().committed = false;
17700 let mut r2 = passing_round(2, Some(FINAL));
17701 r2.e2e[0].code = Some(1);
17702 state.reviews = vec![r1, r2];
17703 let body = body_at(&state);
17704 assert!(!body.contains("could not commit"), "{body}");
17705 assert!(!body.contains("could not run"), "{body}");
17706 assert!(body.contains("This round produced no commit"), "{body}");
17707 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17708 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17709 assert!(!body.contains("passed"), "{body}");
17710 }
17711
17712 #[test]
17713 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17714 let mut state = review_state(&["feat: x"]);
17715 state.reviews = vec![
17716 round_with_notes(1, Some(SANDBOXED)),
17717 passing_round(2, Some("0123456789abcdef")),
17718 ];
17719 ok_gate(&mut state, Some("0123456789abcdef"));
17720 let body = body_at(&state);
17721 assert!(!body.contains("HTTP tests could not run"), "{body}");
17722 assert!(!body.contains("could not commit"), "{body}");
17724 assert!(
17725 body.contains("passed on an earlier commit (`0123456`)"),
17726 "{body}"
17727 );
17728 assert!(!body.contains("passed on the final commit"), "{body}");
17729
17730 ok_gate(&mut state, None);
17732 let body = body_at(&state);
17733 assert!(
17734 body.contains("passed on a commit that was not recorded"),
17735 "{body}"
17736 );
17737 }
17738
17739 #[test]
17740 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17741 let state = state_with_summary(
17742 "add retries to the client",
17743 "TITLE: feat: retries\n\nDid it.",
17744 );
17745 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17746 assert_eq!(m.title, "feat: retries");
17747 assert!(
17748 m.body.contains("<summary>Original task</summary>"),
17749 "{}",
17750 m.body
17751 );
17752 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17753 assert!(
17754 m.body
17755 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17756 "{}",
17757 m.body
17758 );
17759 }
17760
17761 #[test]
17762 fn pr_message_review_bounds_a_long_english_subject() {
17763 let long = format!("feat: {}", "word ".repeat(100));
17764 let m = pr_message(&review_state(&[&long]), 'A');
17765 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17766 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17767 }
17768
17769 #[test]
17770 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17771 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17772 let m = pr_message(&state, 'A');
17773 assert_eq!(m.title, "add retries");
17774 assert!(m.body.contains("<summary>Original task</summary>"));
17775 assert!(!m.body.contains("Commits under review"));
17776 assert_eq!(landing_subject_source(&state), state.instruction);
17777 }
17778
17779 #[test]
17780 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17781 let state = review_state(&["feat: the change", "fix: typo"]);
17782 let source = landing_subject_source(&state);
17783 assert_eq!(land::merge_subject("", &source), "feat: the change");
17784 assert_eq!(
17785 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17786 "feat: the change"
17787 );
17788 assert_eq!(
17790 land::merge_subject("feat: renamed by hand", &source),
17791 "feat: renamed by hand"
17792 );
17793 let blank = review_state(&["日本語"]);
17794 assert!(
17795 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17796 );
17797 }
17798
17799 #[test]
17800 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17801 let state = review_state(&["feat: the change"]);
17802 let source = landing_subject_source(&state);
17803 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17804 assert_eq!(
17805 land::merge_subject(landing_title(&state, old), &source),
17806 "feat: the change"
17807 );
17808 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17809 let task = state_with_summary("add retries", "");
17810 assert_eq!(landing_title(&task, old), old);
17811 }
17812
17813 #[test]
17814 fn pr_message_describes_the_change_not_the_task() {
17815 let state = state_with_summary(
17816 "今回やってほしいこと: results projector を直す",
17817 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17818 );
17819 let m = pr_message(&state, 'A');
17820 assert_eq!(m.title, "fix(web): batch the runs list reads");
17821 assert!(
17822 m.body.starts_with("## Summary\n\n- reads run.json once"),
17823 "{}",
17824 m.body
17825 );
17826 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17827 let task_at = m.body.find("今回やってほしいこと").unwrap();
17828 let details_at = m.body.find("<details>").unwrap();
17829 assert!(
17830 details_at < task_at,
17831 "the task lives inside <details>: {}",
17832 m.body
17833 );
17834 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17835 assert!(m.body.contains("magi:candidate-a"));
17836 }
17837
17838 #[test]
17839 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17840 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17841 let m = pr_message(&state, 'A');
17842 assert_eq!(m.title, "add retries");
17843 assert!(
17844 m.body.contains("## Summary\n\n- did some things"),
17845 "{}",
17846 m.body
17847 );
17848
17849 let none = RunState::new(
17850 PathBuf::from("/repo"),
17851 "main".to_owned(),
17852 "abc1234".to_owned(),
17853 "add retries".to_owned(),
17854 Config::default(),
17855 );
17856 let m = pr_message(&none, 'A');
17857 assert_eq!(m.title, "add retries");
17858 assert!(!m.body.contains("## Summary"), "{}", m.body);
17859 }
17860
17861 #[test]
17862 fn pr_message_refuses_the_candidate_commit_subject() {
17863 for bad in [
17864 "TITLE: magi: candidate A (uncommitted work)",
17865 "TITLE: chore: stuff (uncommitted work)",
17866 "TITLE: ",
17867 ] {
17868 let state = state_with_summary("add retries", bad);
17869 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17870 }
17871 }
17872
17873 #[test]
17874 fn pr_message_bounds_a_very_long_task_and_title() {
17875 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17876 let state = state_with_summary(&long, "- nothing");
17877 let m = pr_message(&state, 'A');
17878 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17879 assert!(!m.title.contains('\n'));
17880
17881 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17882 let m = pr_message(&state, 'A');
17883 assert!(m.title.starts_with("feat: "));
17884 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17885 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17886 }
17887
17888 fn long_title_of(instruction: &str) -> String {
17889 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17890 }
17891
17892 #[test]
17893 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17894 let first = "Make the landing path keep a readable title for long tasks";
17895 let line = format!(
17896 "{first}. {}",
17897 "And then keep going with more words ".repeat(20)
17898 );
17899 let t = long_title_of(&line);
17900 assert_eq!(t, first);
17901 assert!(!t.starts_with("chore: land"));
17902 }
17903
17904 #[test]
17905 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17906 let line = "word ".repeat(200);
17907 let t = long_title_of(&line);
17908 assert!(t.ends_with("word..."), "{t}");
17909 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17910 }
17911
17912 #[test]
17913 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17914 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17915 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17916 }
17917 }
17918
17919 #[test]
17920 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17921 let mut state = state_with_summary("再試行を追加する", "- nothing");
17922 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17923 state.config.graph.github_language = "ja".into();
17924 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17925 }
17926
17927 #[test]
17928 fn pr_message_title_limit_is_exact() {
17929 let at = "a".repeat(PR_TITLE_MAX);
17930 assert_eq!(long_title_of(&at), at);
17931 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17932 assert!(over.ends_with("..."), "{over}");
17933 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17934 }
17935
17936 #[test]
17937 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17938 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17939 let t = long_title_of(&line);
17940 assert!(t.ends_with("..."), "{t}");
17941 assert!(t.is_ascii(), "{t}");
17942 }
17943
17944 #[test]
17945 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17946 let line = format!(
17947 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17948 "plus more filler words ".repeat(20)
17949 );
17950 let t = long_title_of(&line);
17951 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17952 }
17953
17954 #[test]
17955 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17956 let line = format!("\n\n# {}", "title words ".repeat(40));
17957 let state = state_with_summary(&line, "- nothing");
17958 let m = pr_message(&state, 'A');
17959 assert!(m.title.starts_with("title words"), "{}", m.title);
17960 assert_eq!(
17961 land::merge_subject(&m.title, &landing_subject_source(&state)),
17962 m.title
17963 );
17964 assert_eq!(
17966 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17967 "feat: renamed by hand"
17968 );
17969 }
17970
17971 #[tokio::test]
17972 async fn github_text_rewrite_is_bounded_and_falls_back() {
17973 crate::run::pin_test_home();
17974 for (reply, accepted) in [
17975 (
17976 "TITLE: fix: retries\nAdd retries for failed requests.",
17977 true,
17978 ),
17979 (
17980 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17981 false,
17982 ),
17983 (
17984 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17985 true,
17986 ),
17987 (
17988 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17989 true,
17990 ),
17991 (
17992 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17993 true,
17994 ),
17995 (
17996 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17997 false,
17998 ),
17999 ("TITLE: fix: retries\n", false),
18000 ] {
18001 let dir = tempfile::tempdir().unwrap();
18002 let mut runner = runner_at(RunStatus::Gating);
18003 runner.state = state_with_summary(
18004 "add retries",
18005 "TITLE: fix: retries\n日本語の説明を書きます。",
18006 );
18007 runner.state.repo = dir.path().to_owned();
18008 runner.state.candidates[0].worktree = dir.path().to_owned();
18009 let mut author = spec("alpha");
18010 author.command = vec![
18011 "sh".into(),
18012 "-c".into(),
18013 "printf '%s' \"$REWRITE_REPLY\"".into(),
18014 ];
18015 author.env.insert("REWRITE_REPLY".into(), reply.into());
18016 runner.state.config.agents = vec![author];
18017 seed_public_summary(&mut runner);
18018 let winner = runner.state.candidates[0].clone();
18019 let first = runner
18020 .guarded_pr_message(&winner, None, true)
18021 .await
18022 .unwrap();
18023 let message = if accepted {
18024 first.expect("a passing rewrite never asks")
18025 } else {
18026 assert!(first.is_none());
18028 assert!(runner.state.parked);
18029 let store = ask::Questions::open();
18030 let asked: Vec<_> = store
18031 .list()
18032 .into_iter()
18033 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18034 .collect();
18035 assert_eq!(asked.len(), 1);
18036 assert!(!asked[0].detail.contains("secret"));
18037 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18038 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18039 assert_eq!(
18040 asked[0]
18041 .choices
18042 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18043 rewrite.is_some()
18044 );
18045 store
18048 .update(&asked[0].id, |q| {
18049 q.abandon("test");
18050 Ok(())
18051 })
18052 .unwrap();
18053 let again = runner
18054 .guarded_pr_message(&winner, None, true)
18055 .await
18056 .unwrap()
18057 .expect("an abandoned question falls back");
18058 let n = store
18059 .list()
18060 .into_iter()
18061 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18062 .count();
18063 assert_eq!(n, 1, "asked once per rejected text");
18064 again
18065 };
18066 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18067 assert!(!message.body.contains("secret"));
18068 assert!(!message.body.contains("prefix"));
18069 assert!(!message.title.contains("secret"));
18070 assert_eq!(
18071 message.body.contains("Add retries for failed requests."),
18072 accepted
18073 );
18074 assert_eq!(
18075 runner.state.seats["github-text-rewrite"].turns, 1,
18076 "only one rewrite invocation"
18077 );
18078 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18079 if accepted {
18080 assert!(runner.state.github_text.is_none());
18081 assert!(!runner.state.parked);
18082 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18083 assert!(message.body.contains("## Final verification"));
18084 assert!(message.body.contains("Evidence: final diff"));
18085 assert!(
18086 message
18087 .body
18088 .contains(&format!("magi:run/{}", runner.state.id))
18089 );
18090 }
18091 if !accepted {
18092 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18093 assert!(
18094 message
18095 .body
18096 .contains(&format!("magi:run/{}", runner.state.id))
18097 );
18098 }
18099 }
18100 }
18101
18102 fn scripted(id: &str, reply: &str) -> AgentSpec {
18103 let mut a = spec(id);
18104 a.command = vec![
18105 "sh".into(),
18106 "-c".into(),
18107 "printf '%s' \"$REWRITE_REPLY\"".into(),
18108 ];
18109 a.env.insert("REWRITE_REPLY".into(), reply.into());
18110 a
18111 }
18112
18113 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18114 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18115
18116 fn seed_public_summary(runner: &mut Runner) {
18118 let repo = &runner.state.repo;
18119 let g = |args: &[&str]| {
18120 let out = std::process::Command::new("git")
18121 .args(args)
18122 .current_dir(repo)
18123 .quiet()
18124 .output()
18125 .unwrap();
18126 assert!(out.status.success(), "git {args:?}");
18127 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18128 };
18129 g(&["init", "-q", "-b", "main"]);
18130 g(&[
18131 "-c",
18132 "user.name=Test",
18133 "-c",
18134 "user.email=test@example.invalid",
18135 "commit",
18136 "-q",
18137 "--allow-empty",
18138 "-m",
18139 "feat: fixture",
18140 ]);
18141 let head = g(&["rev-parse", "HEAD"]);
18142 g(&["update-ref", "refs/remotes/origin/main", &head]);
18143 let evidence = crate::pr_outcome::summary_evidence(
18144 &runner.state,
18145 crate::pr_outcome::FinalHead::At(&head),
18146 );
18147 let generated = &runner.state.candidates[0].summary;
18148 runner.state.change_summary = Some(crate::run::ChangeSummary {
18149 base: head.clone(),
18150 head,
18151 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18152 title: summary_title(generated).unwrap(),
18153 body: summary_without_title(generated),
18154 });
18155 }
18156
18157 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18158 let mut runner = runner_at(RunStatus::Gating);
18159 runner.state = state_with_summary("add retries", JA_SUMMARY);
18160 if review_only {
18161 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18162 }
18163 runner.state.repo = dir.to_owned();
18164 runner.state.candidates[0].worktree = dir.to_owned();
18165 runner.state.config.agents = agents;
18166 seed_public_summary(&mut runner);
18167 runner
18168 }
18169
18170 fn gate_questions(runner: &Runner) -> usize {
18171 ask::Questions::open()
18172 .list()
18173 .into_iter()
18174 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18175 .count()
18176 }
18177
18178 #[tokio::test]
18179 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18180 crate::run::pin_test_home();
18181 let dir = tempfile::tempdir().unwrap();
18182 let mut runner = rewrite_runner(
18183 dir.path(),
18184 true,
18185 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18186 );
18187 runner.state.config.roles.language_judge =
18188 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18189 let winner = runner.state.candidates[0].clone();
18190 let msg = runner
18191 .guarded_pr_message(&winner, None, true)
18192 .await
18193 .unwrap()
18194 .expect("an English rewrite posts without asking");
18195 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18196 assert!(msg.body.contains("<details>kept</details>"));
18197 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18198 assert_eq!(gate_questions(&runner), 0);
18199 assert!(runner.state.github_text.is_none());
18200 assert!(!runner.state.seats.contains_key("impl-A"));
18201 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18202 }
18203
18204 #[tokio::test]
18205 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18206 crate::run::pin_test_home();
18207 let dir = tempfile::tempdir().unwrap();
18208 let mut runner = rewrite_runner(
18209 dir.path(),
18210 false,
18211 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18212 );
18213 runner.state.config.roles.language_judge =
18214 Some(crate::config::AgentChoice::One("beta".into()));
18215 runner.state.seats.insert(
18216 "impl-A".into(),
18217 SeatState::new("impl-A", "alpha", runner.state.seed),
18218 );
18219 let winner = runner.state.candidates[0].clone();
18220 let msg = runner
18221 .guarded_pr_message(&winner, None, true)
18222 .await
18223 .unwrap()
18224 .expect("rewritten");
18225 assert!(msg.body.contains("<details>kept</details>"));
18226 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18227 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18228 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18229 }
18230
18231 #[tokio::test]
18232 async fn rewrite_reuses_a_resumable_implementer_session() {
18233 crate::run::pin_test_home();
18234 let dir = tempfile::tempdir().unwrap();
18235 let mut runner = rewrite_runner(
18236 dir.path(),
18237 false,
18238 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18239 );
18240 runner.state.config.roles.language_judge =
18241 Some(crate::config::AgentChoice::One("beta".into()));
18242 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18243 seat.turns = 1;
18244 runner.state.seats.insert("impl-A".into(), seat);
18245 let winner = runner.state.candidates[0].clone();
18246 runner
18247 .guarded_pr_message(&winner, None, true)
18248 .await
18249 .unwrap()
18250 .expect("rewritten");
18251 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18252 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18253 }
18254
18255 #[tokio::test]
18256 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18257 crate::run::pin_test_home();
18258 let dir = tempfile::tempdir().unwrap();
18259 let mut broken = spec("bad");
18260 broken.command = vec!["false".into()];
18261 let mut runner = rewrite_runner(
18262 dir.path(),
18263 true,
18264 vec![
18265 broken,
18266 scripted("noformat", "Add retries."),
18267 scripted("good", EN_REWRITE),
18268 ],
18269 );
18270 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18271 "bad".into(),
18272 "noformat".into(),
18273 "good".into(),
18274 ]));
18275 let winner = runner.state.candidates[0].clone();
18276 let msg = runner
18277 .guarded_pr_message(&winner, None, true)
18278 .await
18279 .unwrap()
18280 .expect("the third agent rewrites");
18281 assert!(msg.body.contains("`src/net.rs`"));
18282 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18283 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18284 }
18285
18286 #[tokio::test]
18287 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18288 crate::run::pin_test_home();
18289 for (agents, role, saved) in [
18291 (
18292 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18293 Some("beta"),
18294 true,
18295 ),
18296 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18297 (Vec::new(), None, false),
18298 ] {
18299 let dir = tempfile::tempdir().unwrap();
18300 let mut runner = rewrite_runner(dir.path(), true, agents);
18301 runner.state.config.roles.language_judge =
18302 role.map(|r| crate::config::AgentChoice::One(r.into()));
18303 let winner = runner.state.candidates[0].clone();
18304 let first = runner
18305 .guarded_pr_message(&winner, None, true)
18306 .await
18307 .unwrap();
18308 assert!(first.is_none(), "withheld text parks the run");
18309 assert!(runner.state.parked);
18310 assert_eq!(gate_questions(&runner), 1);
18311 let g = runner.state.github_text.as_ref().unwrap();
18312 assert_eq!(g.rewrite.is_some(), saved);
18313 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18314 }
18315 }
18316
18317 #[tokio::test]
18318 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18319 crate::run::pin_test_home();
18320 let dir = tempfile::tempdir().unwrap();
18321 let mut runner = runner_at(RunStatus::Gating);
18322 runner.state =
18323 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18324 runner.state.repo = dir.path().to_owned();
18325 runner.state.candidates[0].worktree = dir.path().to_owned();
18326 let mut author = spec("alpha");
18327 author.command = vec![
18328 "sh".into(),
18329 "-c".into(),
18330 "printf '%s' \"$REWRITE_REPLY\"".into(),
18331 ];
18332 author.env.insert(
18333 "REWRITE_REPLY".into(),
18334 format!(
18335 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18336 "English sentence. ".repeat(600)
18337 ),
18338 );
18339 runner.state.config.agents = vec![author];
18340 seed_public_summary(&mut runner);
18341 let winner = runner.state.candidates[0].clone();
18342 assert!(
18343 runner
18344 .guarded_pr_message(&winner, None, true)
18345 .await
18346 .unwrap()
18347 .is_none()
18348 );
18349 let record = runner.state.github_text.clone().unwrap();
18350 let rewrite = record.rewrite.clone().unwrap();
18351 assert_eq!(rewrite.failed, ["body-language"]);
18352 assert!(
18353 rewrite
18354 .body
18355 .ends_with(&format!("magi:run/{}", runner.state.id))
18356 );
18357 let store = ask::Questions::open();
18358 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18359 assert!(
18360 q.choices
18361 .contains(&crate::github_text::USE_REWRITE.to_owned())
18362 );
18363 for label in [
18364 "ORIGINAL",
18365 "FALLBACK",
18366 "REWRITE",
18367 "Rules still failed: body-language",
18368 "truncated",
18369 ] {
18370 assert!(q.detail.contains(label), "{label}");
18371 }
18372 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18373 &runner.state,
18374 &crate::github_text::artifact_name(&record.fingerprint),
18375 ))
18376 .unwrap();
18377 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18378 runner.state =
18380 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18381 store
18382 .update(&q.id, |q| {
18383 q.say("まだ日本語")?;
18384 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18385 })
18386 .unwrap();
18387 assert!(
18388 runner
18389 .guarded_pr_message(&winner, None, true)
18390 .await
18391 .unwrap()
18392 .is_none()
18393 );
18394 let retry = runner.state.github_text.as_ref().unwrap();
18395 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18396 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18397 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18398 assert!(
18399 q.posting_comparison
18400 .as_ref()
18401 .unwrap()
18402 .matches_detail(&q.detail)
18403 );
18404 store
18405 .update(&q.id, |q| {
18406 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18407 })
18408 .unwrap();
18409 let message = runner
18410 .guarded_pr_message(&winner, None, true)
18411 .await
18412 .unwrap()
18413 .unwrap();
18414 assert_eq!(message.title, rewrite.title);
18415 assert_eq!(message.body, rewrite.body);
18416 assert_eq!(
18417 runner.state.github_text.as_ref().unwrap().choice,
18418 Some(GithubTextChoice::Rewrite)
18419 );
18420 let again = runner
18421 .guarded_pr_message(&winner, None, true)
18422 .await
18423 .unwrap()
18424 .unwrap();
18425 assert_eq!(again.title, message.title);
18426 assert_eq!(again.body, message.body);
18427 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18428 assert!(
18429 !runner
18430 .state
18431 .events
18432 .iter()
18433 .any(|e| e.message.contains(&rewrite.title)
18434 || e.message.contains("English sentence."))
18435 );
18436 }
18437
18438 #[tokio::test]
18439 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18440 crate::run::pin_test_home();
18441 let dir = tempfile::tempdir().unwrap();
18442 let mut runner = runner_at(RunStatus::Gating);
18443 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18444 runner.state.repo = dir.path().to_owned();
18445 runner.state.candidates[0].worktree = dir.path().to_owned();
18446 seed_public_summary(&mut runner);
18447 let winner = runner.state.candidates[0].clone();
18448 assert!(
18449 runner
18450 .guarded_pr_message(&winner, None, true)
18451 .await
18452 .unwrap()
18453 .is_none()
18454 );
18455 let record = runner.state.github_text.as_ref().unwrap();
18456 assert!(record.rewrite.is_none());
18457 let mut old = serde_json::to_value(record).unwrap();
18459 old.as_object_mut().unwrap().remove("rewrite");
18460 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18461 assert!(old.rewrite.is_none());
18462 let id = record.question.clone().unwrap();
18463 ask::Questions::open()
18464 .update(&id, |q| {
18465 q.choices.push(crate::github_text::USE_REWRITE.into());
18466 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18467 })
18468 .unwrap();
18469 let message = runner
18470 .guarded_pr_message(&winner, None, true)
18471 .await
18472 .unwrap()
18473 .unwrap();
18474 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18475 assert_eq!(
18476 runner.state.github_text.as_ref().unwrap().choice,
18477 Some(GithubTextChoice::Fallback)
18478 );
18479 }
18480
18481 #[tokio::test]
18482 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18483 crate::run::pin_test_home();
18484 let dir = tempfile::tempdir().unwrap();
18485 let mut runner = runner_at(RunStatus::Gating);
18486 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18487 runner.state.repo = dir.path().to_owned();
18488 runner.state.candidates[0].worktree = dir.path().to_owned();
18489 seed_public_summary(&mut runner);
18490 let winner = runner.state.candidates[0].clone();
18491 assert!(
18492 runner
18493 .guarded_pr_message(&winner, None, true)
18494 .await
18495 .unwrap()
18496 .is_none()
18497 );
18498 let store = ask::Questions::open();
18499 let find = |run: &str| {
18500 store
18501 .list()
18502 .into_iter()
18503 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18504 .unwrap()
18505 };
18506 let run = runner.state.id.clone();
18507 let q = find(&run);
18508 let fp = runner
18509 .state
18510 .github_text
18511 .as_ref()
18512 .unwrap()
18513 .fingerprint
18514 .clone();
18515 let name = crate::github_text::artifact_name(&fp);
18516 let path = crate::run::artifact_path(&runner.state, &name);
18517 assert!(q.detail.contains(&path.display().to_string()));
18518 assert!(q.detail.contains("title-language: title"));
18519 let saved = std::fs::read_to_string(&path).expect("artifact written");
18520 assert!(saved.contains("日本語のタイトル"));
18521 assert!(q.cwd.is_none());
18522 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18523 let say = |id: &str, text: &str, choice: &str| {
18525 store
18526 .update(id, |q| {
18527 q.thread.push(ask::Turn {
18528 who: ask::Who::Operator,
18529 body: text.to_owned(),
18530 at: jiff::Timestamp::now(),
18531 note: None,
18532 });
18533 q.answer(ask::Answer::Choice(choice.to_owned()))
18534 })
18535 .unwrap();
18536 };
18537 say(&q.id, "まだ日本語", "use my text");
18538 assert!(
18539 runner
18540 .guarded_pr_message(&winner, None, true)
18541 .await
18542 .unwrap()
18543 .is_none()
18544 );
18545 let second = find(&run);
18546 assert_ne!(second.id, q.id);
18547 assert!(second.detail.contains("did not pass"));
18548 assert!(second.detail.contains("日本語のタイトル"));
18551 assert!(!second.detail.contains("まだ日本語"));
18552 assert!(second.detail.contains(&path.display().to_string()));
18553 say(&second.id, "fix: retry failed requests", "use my text");
18554 let message = runner
18555 .guarded_pr_message(&winner, None, true)
18556 .await
18557 .unwrap()
18558 .expect("a vetted replacement is posted");
18559 assert_eq!(message.title, "fix: retry failed requests");
18560 assert_eq!(
18561 runner
18562 .state
18563 .github_text
18564 .as_ref()
18565 .unwrap()
18566 .chosen_title
18567 .as_deref(),
18568 Some("fix: retry failed requests")
18569 );
18570 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18571 let again = runner
18573 .guarded_pr_message(&winner, None, true)
18574 .await
18575 .unwrap()
18576 .unwrap();
18577 assert_eq!(again.title, message.title);
18578 assert_eq!(
18579 store
18580 .list()
18581 .into_iter()
18582 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18583 .count(),
18584 2
18585 );
18586 }
18587
18588 #[test]
18589 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18590 let mut state = state_with_summary(
18594 "add retries",
18595 "TITLE: fix(web): batch reads\n- reads run.json once",
18596 );
18597 state.config.graph.language = "ja".to_owned();
18598 let m = pr_message(&state, 'A');
18599 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18600 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18601
18602 let task = "今回やってほしいこと: results projector を直す";
18605 let mut state = state_with_summary(task, "- no title line");
18606 state.config.graph.language = "ja".to_owned();
18607 let m = pr_message(&state, 'A');
18608 assert_eq!(
18609 m.title,
18610 format!("chore: land candidate A of run {}", state.id)
18611 );
18612 assert!(
18613 m.body.contains(&format!(
18614 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18615 )),
18616 "{}",
18617 m.body
18618 );
18619 }
18620
18621 #[test]
18622 fn pr_message_scrubs_home_paths_and_addresses() {
18623 let state = state_with_summary(
18624 "fix it in /Users/someone/src/x",
18625 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18626 );
18627 let m = pr_message(&state, 'A');
18628 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18629 assert!(!m.body.contains(leak), "{}", m.body);
18630 }
18631 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18632 }
18633
18634 #[test]
18635 fn pr_message_survives_a_task_that_closes_details() {
18636 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18637 let m = pr_message(&state, 'A');
18638 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18639 }
18640
18641 #[test]
18642 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18643 let cmd = manual_merge_command(
18644 MergeStyle::Squash,
18645 Path::new("/repo"),
18646 "b",
18647 "fix: \"quoted\" $(x) `y`\n\nbody",
18648 );
18649 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18650 }
18651
18652 #[test]
18653 fn manual_merge_command_matches_the_configured_style() {
18654 let repo = Path::new("/repo");
18655 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18656
18657 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18658 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18659
18660 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18661 assert_eq!(
18662 squash,
18663 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18664 \"Merge magi run 0832 (candidate A)\""
18665 );
18666
18667 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18668 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18669 }
18670
18671 #[test]
18672 fn a_nudge_gets_a_quarter_of_the_budget() {
18673 assert_eq!(retry_budget(secs(1200), true), secs(300));
18675 assert_eq!(retry_budget(secs(3600), true), secs(900));
18676 }
18677
18678 #[test]
18679 fn a_resent_prompt_keeps_the_whole_budget() {
18680 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18683 assert_eq!(retry_budget(secs(60), false), secs(60));
18684 }
18685
18686 #[test]
18687 fn the_floor_never_exceeds_the_original_budget() {
18688 assert_eq!(retry_budget(secs(60), true), secs(60));
18692 assert_eq!(retry_budget(secs(480), true), secs(120));
18693 assert_eq!(retry_budget(secs(0), true), secs(0));
18694 }
18695
18696 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18697 agent::CommandEvidence {
18698 id: "item1".to_owned(),
18699 description: "cargo test".to_owned(),
18700 exit_code,
18701 result_summary: String::new(),
18702 source: "codex".to_owned(),
18703 }
18704 }
18705
18706 #[test]
18707 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18708 assert!(!has_unconfirmed_command(&[]));
18712 }
18713
18714 #[test]
18715 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18716 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18720 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18721 assert!(!has_unconfirmed_command(&[
18722 evidence(Some(0)),
18723 evidence(Some(101))
18724 ]));
18725 }
18726
18727 #[test]
18728 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18729 assert!(has_unconfirmed_command(&[
18730 evidence(Some(0)),
18731 evidence(None)
18732 ]));
18733 }
18734
18735 #[test]
18736 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18737 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18738 assert_eq!(
18739 verified_noop_claim(true, &[], text).as_deref(),
18740 Some("already fixed by b32cfc4, on main.")
18741 );
18742 }
18743
18744 #[test]
18745 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18746 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18749 assert!(verified_noop_claim(false, &[], text).is_none());
18750 }
18751
18752 #[test]
18753 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18754 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18755 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18756 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18758 }
18759
18760 #[test]
18761 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18762 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18763 }
18764
18765 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18768 runner.state.candidates = shape
18769 .iter()
18770 .enumerate()
18771 .map(|(i, &(empty, verified))| Candidate {
18772 index: i,
18773 label: (b'A' + i as u8) as char,
18774 agent: "sonnet".to_owned(),
18775 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18776 worktree: PathBuf::from(format!("/wt/{i}")),
18777 summary: String::new(),
18778 stat: String::new(),
18779 files: 0,
18780 commits: 0,
18781 empty,
18782 failed: None,
18783 verified_noop: verified.map(str::to_owned),
18784 duration_ms: 0,
18785 folded: false,
18786 })
18787 .collect();
18788 }
18789
18790 #[test]
18791 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18792 ask_test_home();
18793 let mut runner = runner_at(RunStatus::Implementing);
18794 set_candidates(
18795 &mut runner,
18796 &[
18797 (true, Some("already on main at b32cfc4")),
18798 (true, Some("same fix, see the existing test")),
18799 ],
18800 );
18801
18802 runner
18803 .after_implement()
18804 .expect("a verified no-op is not an error");
18805
18806 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18807 }
18808
18809 #[test]
18810 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18811 ask_test_home();
18812 let mut runner = runner_at(RunStatus::Implementing);
18813 set_candidates(
18817 &mut runner,
18818 &[(true, Some("already on main at b32cfc4")), (true, None)],
18819 );
18820
18821 let err = runner
18822 .after_implement()
18823 .expect_err("an unverified empty candidate must still fail the run");
18824
18825 assert!(
18826 err.to_string().contains("no candidate produced a change"),
18827 "{err}"
18828 );
18829 assert_eq!(runner.state.status, RunStatus::Failed);
18830 }
18831
18832 #[test]
18833 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18834 ask_test_home();
18835 let mut runner = runner_at(RunStatus::Implementing);
18836 set_candidates(&mut runner, &[(true, None), (true, None)]);
18837
18838 let err = runner
18839 .after_implement()
18840 .expect_err("no candidate declared anything; this is an ordinary failure");
18841
18842 assert!(
18843 err.to_string().contains("no candidate produced a change"),
18844 "{err}"
18845 );
18846 assert_eq!(runner.state.status, RunStatus::Failed);
18847 }
18848}
18849
18850#[derive(Debug)]
18853struct GraftRestoreFailed(String);
18854
18855impl std::fmt::Display for GraftRestoreFailed {
18856 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18857 write!(
18858 f,
18859 "could not restore the winner to its pre-graft commit: {}",
18860 self.0
18861 )
18862 }
18863}
18864
18865impl std::error::Error for GraftRestoreFailed {}
18866
18867#[cfg(test)]
18871mod resource_wait_tests {
18872 use super::*;
18873 use std::collections::BTreeMap;
18874
18875 fn sh(id: &str, script: &str) -> AgentSpec {
18876 AgentSpec {
18877 id: id.to_owned(),
18878 kind: crate::config::AgentKind::Command,
18879 model: None,
18880 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18881 extra_args: Vec::new(),
18882 env: BTreeMap::new(),
18883 prompt_delivery: None,
18884 max_turn_secs: None,
18885 }
18886 }
18887
18888 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18889 SeatJob {
18890 seat: SeatState::new("impl-A", &spec.id, 1),
18891 spec,
18892 cwd: dir.to_path_buf(),
18893 prompt: "do it".to_owned(),
18894 timeout,
18895 allow_write: true,
18896 sessions: false,
18897 artifacts: dir.join("art"),
18898 stem: "impl-A".to_owned(),
18899 handover: None,
18900 }
18901 }
18902
18903 fn state(resource_wait_secs: u64) -> RunState {
18904 let mut config = Config::default();
18905 config.graph.timeout_resource_wait = resource_wait_secs;
18906 RunState::new(
18907 PathBuf::from("/repo"),
18908 "main".to_owned(),
18909 "abc1234".to_owned(),
18910 "task".to_owned(),
18911 config,
18912 )
18913 }
18914
18915 fn hold(cache: &Path) -> crate::cache::Guard {
18916 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18917 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18918 crate::cache::AcquireOutcome::Acquired(g) => g,
18919 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18920 }
18921 }
18922
18923 async fn run(
18924 jobs: Vec<SeatJob>,
18925 sem: Arc<Semaphore>,
18926 cache: Option<&Path>,
18927 st: &mut RunState,
18928 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18929 let prompts = Prompts::default();
18930 let ctx = WaveCtx {
18931 run: "r",
18932 node: "implement",
18933 prompts: &prompts,
18934 cache,
18935 round: None,
18936 carry_seats: false,
18937 pauses: None,
18938 };
18939 wave(jobs, sem, &ctx, st, 0).await
18940 }
18941
18942 #[tokio::test]
18943 async fn verification_cache_wait_preserves_the_command_budget() {
18944 crate::run::pin_test_home();
18945 let tmp = tempfile::tempdir().unwrap();
18946 let cache = tmp.path().join("target");
18947 let guard = hold(&cache);
18948 let release = tokio::spawn(async move {
18949 tokio::time::sleep(Duration::from_millis(100)).await;
18950 guard.release();
18951 });
18952 let mut st = state(10);
18953 let budget = Duration::from_millis(50);
18954 let wait = repository_wait(&st, None);
18955 let (outcomes, _) = with_cache_lease(
18956 &mut st,
18957 Some(&cache),
18958 "gate",
18959 "gate",
18960 tmp.path(),
18961 "abc1234",
18962 budget,
18963 "test gate",
18964 &wait,
18965 |_, available| async move {
18966 assert_eq!(
18967 available, budget,
18968 "contention must not shorten verification"
18969 );
18970 (Vec::new(), false, Vec::new())
18971 },
18972 )
18973 .await
18974 .unwrap();
18975 release.await.unwrap();
18976 assert!(outcomes.is_empty(), "{outcomes:?}");
18977 assert!(
18978 st.events
18979 .iter()
18980 .any(|event| event.message.contains("waiting for"))
18981 );
18982 assert!(st.resource_wait.is_none());
18983 hold(&cache).release();
18985 }
18986
18987 #[tokio::test]
18988 async fn verification_cache_wait_expires_without_starting_commands() {
18989 crate::run::pin_test_home();
18990 let tmp = tempfile::tempdir().unwrap();
18991 let cache = tmp.path().join("target");
18992 let _held = hold(&cache);
18993 let mut st = state(0);
18994 let wait = repository_wait(&st, None);
18995 let (outcomes, _) = with_cache_lease(
18996 &mut st,
18997 Some(&cache),
18998 "gate",
18999 "gate",
19000 tmp.path(),
19001 "abc1234",
19002 Duration::from_secs(60),
19003 "test gate",
19004 &wait,
19005 |_, _| async { panic!("commands must not start without the cache lease") },
19006 )
19007 .await
19008 .unwrap();
19009 assert_eq!(outcomes.len(), 1);
19010 assert!(outcomes[0].resource_blocked);
19011 assert!(st.resource_wait.is_none());
19012 }
19013
19014 #[tokio::test]
19015 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19016 crate::run::pin_test_home();
19017 let tmp = tempfile::tempdir().unwrap();
19018 let cache = tmp.path().join("target-a");
19019 let guard = hold(&cache);
19020 tokio::spawn(async move {
19022 tokio::time::sleep(Duration::from_millis(1500)).await;
19023 guard.release();
19024 });
19025 let mut st = state(60);
19026 let results = run(
19027 vec![job(
19028 sh("alpha", "echo ok"),
19029 tmp.path(),
19030 Duration::from_secs(1),
19031 )],
19032 Arc::new(Semaphore::new(1)),
19033 Some(&cache),
19034 &mut st,
19035 )
19036 .await;
19037 assert!(
19038 matches!(results[0].2, AgentOutcome::Ok(_)),
19039 "the wait must not be charged to the agent"
19040 );
19041 assert!(st.resource_block.is_none());
19042 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19043 assert!(st.resource_wait.is_none() && st.active.is_empty());
19044 }
19045
19046 #[tokio::test]
19047 #[cfg(unix)]
19048 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19049 use std::os::fd::AsRawFd as _;
19050 crate::run::pin_test_home();
19051 let tmp = tempfile::tempdir().unwrap();
19052 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19053 let common = tmp.path().join(".git");
19054 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19055 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19056 assert_eq!(
19057 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19058 0
19059 );
19060 let mut st = state(1);
19061 st.config.coordination = crate::config::test_coordination();
19062 let mut reader = job(
19063 sh("reader", "echo must-not-run"),
19064 tmp.path(),
19065 Duration::from_secs(30),
19066 );
19067 reader.allow_write = false;
19068 let results = run(
19069 vec![reader.clone()],
19070 Arc::new(Semaphore::new(1)),
19071 None,
19072 &mut st,
19073 )
19074 .await;
19075 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19076 assert_eq!(results[0].1.turns, 0);
19077 assert!(FailClass::of(&results[0].2).is_none());
19078 assert_eq!(
19079 st.resource_block.as_ref().unwrap().kind,
19080 "repository coordination"
19081 );
19082 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19083 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19084 drop(external);
19085 let editor = crate::repo_lock::SeatLease::for_checkout(
19086 tmp.path(),
19087 true,
19088 crate::config::test_coordination().as_ref(),
19089 )
19090 .await
19091 .unwrap();
19092 let pause = Pause::new();
19093 let interrupt = Pause::new();
19094 let prompts = Prompts::default();
19095 let ctx = WaveCtx {
19096 run: "r",
19097 node: "review",
19098 prompts: &prompts,
19099 cache: None,
19100 round: None,
19101 carry_seats: false,
19102 pauses: Some((&pause, &interrupt)),
19103 };
19104 let (results, ()) = tokio::join!(
19105 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19106 async {
19107 tokio::time::sleep(Duration::from_millis(50)).await;
19108 pause.park();
19109 }
19110 );
19111 assert!(
19112 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19113 );
19114 assert_eq!(results[0].1.turns, 0);
19115 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19116 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19117 drop(editor);
19118 }
19119
19120 #[tokio::test]
19121 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19122 crate::run::pin_test_home();
19123 let tmp = tempfile::tempdir().unwrap();
19124 let cache = tmp.path().join("target-b");
19125 let _held = hold(&cache);
19126 let mut st = state(1);
19127 let results = run(
19128 vec![job(
19129 sh("alpha", "echo ok"),
19130 tmp.path(),
19131 Duration::from_secs(1),
19132 )],
19133 Arc::new(Semaphore::new(1)),
19134 Some(&cache),
19135 &mut st,
19136 )
19137 .await;
19138 let (_, seat, out) = &results[0];
19139 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19140 assert!(wave_blocked(&results));
19141 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19142 assert_eq!(seat.turns, 0, "nothing was asked");
19143 assert!(
19144 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19145 "no agent was spawned"
19146 );
19147 assert!(FailClass::of(out).is_none(), "not a failure class");
19148 let block = st.resource_block.as_ref().expect("recorded for the resume");
19149 assert_eq!(block.kind, "shared build cache");
19150 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19151 assert!(st.active.is_empty());
19152 }
19153
19154 #[tokio::test]
19155 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19156 crate::run::pin_test_home();
19157 let tmp = tempfile::tempdir().unwrap();
19158 let mut st = state(1);
19159 let results = run(
19160 vec![job(
19161 sh("alpha", "echo ok"),
19162 tmp.path(),
19163 Duration::from_secs(1),
19164 )],
19165 Arc::new(Semaphore::new(0)),
19166 None,
19167 &mut st,
19168 )
19169 .await;
19170 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19171 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19172 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19173 }
19174
19175 #[tokio::test]
19176 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19177 crate::run::pin_test_home();
19178 let tmp = tempfile::tempdir().unwrap();
19179 let mut st = state(60);
19180 let results = run(
19181 vec![job(
19182 sh("alpha", "sleep 30"),
19183 tmp.path(),
19184 Duration::from_secs(1),
19185 )],
19186 Arc::new(Semaphore::new(1)),
19187 None,
19188 &mut st,
19189 )
19190 .await;
19191 let out = &results[0].2;
19192 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19193 let class = FailClass::of(out).expect("a failure");
19194 assert_eq!(class, FailClass::Timeout);
19195 assert!(should_hand_over(None, &class), "existing handover policy");
19196 assert!(st.resource_block.is_none());
19197 }
19198}