1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 let worktree = state.worktree_root().join("under-review");
1033 if let Some(parent) = worktree.parent() {
1034 tokio::fs::create_dir_all(parent).await.ok();
1035 }
1036 let path = worktree.to_string_lossy().to_string();
1037 git::git(repo, &["worktree", "add", &path, branch])
1038 .await
1039 .with_context(|| {
1040 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1041 })?;
1042
1043 let setup = match ensure_setup_config(&mut state, repo).await {
1044 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1045 Err(e) => Err(e),
1046 };
1047 if let Err(e) = setup {
1048 git::worktree_remove_clean(repo, &worktree).await.ok();
1049 return Err(e);
1050 }
1051 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1052 .await
1053 .unwrap_or(0);
1054 if commits == 0 {
1055 git::worktree_remove_clean(repo, &worktree).await.ok();
1056 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1057 }
1058 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1059 .await
1060 .map(|f| f.len())
1061 .unwrap_or(0);
1062 if files == 0
1063 && let (Ok(head_tree), Ok(base_tree)) = (
1064 git::tree_of(&worktree, "HEAD").await,
1065 git::tree_of(&worktree, &base_commit).await,
1066 )
1067 && head_tree == base_tree
1068 {
1069 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1070 git::worktree_remove_clean(repo, &worktree).await.ok();
1071 bail!(
1072 "`{branch}` at {} has a tree identical to base {}; this usually means \
1073 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1074 against `{}/{branch}`) rather than an empty change",
1075 short(&head),
1076 short(&base_commit),
1077 state.config.merge.remote
1078 );
1079 }
1080 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1081 .await
1082 .unwrap_or_default();
1083
1084 state.candidates.push(Candidate {
1085 index: 0,
1086 label: 'A',
1087 agent: EXISTING_BRANCH.to_owned(),
1090 branch: branch.to_owned(),
1091 worktree,
1092 summary: String::new(),
1093 stat,
1094 files,
1095 commits,
1096 empty: false,
1097 failed: None,
1098 verified_noop: None,
1099 duration_ms: 0,
1100 folded: false,
1101 });
1102 state.tally = Some(Tally {
1103 first_choice: BTreeMap::from([('A', 0)]),
1104 borda: BTreeMap::new(),
1105 winner: 'A',
1106 rankings: 0,
1107 unanimous_initial: false,
1108 deliberated: false,
1109 changed_votes: 0,
1110 unanimous_final: false,
1111 tie_break: None,
1112 judges: 0,
1116 present: 0,
1117 quorum: 0,
1118 met_quorum: true,
1119 uncontested: Some("review-only run: nothing competed".to_owned()),
1120 });
1121 state.status = RunStatus::Reviewing;
1122 state.prep_complete = Some(true);
1123 state.event(
1124 "start",
1125 format!(
1126 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1127 state.id
1128 ),
1129 );
1130 state.save()?;
1131 Ok(Self {
1132 state,
1133 roles,
1134 sem: Arc::new(Semaphore::new(max_parallel)),
1135 pause: Pause::new(),
1136 interrupt: Pause::new(),
1137 })
1138 }
1139
1140 pub fn resume(id: &str) -> Result<Self> {
1142 let state = RunState::load(id)?;
1143 crate::pr_head::ensure_owner(&state)?;
1144 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1145 bail!(
1146 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1147 );
1148 }
1149 if let Some(to) = &state.released_to {
1150 bail!(
1151 "run {} cannot be resumed: its worktree was released to run {}",
1152 state.short(),
1153 crate::run::short_of(to)
1154 );
1155 }
1156 let roles = state.config.resolve_roles()?;
1157 if let Some(id) = &state.config.graph.selected_implementer
1158 && state.candidates.iter().any(|c| c.agent != *id)
1159 {
1160 bail!(
1161 "replacement implementer pin does not match the saved candidates; task must stay held"
1162 );
1163 }
1164 let max_parallel = state.config.graph.max_parallel.max(1);
1165 Ok(Self {
1166 state,
1167 roles,
1168 sem: Arc::new(Semaphore::new(max_parallel)),
1169 pause: Pause::new(),
1170 interrupt: Pause::new(),
1171 })
1172 }
1173
1174 pub async fn execute(&mut self) -> Result<()> {
1181 if let Some(selection) = &self.state.pr_recovery
1182 && (selection.source_proof.is_some() || selection.source_run.is_some())
1183 {
1184 Box::pin(selection.verify_source(&self.state.repo)).await?;
1185 }
1186 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1189 || self.state.land_approval.is_some()
1190 || self.state.merge_decision.is_some()
1191 || !self.state.merge_repairs.is_empty()
1192 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1193 {
1194 let claim = FixClaim::acquire(&self.state.dir())?;
1195 let disk = RunState::load(&self.state.id)?;
1196 let checkpoint = |s: &RunState| {
1197 (
1198 s.merge_repairs.len(),
1199 s.merge_repairs
1200 .last()
1201 .map(|r| (r.question.clone(), r.phase)),
1202 s.land_approval
1203 .as_ref()
1204 .map(|a| (a.question.clone(), a.head.clone())),
1205 )
1206 };
1207 if checkpoint(&disk) != checkpoint(&self.state) {
1208 bail!(
1209 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1210 );
1211 }
1212 Some(claim)
1213 } else {
1214 None
1215 };
1216 let mut execution = self.claim_driver()?;
1217 let runner = &mut *execution.runner;
1218 let wait = runner.repository_wait();
1219 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1222 let result = match result {
1223 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1224 record_repository_block(&mut runner.state, &e);
1225 runner.park_here()?;
1226 Ok(())
1227 }
1228 other => other,
1229 };
1230 execution.driver.finish(&mut runner.state)?;
1231 let ended = if result.is_err() {
1232 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1233 } else {
1234 crate::notices::run_ended(&runner.state)
1235 };
1236 if let Some(notice) = ended {
1237 crate::notices::raise(notice);
1238 }
1239 result
1240 }
1241
1242 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1243 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1244 }
1245
1246 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1247 let wait = self.repository_wait();
1248 coordinate_operation(&mut self.state, cwd, &wait).await
1249 }
1250
1251 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1253 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1254 Ok(owner) => owner,
1255 Err(error) => {
1256 let detail = format!("worktree maintenance or ownership update: {error:#}");
1257 self.state.resource_block = Some(crate::run::ResourceBlock {
1261 node: "start".into(),
1262 kind: "worktree maintenance".into(),
1263 detail: detail.clone(),
1264 at: Timestamp::now(),
1265 });
1266 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1267 }
1268 };
1269 let disk = RunState::load(&self.state.id)?;
1270 crate::pr_head::ensure_owner(&disk)?;
1271 if disk.liveness(false) == crate::run::Liveness::Live {
1272 bail!("run already has a live driver; wait for its execution to end");
1273 }
1274 if let Some(to) = &disk.released_to {
1275 bail!(
1276 "run {} cannot continue: its worktree was released to run {}",
1277 self.state.short(),
1278 crate::run::short_of(to)
1279 );
1280 }
1281 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1282 let mut execution = DriverExecution {
1283 runner: self,
1284 driver,
1285 };
1286 let _owner = owner;
1289 let state = &mut execution.runner.state;
1290 state.parked = false;
1295 state.resource_wait = None;
1296 state.resource_block = None;
1297 state.clear_active();
1304 let pid = std::process::id();
1319 state.driver_pid = Some(pid);
1320 state.driver_started_at = crate::proc::process_started_at(pid);
1321 state.driver_exited = false;
1322 state.begin_pass();
1323 execution.driver.bind(state);
1324 state.save()?;
1325 execution.driver.record(state)?;
1326 Ok(execution)
1327 }
1328
1329 async fn execute_graph(&mut self) -> Result<()> {
1330 self.state.configure_recovery_resume()?;
1331 self.persist_candidates().await?;
1334 if self
1335 .state
1336 .merge_decision
1337 .as_ref()
1338 .is_some_and(|decision| !decision.closed)
1339 && self.state.status != RunStatus::Landing
1340 {
1341 if crate::merge_decision::resumable_hold(&self.state) {
1342 self.state.merge = Some(MergeOutcome {
1343 mode: MergeMode::Pr,
1344 ok: false,
1345 detail: self
1346 .state
1347 .merge_decision
1348 .as_ref()
1349 .unwrap()
1350 .target
1351 .pr
1352 .clone(),
1353 empty: false,
1354 });
1355 self.run_land().await?;
1356 self.settle_questions();
1357 return Ok(());
1358 }
1359 bail!(
1360 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1361 );
1362 }
1363 if self.state.config.graph.selected_implementer.is_some()
1364 && self.state.status == RunStatus::Blocked
1365 && self.state.tally.is_none()
1366 && self.state.candidate_preservation.is_empty()
1367 && self.state.interrupted_implementations.is_empty()
1368 {
1369 bail!(
1370 "selected implementation is held; explicitly release the owning task to retry it"
1371 );
1372 }
1373
1374 if self.state.merge_repairs.last().is_some_and(|r| {
1375 matches!(
1376 r.phase,
1377 crate::run::MergeRepairPhase::Requested
1378 | crate::run::MergeRepairPhase::Fixing
1379 | crate::run::MergeRepairPhase::Reviewing
1380 )
1381 }) {
1382 if self.repair_merge_review().await? {
1383 self.run_land().await?;
1384 }
1385 self.settle_questions();
1386 return Ok(());
1387 }
1388 if self.state.pr_publication_pending {
1391 self.merge().await?;
1392 if self.resource_halt()? {
1393 return Ok(());
1394 }
1395 self.state.save()?;
1396 self.settle_questions();
1397 return Ok(());
1398 }
1399 if self.state.status == RunStatus::Stalled {
1412 if self.recover_stall().await? {
1413 self.finish_after_tally().await?;
1414 } else {
1415 self.state.save()?;
1417 }
1418 return Ok(());
1419 }
1420 if self.state.status == RunStatus::Landing
1430 || crate::handover::landing_checkpoint(&self.state)
1431 {
1432 self.state.land_sync_pending = true;
1433 self.state.save()?;
1434 self.run_land().await?;
1435 self.settle_questions();
1440 return Ok(());
1441 }
1442 if self.state.github_text.is_some()
1447 && self.state.merge.is_none()
1448 && self.state.status == RunStatus::Gating
1449 {
1450 self.merge().await?;
1451 if self.resource_halt()? {
1452 return Ok(());
1453 }
1454 self.state.save()?;
1455 return Ok(());
1456 }
1457 if let Err(e) = self.prep().await {
1458 if e.is::<crate::repo_lock::ResourceBlocked>() {
1459 return Err(e);
1460 }
1461 self.state.status = RunStatus::Blocked;
1462 self.state
1463 .prep_failure
1464 .get_or_insert_with(|| format!("{e:#}"));
1465 self.state
1466 .event("prep", format!("preparation stopped: {e:#}"));
1467 self.state.save()?;
1468 return Err(e);
1469 }
1470 if self.park_here()? {
1471 return Ok(());
1472 }
1473 self.advise().await?;
1474 if self.park_here()? {
1475 return Ok(());
1476 }
1477 self.implement().await?;
1478 if self.park_here()? {
1479 return Ok(());
1480 }
1481 if self.state.status == RunStatus::VerifiedNoop {
1485 return Ok(());
1486 }
1487 self.judge().await?;
1488 if self.park_here()? {
1489 return Ok(());
1490 }
1491 self.deliberate().await?;
1492 if self.park_here()? {
1493 return Ok(());
1494 }
1495 self.vote().await?;
1496 if self.park_here()? {
1497 return Ok(());
1498 }
1499 self.tally()?;
1500 if self.state.status == RunStatus::Stalled {
1505 self.state.save()?;
1509 return Ok(());
1510 }
1511 self.finish_after_tally().await?;
1512 Ok(())
1513 }
1514
1515 fn stop_for_resource(&mut self) -> Result<()> {
1520 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1522 {
1523 self.state.status = RunStatus::Blocked;
1524 }
1525 self.state.save()
1526 }
1527
1528 fn resource_halt(&mut self) -> Result<bool> {
1530 if self.state.resource_block.is_none() {
1531 return Ok(false);
1532 }
1533 self.stop_for_resource()?;
1534 Ok(true)
1535 }
1536
1537 fn park_here(&mut self) -> Result<bool> {
1544 let parking = self.pause.parked() || self.interrupt.parked();
1545 if !parking && self.state.resource_block.is_some() {
1546 self.stop_for_resource()?;
1547 return Ok(true);
1548 }
1549 if !self.pause.parked() && !self.interrupt.parked() {
1554 return Ok(false);
1555 }
1556 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1557 let why = match cause.as_ref().map(ParkCause::label) {
1558 Some(reason) => format!(
1559 "parked after `{}` ({reason}) — resume to carry on from here",
1560 self.state.status.as_str()
1561 ),
1562 None => format!(
1563 "parked after `{}` — resume to carry on from here",
1564 self.state.status.as_str()
1565 ),
1566 };
1567 self.state.event("park", why);
1568 self.state
1569 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1570 self.state.parked = true;
1571 self.state.save()?;
1572 Ok(true)
1573 }
1574
1575 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1577 if self.state.prep_failure.is_none() {
1578 return Ok(());
1579 }
1580 let failed_at = self
1581 .state
1582 .events
1583 .iter()
1584 .rev()
1585 .find(|event| event.node == "prep")
1586 .map_or(self.state.updated_at, |event| event.at);
1587 anyhow::ensure!(
1588 approval_at > failed_at,
1589 "preparation retry requires a new explicit release after the failure"
1590 );
1591 self.state.prep_failure = None;
1592 self.state.prep_complete = Some(false);
1593 self.state
1594 .event("prep-retry", "operator approved retry of preparation");
1595 Ok(())
1596 }
1597
1598 pub fn on_pause(&mut self, pause: Pause) {
1600 self.pause = pause;
1601 }
1602
1603 pub fn watch_interrupt(&mut self, pause: Pause) {
1609 self.interrupt = pause;
1610 }
1611
1612 fn settle_questions(&mut self) {
1632 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1633 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1634 }
1635 }
1636
1637 async fn finish_after_tally(&mut self) -> Result<()> {
1640 self.graft().await?;
1643 if self.resource_halt()? {
1644 return Ok(());
1645 }
1646 self.fold_losers().await?;
1647 self.sync_to_base().await?;
1652 if self.resource_halt()? {
1653 return Ok(());
1654 }
1655 if self.state.status == RunStatus::AlreadyInBase {
1656 return Ok(());
1657 }
1658 self.review_loop().await?;
1659 if self.resource_halt()? {
1660 return Ok(());
1661 }
1662 self.sync_to_base().await?;
1663 if self.resource_halt()? {
1664 return Ok(());
1665 }
1666 if self.state.status == RunStatus::AlreadyInBase {
1667 return Ok(());
1668 }
1669 self.gate().await?;
1670 self.merge().await?;
1671 if self.resource_halt()? {
1672 return Ok(());
1673 }
1674 self.state.save()?;
1675 Ok(())
1676 }
1677
1678 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1685 let result = match ensure_setup_config(&mut self.state, repo).await {
1686 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1687 Err(e) => Err(e),
1688 };
1689 if let Err(e) = result {
1690 git::worktree_remove_clean(repo, wt).await.ok();
1691 if in_prep {
1692 self.rollback_prep(repo).await;
1693 }
1694 self.state.status = RunStatus::Blocked;
1695 self.state.event(
1696 "setup",
1697 format!("worktree setup failed in {}: {e:#}", wt.display()),
1698 );
1699 self.state.save()?;
1700 return Err(e);
1701 }
1702 Ok(())
1703 }
1704
1705 async fn rollback_prep(&mut self, repo: &Path) {
1707 let root = self.state.worktree_root();
1708 for c in std::mem::take(&mut self.state.candidates) {
1709 if matches!(
1710 git::worktree_remove_clean(repo, &c.worktree).await,
1711 Ok(true)
1712 ) {
1713 git::branch_delete(repo, &c.branch).await.ok();
1714 } else {
1715 self.state.candidates.push(c);
1716 }
1717 }
1718 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1719 for j in 1..=seats {
1720 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1721 .await
1722 .ok();
1723 }
1724 for k in 1..=self.state.config.graph.advisors {
1725 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1726 .await
1727 .ok();
1728 }
1729 }
1730
1731 async fn prep(&mut self) -> Result<()> {
1732 if let Some(reason) = &self.state.prep_failure {
1733 bail!(
1734 "candidate preparation previously failed: {reason}; correct the source before retrying"
1735 );
1736 }
1737 if self
1738 .state
1739 .prep_complete
1740 .unwrap_or(!self.state.candidates.is_empty())
1741 {
1742 return Ok(());
1743 }
1744 let repo = self.state.repo.clone();
1745 let _coordination = self.coordinate(&repo).await?;
1748 let base = self.state.base_commit.clone();
1749 let plan = self.state.sources.plan(&repo, &base).await?;
1750 self.rollback_prep(&repo).await;
1751 if !self.state.candidates.is_empty() {
1752 bail!(
1753 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1754 );
1755 }
1756 self.state.prep_complete = Some(false);
1757 self.state.status = RunStatus::Prep;
1758 self.state.save()?;
1759 let repo = self.state.repo.clone();
1760 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1761 let root = self.state.worktree_root();
1762 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1763
1764 let hooks_dir = self.state.dir().join("hooks");
1767 if self.state.config.blind.commit_msg_hook {
1768 std::fs::create_dir_all(&hooks_dir)
1769 .with_context(|| format!("create {}", hooks_dir.display()))?;
1770 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1771 let path = hooks_dir.join("commit-msg");
1772 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1773 make_executable(&path)?;
1774 git::acquire_worktree_config(&repo).await?;
1782 self.state.enabled_worktree_config = true;
1783 }
1784
1785 ensure_setup_config(&mut self.state, &repo).await?;
1788
1789 for (index, (spec, label)) in self
1790 .roles
1791 .implementers
1792 .clone()
1793 .into_iter()
1794 .zip(labels)
1795 .enumerate()
1796 {
1797 let branch = self.state.branch_for(label);
1798 let worktree = root.join(format!("cand-{label}"));
1799 self.state.event(
1803 "prep-intent",
1804 serde_json::json!({
1805 "branch": branch,
1806 "worktree": worktree,
1807 "start": start,
1808 "picks": plan.picks,
1809 })
1810 .to_string(),
1811 );
1812 self.state.save()?;
1813 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1814 if self.state.config.blind.commit_msg_hook {
1815 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1816 }
1817 git::local_exclude(&worktree, "/.magi/").await?;
1818 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1819 git::worktree_remove_clean(&repo, &worktree).await.ok();
1822 git::branch_delete(&repo, &branch).await.ok();
1823 self.rollback_prep(&repo).await;
1824 self.state.status = RunStatus::Blocked;
1825 self.state
1826 .event("prep", format!("worktree setup failed: {e:#}"));
1827 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1828 self.state.save()?;
1829 return Err(e);
1830 }
1831 self.state.candidates.push(Candidate {
1832 index,
1833 label,
1834 agent: spec.id.clone(),
1835 branch,
1836 worktree,
1837 summary: String::new(),
1838 stat: String::new(),
1839 files: 0,
1840 commits: 0,
1841 empty: false,
1842 failed: None,
1843 verified_noop: None,
1844 duration_ms: 0,
1845 folded: false,
1846 });
1847 self.state.save()?;
1848 for pick in &plan.picks {
1849 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1850 {
1851 self.state.status = RunStatus::Blocked;
1852 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1853 self.state
1854 .event("prep", format!("cannot apply referenced commit: {e}"));
1855 self.state.save()?;
1856 return Err(e);
1857 }
1858 }
1859 }
1860
1861 for j in 1..=self.roles.judges.len() {
1862 let wt = root.join(format!("judge-{j}"));
1863 if !wt.exists() {
1864 git::worktree_add_detached(&repo, &wt, &base).await?;
1865 self.setup_seat_worktree(&repo, &wt, true).await?;
1866 }
1867 }
1868
1869 if self.state.config.graph.advise {
1877 for k in 1..=self.state.config.graph.advisors {
1878 let wt = root.join(format!("advisor-{k}"));
1879 if !wt.exists() {
1880 git::worktree_add_detached(&repo, &wt, &base).await?;
1881 self.setup_seat_worktree(&repo, &wt, true).await?;
1882 }
1883 }
1884 }
1885
1886 let authors: Vec<&str> = self
1891 .roles
1892 .implementers
1893 .iter()
1894 .map(|a| a.id.as_str())
1895 .collect();
1896 let overlap: Vec<String> = self
1897 .roles
1898 .judges
1899 .iter()
1900 .enumerate()
1901 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1902 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1903 .collect();
1904 if !overlap.is_empty() {
1905 let note = format!(
1906 "{} also authored a candidate; blind, but the panel is less \
1907 independent than {} distinct agents would be",
1908 overlap.join(", "),
1909 self.roles.judges.len()
1910 );
1911 self.state.event("prep", note);
1912 }
1913
1914 self.state.event(
1915 "prep",
1916 format!(
1917 "{} candidates, {} judges, base {} ({})",
1918 self.state.candidates.len(),
1919 self.roles.judges.len(),
1920 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1921 self.state.base_branch
1922 ),
1923 );
1924 self.state.prep_complete = Some(true);
1925 self.state.status = RunStatus::Implementing;
1926 self.state.save()?;
1927 Ok(())
1928 }
1929
1930 async fn advise(&mut self) -> Result<()> {
1963 let implement_untouched = self
1964 .state
1965 .candidates
1966 .iter()
1967 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1968 if !self.state.config.graph.advise || self.state.advise_attempted {
1969 return Ok(());
1970 }
1971 if !implement_untouched {
1972 self.state.event(
1973 "advise",
1974 "skipping the design-deliberation stage: at least one \
1975 candidate already shows implementation progress, so this \
1976 run is past the point the stage exists to run before"
1977 .to_owned(),
1978 );
1979 self.state.advise_attempted = true;
1980 self.state.save()?;
1981 return Ok(());
1982 }
1983 let run_id = self.state.id.clone();
1984 let prompts = self.state.config.prompts.clone();
1985 let instruction = self.state.instruction.clone();
1986 let language = self.state.config.graph.language.clone();
1987 let root = self.state.worktree_root();
1988 let n = self.state.config.graph.advisors;
1989 let where_recorded = self.state.dir().join("run.json");
1990
1991 let seats = match self.state.config.advisors() {
1992 Ok(seats) if !seats.is_empty() => seats,
1993 Ok(_) => {
1994 self.state.event(
1995 "advise",
1996 format!(
1997 "[graph] advisors is 0; skipping the design-deliberation \
1998 stage and continuing without a synthesis brief (see {})",
1999 where_recorded.display()
2000 ),
2001 );
2002 self.state.advise_attempted = true;
2003 self.state.save()?;
2004 return Ok(());
2005 }
2006 Err(e) => {
2007 self.state.event(
2008 "advise",
2009 format!(
2010 "could not resolve advisor seats ({e:#}); continuing \
2011 without a design-deliberation brief (see {})",
2012 where_recorded.display()
2013 ),
2014 );
2015 self.state.advise_attempted = true;
2016 self.state.save()?;
2017 return Ok(());
2018 }
2019 };
2020
2021 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2022 let artifacts = agent::artifacts_dir(&self.state.dir());
2023 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2024
2025 let mut jobs = Vec::new();
2026 for (i, spec) in seats.iter().cloned().enumerate() {
2027 let seat_key = format!("advisor-{}", i + 1);
2028 let seat = self.seat(&seat_key, &spec.id);
2029 jobs.push(SeatJob {
2030 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2031 spec,
2032 seat,
2033 cwd: worktrees[i % worktrees.len()].clone(),
2034 timeout,
2035 allow_write: false,
2036 sessions: false,
2037 artifacts: artifacts.clone(),
2038 stem: seat_key,
2039 handover: None,
2040 });
2041 }
2042
2043 self.state.event(
2044 "advise",
2045 format!(
2046 "{} advisor seat(s) sketching a design in parallel",
2047 jobs.len()
2048 ),
2049 );
2050 let mut quota_losses = Vec::new();
2051 let cache = self.state.config.cache_dir();
2052 let ctx = WaveCtx {
2053 carry_seats: false,
2054 pauses: Some((&self.pause, &self.interrupt)),
2055 run: &run_id,
2056 node: "advise",
2057 prompts: &prompts,
2058 cache: cache.as_deref(),
2059 round: None,
2060 };
2061 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2062 let results = ask_json_wave::<Proposal>(
2063 jobs,
2064 Arc::clone(&self.sem),
2065 self.state.config.graph.retries,
2066 &advisor_roster,
2067 &ctx,
2068 &mut quota_losses,
2069 &mut self.state,
2070 &|p: &Proposal| p.validate(),
2071 )
2072 .await;
2073 if self.state.resource_block.is_some() {
2074 return self.stop_for_resource();
2075 }
2076 self.state.quota.extend(quota_losses);
2077
2078 let mut records = Vec::with_capacity(results.len());
2079 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2080 let agent_id = seat.agent.clone();
2081 self.state.seats.insert(seat.key.clone(), seat);
2082 match res {
2083 Ok((proposal, out)) => {
2084 self.state
2085 .event("advise", format!("advisor-{} proposed a design", i + 1));
2086 records.push(advise::AdvisorRecord::proposed(
2087 i + 1,
2088 agent_id,
2089 proposal,
2090 out.duration_ms,
2091 ));
2092 }
2093 Err(e) => {
2094 self.state.event(
2095 "advise",
2096 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2097 );
2098 records.push(advise::AdvisorRecord::failed(
2099 i + 1,
2100 agent_id,
2101 e.to_string(),
2102 ));
2103 }
2104 }
2105 }
2106
2107 let mut advice = advise::Advice {
2108 records,
2109 synthesis: None,
2110 };
2111 if advice.proposals().is_empty() {
2112 self.state.event(
2113 "advise",
2114 "no advisor produced a usable proposal; continuing without a \
2115 synthesis brief"
2116 .to_owned(),
2117 );
2118 } else {
2119 match self
2120 .synthesize_brief(
2121 &advice,
2122 &instruction,
2123 &language,
2124 &worktrees[0],
2125 &artifacts,
2126 &run_id,
2127 &prompts,
2128 cache.as_deref(),
2129 )
2130 .await
2131 {
2132 Ok(Some(text)) => {
2133 self.state.event(
2134 "advise",
2135 "synthesized a design brief for the implementer".to_owned(),
2136 );
2137 advice.synthesis = Some(text);
2138 }
2139 Ok(None) => {
2140 self.state.event(
2141 "advise",
2142 "the synthesis seat produced nothing usable; continuing \
2143 without a design brief"
2144 .to_owned(),
2145 );
2146 }
2147 Err(e) => {
2148 if e.is::<crate::repo_lock::ResourceBlocked>() {
2149 return Err(e);
2150 }
2151 self.state.event(
2152 "advise",
2153 format!("could not synthesize a design brief: {e:#}"),
2154 );
2155 }
2156 }
2157 }
2158 advise::apply_reflection(&mut advice);
2159
2160 self.state.advice = Some(advice);
2161 self.state.advise_attempted = true;
2162 self.state.save()?;
2163 Ok(())
2164 }
2165
2166 #[allow(clippy::too_many_arguments)]
2178 async fn synthesize_brief(
2179 &mut self,
2180 advice: &advise::Advice,
2181 instruction: &str,
2182 language: &str,
2183 cwd: &Path,
2184 artifacts: &Path,
2185 run_id: &str,
2186 prompts: &Prompts,
2187 cache: Option<&Path>,
2188 ) -> Result<Option<String>> {
2189 let chain = agent::pick_chain(
2190 &self.state.config.agents,
2191 self.state.config.roles.synthesizer.as_ref(),
2192 &agent::installed,
2193 "synthesizer",
2194 )?;
2195 let proposals = advice.proposals();
2196 let mut prompt = prompt::with_overlay(
2197 prompt::synthesize_brief(instruction, &proposals, language),
2198 prompts.overlay("advise"),
2199 );
2200 if cache.is_some() {
2201 prompt.push('\n');
2206 prompt.push_str(&prompt::build_cache_note("advise", false));
2207 }
2208 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2209 let mut last = None;
2214 for (n, spec) in chain.iter().enumerate() {
2215 if n > 0 {
2216 self.state
2217 .event("advise", format!("synthesis falling back to {}", spec.id));
2218 }
2219 let mut seat = self.seat("advise-synthesis", &spec.id);
2220 let wait = self.repository_wait();
2221 let outcome = wait
2222 .scope(agent::invoke(
2223 spec,
2224 &mut seat,
2225 &Invocation {
2226 scope: crate::agent::InvocationScope::Repository,
2227 coordination: self.state.config.coordination.as_ref(),
2228 fetched_base: None,
2229 cwd,
2230 prompt: &prompt,
2231 timeout,
2232 allow_write: false,
2233 unsandboxed: false,
2234 sessions: false,
2235 artifacts,
2236 stem: &if n == 0 {
2237 "advise-synthesis".to_owned()
2238 } else {
2239 format!("advise-synthesis-{}", spec.id)
2240 },
2241 run: run_id,
2242 node: "advise",
2243 cache_dir: None,
2244 attachments: &[],
2245 writable: &[],
2246 },
2247 ))
2248 .await;
2249 if outcome.is_ok() {
2250 self.state.seats.insert(seat.key.clone(), seat);
2251 }
2252 let advance = agent::chain_advances(&outcome);
2253 last = Some(outcome);
2254 if !advance {
2255 break;
2256 }
2257 }
2258 let out = last.expect("a chain holds at least one agent")?;
2261 if !out.usable() {
2262 return Ok(None);
2263 }
2264 let text =
2265 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2266 Ok((!text.trim().is_empty()).then_some(text))
2267 }
2268
2269 async fn implement(&mut self) -> Result<()> {
2272 self.persist_candidates().await?;
2273 let run_id = self.state.id.clone();
2278 let prompts = self.state.config.prompts.clone();
2279 let todo: Vec<usize> = self
2280 .state
2281 .candidates
2282 .iter()
2283 .enumerate()
2284 .filter(|(_, c)| {
2285 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2286 && c.failed.is_none()
2287 && !c.empty
2288 })
2289 .map(|(i, _)| i)
2290 .collect();
2291 if todo.is_empty() {
2292 return self.after_implement();
2293 }
2294 self.state.status = RunStatus::Implementing;
2295
2296 let language = self.state.config.graph.language.clone();
2297 let github_language = self.state.config.graph.github_language.clone();
2298 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2299 let sessions = self.state.config.graph.sessions;
2300 let artifacts = agent::artifacts_dir(&self.state.dir());
2301 let brief = self
2305 .state
2306 .advice
2307 .as_ref()
2308 .and_then(|a| a.synthesis.as_deref())
2309 .map(str::to_owned);
2310 let attachments = self.state.attachments.clone();
2311
2312 let mut jobs = Vec::new();
2313 for &i in &todo {
2314 let (index, label, worktree) = {
2315 let c = &self.state.candidates[i];
2316 (c.index, c.label, c.worktree.clone())
2317 };
2318 let spec = self.roles.implementers[index].clone();
2319 let seat_key = format!("impl-{label}");
2320 let seat = self.seat(&seat_key, &spec.id);
2321 let instruction = seeded_instruction(&self.state);
2322 jobs.push(SeatJob {
2323 spec,
2324 seat,
2325 prompt: prompt::implement_in(
2326 &instruction,
2327 &worktree.to_string_lossy(),
2328 &language,
2329 &github_language,
2330 brief.as_deref(),
2331 &attachments,
2332 ),
2333 cwd: worktree,
2334 timeout,
2335 allow_write: true,
2336 sessions,
2337 artifacts: artifacts.clone(),
2338 stem: format!("impl-{label}"),
2339 handover: None,
2340 });
2341 }
2342
2343 self.state.event(
2344 "implement",
2345 format!("{} candidates in parallel", jobs.len()),
2346 );
2347 let mut sent = jobs.clone();
2353 let cache = self.state.config.cache_dir();
2354 let ctx = WaveCtx {
2355 carry_seats: false,
2356 pauses: Some((&self.pause, &self.interrupt)),
2357 run: &run_id,
2358 node: "implement",
2359 prompts: &prompts,
2360 cache: cache.as_deref(),
2361 round: None,
2362 };
2363 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2364 if !wave_blocked(&results) {
2365 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2366 .await;
2367 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2368 .await?;
2369 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2370 .await;
2371 }
2372 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2373
2374 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2375 && results
2376 .iter()
2377 .any(|(_, _, out)| FailClass::of(out).is_some());
2378 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2379 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2380 if self
2384 .state
2385 .candidate_preservation
2386 .contains_key(&self.state.candidates[i].label)
2387 {
2388 self.state.seats.insert(seat.key.clone(), seat);
2389 }
2390 continue;
2391 }
2392 let seat_key = seat.key.clone();
2393 let agent = seat.agent.clone();
2402 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2403 self.state.seats.insert(seat.key.clone(), seat);
2404 let label = self.state.candidates[i].label;
2405 self.state.interrupted_implementations.remove(&label);
2406
2407 let (summary, duration, failed, verified_claim) = match out {
2408 AgentOutcome::Ok(o) => {
2409 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2410 let failed = (!o.usable()).then(|| {
2411 if o.timed_out {
2412 "agent timed out".to_owned()
2413 } else {
2414 format!(
2415 "agent exited with {}",
2416 crate::notices::exit_status(o.exit_code)
2417 )
2418 }
2419 });
2420 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2421 (text, o.duration_ms, failed, verified_claim)
2422 }
2423 AgentOutcome::Dropped(o) => {
2429 let why = o
2430 .dropped
2431 .as_ref()
2432 .map(|d| d.why.as_str())
2433 .unwrap_or("the CLI ended the stream without delivering its answer");
2434 (
2435 String::new(),
2436 o.duration_ms,
2437 Some(format!("the CLI dropped the stream ({why})")),
2438 None,
2439 )
2440 }
2441 AgentOutcome::Quota(o) => {
2442 self.state.quota.push(QuotaLoss {
2443 seat: seat_key,
2444 node: "implement".to_owned(),
2445 at: Timestamp::now(),
2446 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2447 });
2448 (
2449 String::new(),
2450 o.duration_ms,
2451 Some("rate limited (quota); produced no change".to_owned()),
2452 None,
2453 )
2454 }
2455 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2456 (String::new(), 0, Some(e), None)
2457 }
2458 };
2459
2460 let c = &mut self.state.candidates[i];
2463 if !exhausted_the_fallback_chain {
2464 c.agent = agent;
2465 }
2466 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2467 c.duration_ms = duration;
2468 c.failed = failed;
2469 c.verified_noop = verified_claim;
2470 self.state
2471 .candidate_preservation
2472 .insert(label, "awaiting persistence".to_owned());
2473 }
2474 self.state.save()?;
2475 self.persist_candidates().await?;
2476
2477 if resource_blocked {
2478 return self.stop_for_resource();
2479 }
2480 if selected_failed {
2481 self.state.status = RunStatus::Blocked;
2482 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2483 self.state.save()?;
2484 bail!(
2485 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2486 );
2487 }
2488 self.after_implement()
2489 }
2490
2491 async fn persist_candidates(&mut self) -> Result<()> {
2493 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2494 for label in labels {
2495 let i = self
2496 .state
2497 .candidates
2498 .iter()
2499 .position(|c| c.label == label)
2500 .context("pending candidate is missing")?;
2501 let c = self.state.candidates[i].clone();
2502 let interrupted = self.state.interrupted_implementations.contains(&label);
2503 let result = async {
2504 let wait = crate::repo_lock::WaitPolicy::new(
2505 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2506 || false,
2507 );
2508 let _coordination =
2509 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2510 if let Err(e) = git::snapshot_uncommitted(
2513 &c.worktree,
2514 &self
2515 .state
2516 .dir()
2517 .join("artifacts")
2518 .join(format!("candidate-{label}-recovery")),
2519 )
2520 .await
2521 {
2522 self.state.event(
2523 "preserve",
2524 format!("candidate {label} snapshot incomplete: {e:#}"),
2525 );
2526 }
2527 let rescued = git::rescue_commit(
2528 &c.worktree,
2529 &format!("magi: candidate {label} (uncommitted work)"),
2530 )
2531 .await?;
2532 self.state.note_withheld("implement", &rescued.withheld);
2533 let base = &self.state.base_commit;
2534 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2535 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2536 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2537 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2538 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2539 if c.empty && commits > 0 && !patch.trim().is_empty()
2540 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2541 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2542 || self.state.tally.is_some())
2543 {
2544 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2548 }
2549 let c = &mut self.state.candidates[i];
2550 c.commits = commits;
2551 c.stat = stat;
2552 c.files = files;
2553 let no_changes = commits == 0 || patch.trim().is_empty();
2554 c.empty = no_changes && !interrupted;
2555 if interrupted && no_changes {
2556 c.failed = None;
2557 c.verified_noop = None;
2558 }
2559 if !c.empty {
2560 c.failed = None;
2561 c.verified_noop = None;
2562 }
2563 self.state.event(
2564 "preserve",
2565 format!("candidate {label}: {files} files, {commits} commits"),
2566 );
2567 Ok::<_, anyhow::Error>(())
2568 }
2569 .await;
2570 match result {
2571 Ok(()) => {
2572 self.state.candidate_preservation.remove(&label);
2573 if !interrupted || self.state.candidates[i].files > 0 {
2574 self.state.interrupted_implementations.remove(&label);
2575 }
2576 }
2577 Err(e) => {
2578 let reason = format!("candidate {label} preservation failed: {e:#}");
2579 self.state.event("preserve", &reason);
2580 self.state.candidate_preservation.insert(label, reason);
2581 }
2582 }
2583 if self.state.candidate_preservation.is_empty()
2584 && self.state.status == RunStatus::Blocked
2585 && self.state.candidates.iter().all(|c| c.failed.is_none())
2586 {
2587 self.state.status = RunStatus::Implementing;
2591 self.state.event(
2592 "preserve",
2593 "candidate persistence recovered; implementation can continue",
2594 );
2595 }
2596 self.state.save()?;
2597 }
2598 if !self.state.candidate_preservation.is_empty() {
2599 self.state.status = RunStatus::Blocked;
2600 self.state.save()?;
2601 bail!(
2602 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2603 );
2604 }
2605 Ok(())
2606 }
2607
2608 async fn resume_undelivered(
2636 &mut self,
2637 results: &mut [(usize, SeatState, AgentOutcome)],
2638 sent: &[SeatJob],
2639 prompts: &Prompts,
2640 run_id: &str,
2641 ) {
2642 for (wi, seat, out) in results.iter_mut() {
2643 let Some(dropped) = (match &*out {
2644 AgentOutcome::Dropped(o) => o.dropped.clone(),
2645 _ => None,
2646 }) else {
2647 continue;
2648 };
2649 let Some(job) = sent.get(*wi) else { continue };
2650 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2652 self.state.event(
2653 "implement",
2654 format!(
2655 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2656 work is in the tree",
2657 seat.key, dropped.output_tokens, dropped.why
2658 ),
2659 );
2660 continue;
2661 }
2662 if !has_context(&job.spec, seat, job.sessions) {
2670 self.state.event(
2671 "implement",
2672 format!(
2673 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2674 is no session left to resume",
2675 seat.key, dropped.output_tokens, dropped.why
2676 ),
2677 );
2678 continue;
2679 }
2680 self.state.event(
2681 "implement",
2682 format!(
2683 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2684 conversation",
2685 seat.key, dropped.output_tokens, dropped.why
2686 ),
2687 );
2688 let mut retry = job.clone();
2689 retry.seat = seat.clone();
2690 retry.prompt = prompt::resume_after_drop(&dropped.why);
2691 retry.timeout = retry_budget(job.timeout, true);
2692 retry.stem = format!("{}-resume", job.stem);
2693 let cache = self.state.config.cache_dir();
2694 let ctx = WaveCtx {
2695 carry_seats: false,
2696 pauses: Some((&self.pause, &self.interrupt)),
2697 run: run_id,
2698 node: "implement",
2699 prompts,
2700 cache: cache.as_deref(),
2701 round: None,
2702 };
2703 let (resumed_seat, resumed) =
2704 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2705 *seat = resumed_seat;
2706 *out = resumed;
2707 }
2708 }
2709
2710 async fn resume_seat_handovers(
2775 &mut self,
2776 results: &mut [(usize, SeatState, AgentOutcome)],
2777 sent: &mut [SeatJob],
2778 prompts: &Prompts,
2779 run_id: &str,
2780 ) -> Result<()> {
2781 let instruction = seeded_instruction(&self.state);
2782 let language = self.state.config.graph.language.clone();
2783 let github_language = self.state.config.graph.github_language.clone();
2784 let brief = self
2785 .state
2786 .advice
2787 .as_ref()
2788 .and_then(|a| a.synthesis.as_deref())
2789 .map(str::to_owned);
2790 let attachments = self.state.attachments.clone();
2791 for (wi, seat, out) in results.iter_mut() {
2792 let others: BTreeSet<String> = sent
2795 .iter()
2796 .enumerate()
2797 .filter(|(j, _)| j != wi)
2798 .map(|(_, j)| j.spec.id.clone())
2799 .collect();
2800 let Some(job) = sent.get_mut(*wi) else {
2801 continue;
2802 };
2803 let start = self
2808 .roles
2809 .implementer_roster
2810 .iter()
2811 .position(|s| s.id == job.spec.id)
2812 .unwrap_or(0);
2813 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2814 let mut fallback_attempt = 0usize;
2815 let mut prev: Option<FailClass> = None;
2816 while let Some(cur) = FailClass::of(&*out) {
2817 if !should_hand_over(prev.as_ref(), &cur) {
2818 break;
2819 }
2820 let Some(next) =
2821 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2822 .cloned()
2823 else {
2824 break;
2825 };
2826 tried.insert(next.id.clone());
2827 fallback_attempt += 1;
2828
2829 let coordination = self.coordinate(&job.cwd).await?;
2830 if let Ok(r) = git::rescue_commit(
2831 &job.cwd,
2832 &format!(
2833 "magi: candidate {} (uncommitted work before {} fallback)",
2834 seat.key,
2835 if cur == FailClass::Quota {
2836 "quota"
2837 } else {
2838 "handover"
2839 }
2840 ),
2841 )
2842 .await
2843 {
2844 self.state.note_withheld("implement", &r.withheld);
2845 }
2846 drop(coordination);
2847
2848 record_handover(
2849 &mut self.state,
2850 "implement",
2851 &seat.key,
2852 &seat.agent,
2853 &next.id,
2854 &cur,
2855 &fail_reason(&*out),
2856 );
2857 prev = Some(cur.clone());
2858
2859 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2860 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2861 job.spec = next.clone();
2869 let mut retry = job.clone();
2870 retry.seat = new_seat;
2871 retry.prompt = prompt::implement_in(
2872 &instruction,
2873 &job.cwd.to_string_lossy(),
2874 &language,
2875 &github_language,
2876 brief.as_deref(),
2877 &attachments,
2878 );
2879 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2880 let cache = self.state.config.cache_dir();
2881 let ctx = WaveCtx {
2882 carry_seats: false,
2883 pauses: Some((&self.pause, &self.interrupt)),
2884 run: run_id,
2885 node: "implement",
2886 prompts,
2887 cache: cache.as_deref(),
2888 round: None,
2889 };
2890 let (fallback_seat, fallback_out) = run_one(
2891 retry,
2892 Arc::clone(&self.sem),
2893 &ctx,
2894 &mut self.state,
2895 fallback_attempt,
2896 )
2897 .await;
2898 *seat = fallback_seat;
2899 *out = fallback_out;
2900 }
2901 }
2902 Ok(())
2903 }
2904
2905 async fn resume_unconfirmed_commands(
2929 &mut self,
2930 results: &mut [(usize, SeatState, AgentOutcome)],
2931 sent: &[SeatJob],
2932 prompts: &Prompts,
2933 run_id: &str,
2934 ) {
2935 for (wi, seat, out) in results.iter_mut() {
2936 let AgentOutcome::Ok(o) = &*out else {
2937 continue;
2938 };
2939 if !has_unconfirmed_command(&o.commands) {
2940 continue;
2941 }
2942 let Some(job) = sent.get(*wi) else { continue };
2943 if !has_context(&job.spec, seat, job.sessions) {
2944 self.state.event(
2945 "implement",
2946 format!(
2947 "{}: the reply named a command whose own CLI never confirmed the exit \
2948 status of, but there is no session left to resume",
2949 seat.key
2950 ),
2951 );
2952 continue;
2953 }
2954 self.state.event(
2955 "implement",
2956 format!(
2957 "{}: the reply named a command whose own CLI never confirmed the exit \
2958 status of; resuming the conversation",
2959 seat.key
2960 ),
2961 );
2962 let mut retry = job.clone();
2963 retry.seat = seat.clone();
2964 retry.prompt = prompt::resume_incomplete(
2965 "a command in your last reply had no confirmed exit status",
2966 );
2967 retry.timeout = retry_budget(job.timeout, true);
2968 retry.stem = format!("{}-confirm", job.stem);
2969 let cache = self.state.config.cache_dir();
2970 let ctx = WaveCtx {
2971 carry_seats: false,
2972 pauses: Some((&self.pause, &self.interrupt)),
2973 run: run_id,
2974 node: "implement",
2975 prompts,
2976 cache: cache.as_deref(),
2977 round: None,
2978 };
2979 let (resumed_seat, resumed) =
2980 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2981 *seat = resumed_seat;
2982 *out = resumed;
2983 }
2984 }
2985
2986 async fn continue_fix_report(
3007 &mut self,
3008 mut seat: SeatState,
3009 parse_err: String,
3010 job: &SeatJob,
3011 prompts: &Prompts,
3012 run_id: &str,
3013 round: usize,
3014 ) -> (
3015 SeatState,
3016 Option<FixReport>,
3017 Option<String>,
3018 ContinuationRecord,
3019 ) {
3020 let mut last_err = parse_err;
3021 let mut cumulative_wait_ms = 0u64;
3022 let mut attempts = 0usize;
3023 loop {
3024 if !has_context(&job.spec, &seat, job.sessions) {
3025 self.state.event(
3026 "fix",
3027 format!(
3028 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3029 session left to resume into"
3030 ),
3031 );
3032 let outcome = if attempts == 0 {
3033 ContinuationOutcome::NoSession
3034 } else {
3035 ContinuationOutcome::Exhausted
3036 };
3037 return (
3038 seat,
3039 None,
3040 Some(format!("unparsable fix report: {last_err}")),
3041 ContinuationRecord {
3042 attempts,
3043 cumulative_wait_ms,
3044 outcome,
3045 },
3046 );
3047 }
3048 if attempts >= MAX_FIX_CONTINUATIONS {
3049 self.state.event(
3050 "fix",
3051 format!(
3052 "round {round}: fixer's reply still had no adoption report after \
3053 {attempts} continuation(s) ({last_err}); giving up"
3054 ),
3055 );
3056 return (
3057 seat,
3058 None,
3059 Some(format!(
3060 "unparsable fix report after {attempts} continuation(s): {last_err}"
3061 )),
3062 ContinuationRecord {
3063 attempts,
3064 cumulative_wait_ms,
3065 outcome: ContinuationOutcome::Exhausted,
3066 },
3067 );
3068 }
3069 attempts += 1;
3070 self.state.event(
3071 "fix",
3072 format!(
3073 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3074 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3075 ),
3076 );
3077 let mut retry = job.clone();
3078 retry.seat = seat.clone();
3079 retry.prompt = prompt::resume_incomplete(&last_err);
3080 retry.timeout = retry_budget(job.timeout, true);
3081 retry.stem = format!("{}-continue{attempts}", job.stem);
3082 let cache = self.state.config.cache_dir();
3083 let ctx = WaveCtx {
3084 carry_seats: false,
3085 pauses: Some((&self.pause, &self.interrupt)),
3086 run: run_id,
3087 node: "fix",
3088 prompts,
3089 cache: cache.as_deref(),
3090 round: Some(round),
3091 };
3092 let (resumed_seat, resumed_out) = run_one(
3093 retry,
3094 Arc::clone(&self.sem),
3095 &ctx,
3096 &mut self.state,
3097 attempts,
3098 )
3099 .await;
3100 seat = resumed_seat;
3101 match resumed_out {
3102 AgentOutcome::Ok(o) => {
3103 cumulative_wait_ms += o.duration_ms;
3104 match verdict::extract_json::<FixReport>(&o.text) {
3105 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3106 self.state.event(
3107 "fix",
3108 format!(
3109 "round {round}: fixer's adoption report recovered after \
3110 {attempts} continuation(s)"
3111 ),
3112 );
3113 return (
3114 seat,
3115 Some(report),
3116 None,
3117 ContinuationRecord {
3118 attempts,
3119 cumulative_wait_ms,
3120 outcome: ContinuationOutcome::Resumed,
3121 },
3122 );
3123 }
3124 Ok(_) => {
3132 last_err = "the reply parsed, but it reported a command whose own CLI \
3133 never confirmed an exit status"
3134 .to_owned();
3135 }
3136 Err(e) => last_err = e.to_string(),
3137 }
3138 }
3139 AgentOutcome::Quota(o) => {
3140 cumulative_wait_ms += o.duration_ms;
3141 self.state.quota.push(QuotaLoss {
3142 seat: seat.key.clone(),
3143 node: "fix".to_owned(),
3144 at: Timestamp::now(),
3145 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3146 });
3147 self.state.event(
3148 "fix",
3149 format!(
3150 "round {round}: continuation rate limited (quota); not retrying now"
3151 ),
3152 );
3153 return (
3154 seat,
3155 None,
3156 Some("rate limited (quota) while recovering the fix report".to_owned()),
3157 ContinuationRecord {
3158 attempts,
3159 cumulative_wait_ms,
3160 outcome: ContinuationOutcome::QuotaLost,
3161 },
3162 );
3163 }
3164 AgentOutcome::Dropped(o) => {
3165 cumulative_wait_ms += o.duration_ms;
3166 let why = o
3167 .dropped
3168 .as_ref()
3169 .map(|d| d.why.as_str())
3170 .unwrap_or("the CLI ended the stream without delivering its answer");
3171 last_err = format!("the CLI dropped the stream ({why})");
3172 }
3173 AgentOutcome::Failed(e) => last_err = e,
3174 AgentOutcome::ResourceBlocked(e) => {
3177 return (
3178 seat,
3179 None,
3180 Some(format!("the continuation was not started: {e}")),
3181 ContinuationRecord {
3182 attempts: attempts.saturating_sub(1),
3183 cumulative_wait_ms,
3184 outcome: ContinuationOutcome::NoSession,
3185 },
3186 );
3187 }
3188 }
3189 }
3190 }
3191
3192 fn after_implement(&mut self) -> Result<()> {
3193 if !self.state.candidate_preservation.is_empty() {
3194 bail!("candidate preservation incomplete; judging is unsafe");
3195 }
3196 if self.state.leaks.is_empty() {
3198 let cfg = self.state.config.blind.clone();
3199 let mut leaks = Vec::new();
3200 for c in &self.state.candidates {
3201 let Some(patch) =
3202 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3203 else {
3204 continue;
3205 };
3206 leaks.extend(blind::scan(
3207 &format!("candidate {} patch", c.label),
3208 &patch,
3209 &cfg.vendor_tokens,
3210 ));
3211 }
3212 if !leaks.is_empty() {
3213 let summary = leaks
3214 .iter()
3215 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3216 .collect::<Vec<_>>()
3217 .join(", ");
3218 match cfg.on_leak {
3219 LeakPolicy::Fail => {
3220 self.state.status = RunStatus::Failed;
3221 self.state
3222 .event("blind", format!("vendor text in a patch: {summary}"));
3223 self.state.leaks = leaks;
3224 self.state.save()?;
3225 self.settle_questions();
3226 bail!(
3227 "blind.on_leak = \"fail\" and vendor text reached a \
3228 judged patch: {summary}"
3229 );
3230 }
3231 LeakPolicy::Redact => self.state.event(
3232 "blind",
3233 format!("redacting vendor text for judging: {summary}"),
3234 ),
3235 LeakPolicy::Warn => self.state.event(
3236 "blind",
3237 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3238 ),
3239 }
3240 self.state.leaks = leaks;
3241 }
3242 }
3243
3244 if self.state.viable().is_empty() {
3245 if self.state.all_candidates_verified_noop() {
3246 self.state.status = RunStatus::VerifiedNoop;
3257 self.state.save()?;
3258 self.settle_questions();
3259 return Ok(());
3260 }
3261 self.state.status = RunStatus::Failed;
3262 self.state.save()?;
3263 self.settle_questions();
3264 bail!("no candidate produced a change; nothing to judge");
3265 }
3266 self.state.status = RunStatus::Judging;
3267 self.state.save()?;
3268 Ok(())
3269 }
3270
3271 async fn judge(&mut self) -> Result<()> {
3274 let run_id = self.state.id.clone();
3279 let prompts = self.state.config.prompts.clone();
3280 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3281 return Ok(());
3282 }
3283 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3284 if viable.len() == 1 {
3285 self.state.judge_skipped = true;
3292 self.state.event(
3293 "judge",
3294 format!(
3295 "only candidate {} produced a change; judging skipped",
3296 viable[0].label
3297 ),
3298 );
3299 self.state.save()?;
3300 return Ok(());
3301 }
3302 self.state.status = RunStatus::Judging;
3303
3304 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3305 let language = self.state.config.graph.language.clone();
3306 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3307 let sessions = self.state.config.graph.sessions;
3308 let artifacts = agent::artifacts_dir(&self.state.dir());
3309 let root = self.state.worktree_root();
3310 let base_short = short(&self.state.base_commit);
3311
3312 let mut jobs = Vec::new();
3313 let mut orders = Vec::new();
3314 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3315 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3316 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3317 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3318 let seat_key = format!("judge-{}", j + 1);
3319 let seat = self.seat(&seat_key, &spec.id);
3320 jobs.push(SeatJob {
3321 prompt: prompt::judge(
3322 &self.state.instruction,
3323 &views,
3324 self.roles.judges.len(),
3325 &base_short,
3326 &language,
3327 ),
3328 spec,
3329 seat,
3330 cwd: root.join(format!("judge-{}", j + 1)),
3331 timeout,
3332 allow_write: false,
3333 sessions,
3334 artifacts: artifacts.clone(),
3335 stem: format!("judge-{}", j + 1),
3336 handover: None,
3337 });
3338 }
3339
3340 self.state.event(
3341 "judge",
3342 format!(
3343 "{} judges ranking {} candidates blind",
3344 jobs.len(),
3345 viable.len()
3346 ),
3347 );
3348 let labels_for_check = labels.clone();
3349 let mut quota_losses = Vec::new();
3350 let cache = self.state.config.cache_dir();
3351 let ctx = WaveCtx {
3352 carry_seats: false,
3353 pauses: Some((&self.pause, &self.interrupt)),
3354 run: &run_id,
3355 node: "judge",
3356 prompts: &prompts,
3357 cache: cache.as_deref(),
3358 round: None,
3359 };
3360 let results = ask_json_wave::<Ranking>(
3361 jobs,
3362 Arc::clone(&self.sem),
3363 self.state.config.graph.retries,
3364 &self.roles.judge_roster,
3365 &ctx,
3366 &mut quota_losses,
3367 &mut self.state,
3368 &move |r: &Ranking| r.validate(&labels_for_check),
3369 )
3370 .await;
3371 if self.state.resource_block.is_some() {
3372 return self.stop_for_resource();
3373 }
3374 self.state.quota.extend(quota_losses);
3375
3376 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3377 let agent_id = seat.agent.clone();
3378 self.state.seats.insert(seat.key.clone(), seat);
3379 let mut record = Judgement {
3380 judge: j + 1,
3381 seat: format!("judge-{}", j + 1),
3382 agent: agent_id,
3383 ranking: Vec::new(),
3384 reasons: BTreeMap::new(),
3385 confidence: None,
3386 strengths: BTreeMap::new(),
3387 order: orders[j].clone(),
3388 failed: None,
3389 duration_ms: 0,
3390 };
3391 match res {
3392 Ok((ranking, out)) => {
3393 record.ranking = ranking.normalized();
3394 record.reasons = ranking.reasons;
3395 record.strengths = ranking.strengths;
3396 record.confidence = ranking.confidence;
3397 record.duration_ms = out.duration_ms;
3398 self.state.event(
3399 "judge",
3400 format!(
3401 "judge {} ranked {}",
3402 j + 1,
3403 record.ranking.iter().collect::<String>()
3404 ),
3405 );
3406 }
3407 Err(e) => {
3408 record.failed = Some(e.to_string());
3409 self.state
3410 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3411 }
3412 }
3413 self.state.judgements.push(record);
3414 self.state.save()?;
3415 }
3416 Ok(())
3417 }
3418
3419 async fn deliberate(&mut self) -> Result<()> {
3422 let run_id = self.state.id.clone();
3427 let prompts = self.state.config.prompts.clone();
3428 if !self.state.deliberation.is_empty() {
3429 return Ok(());
3430 }
3431 let tops: Vec<char> = self
3432 .state
3433 .judgements
3434 .iter()
3435 .filter_map(|j| j.ranking.first().copied())
3436 .collect();
3437 let rounds = self.state.config.graph.deliberate_rounds;
3438 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3439 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3440 self.state.event(
3441 "deliberate",
3442 format!("judges agreed on {} outright; no deliberation", tops[0]),
3443 );
3444 }
3445 self.state.status = RunStatus::Voting;
3446 self.state.save()?;
3447 return Ok(());
3448 }
3449
3450 self.state.status = RunStatus::Deliberating;
3451 self.state.event(
3452 "deliberate",
3453 format!(
3454 "split: first choices were {} — opening {rounds} round(s)",
3455 tops.iter().collect::<String>()
3456 ),
3457 );
3458
3459 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3460 let language = self.state.config.graph.language.clone();
3461 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3462 let sessions = self.state.config.graph.sessions;
3463 let artifacts = agent::artifacts_dir(&self.state.dir());
3464 let root = self.state.worktree_root();
3465 let base_short = short(&self.state.base_commit);
3466
3467 for round in 1..=rounds {
3471 let mut turns: Vec<DeliberationTurn> = Vec::new();
3472 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3473 if self.state.judgements[j].failed.is_some() {
3474 continue;
3475 }
3476 let seat_key = format!("judge-{}", j + 1);
3477 let spec = self.occupant(&seat_key, spec);
3478 let mut seat = self.seat(&seat_key, &spec.id);
3479 let transcript = self.transcript(&turns, j);
3480 let build = |context: Option<&str>| {
3481 prompt::deliberate(
3482 &self.state.instruction,
3483 context,
3484 &transcript,
3485 round,
3486 rounds,
3487 &language,
3488 )
3489 };
3490 let block = self.candidate_block(&viable, &base_short);
3491 let full = build(Some(&block));
3492 let text = if has_context(&spec, &seat, sessions) {
3493 build(None)
3494 } else {
3495 full.clone()
3496 };
3497 let job = SeatJob {
3498 spec,
3499 seat: seat.clone(),
3500 prompt: text,
3501 cwd: root.join(format!("judge-{}", j + 1)),
3502 timeout,
3503 allow_write: false,
3504 sessions,
3505 artifacts: artifacts.clone(),
3506 stem: format!("delib-{round}-judge-{}", j + 1),
3507 handover: Some(full),
3508 };
3509 let cache = self.state.config.cache_dir();
3510 let ctx = WaveCtx {
3511 carry_seats: false,
3512 pauses: Some((&self.pause, &self.interrupt)),
3513 run: &run_id,
3514 node: "deliberate",
3515 prompts: &prompts,
3516 cache: cache.as_deref(),
3517 round: None,
3518 };
3519 let mut losses = Vec::new();
3523 let mut results = ask_wave_with::<String>(
3524 vec![job],
3525 Arc::clone(&self.sem),
3526 0,
3527 &self.roles.judge_roster,
3528 &ctx,
3529 &mut losses,
3530 &mut self.state,
3531 &|text: &str| {
3532 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3533 },
3534 )
3535 .await;
3536 self.state.quota.extend(losses);
3537 let (updated, res, _) = results.pop().expect("one job in, one result out");
3538 seat = updated;
3539 let agent_id = seat.agent.clone();
3540 self.state.seats.insert(seat.key.clone(), seat);
3541 let body = match res {
3542 Ok((body, _)) => body,
3543 Err(e) => {
3546 self.state
3547 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3548 continue;
3549 }
3550 };
3551 let tentative = verdict::extract_json::<Position>(&body)
3552 .ok()
3553 .and_then(|p| p.tentative)
3554 .and_then(|s| s.trim().chars().next())
3555 .map(|c| c.to_ascii_uppercase());
3556 self.state.event(
3557 "deliberate",
3558 format!(
3559 "round {round}: judge {} now favours {}",
3560 j + 1,
3561 tentative.map_or("—".to_owned(), |c| c.to_string())
3562 ),
3563 );
3564 turns.push(DeliberationTurn {
3565 judge: j + 1,
3566 agent: agent_id,
3567 body: blind::sanitize_prose(&body, &self.state.config.blind),
3568 tentative,
3569 });
3570 }
3571 self.state
3572 .deliberation
3573 .push(DeliberationRound { round, turns });
3574 self.state.save()?;
3575 }
3576
3577 self.state.status = RunStatus::Voting;
3578 self.state.save()?;
3579 Ok(())
3580 }
3581
3582 async fn vote(&mut self) -> Result<()> {
3585 let run_id = self.state.id.clone();
3590 let prompts = self.state.config.prompts.clone();
3591 if !self.state.votes.is_empty() {
3592 return Ok(());
3593 }
3594 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3595 if viable.len() == 1 {
3596 return Ok(());
3597 }
3598 self.state.status = RunStatus::Voting;
3599
3600 let language = self.state.config.graph.language.clone();
3601 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3602 let sessions = self.state.config.graph.sessions;
3603 let artifacts = agent::artifacts_dir(&self.state.dir());
3604 let root = self.state.worktree_root();
3605 let base_short = short(&self.state.base_commit);
3606 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3607
3608 let mut jobs = Vec::new();
3609 let mut seats_at = Vec::new();
3610 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3611 if self
3612 .state
3613 .judgements
3614 .get(j)
3615 .is_some_and(|r| r.failed.is_some())
3616 {
3617 continue;
3618 }
3619 let seat_key = format!("judge-{}", j + 1);
3620 let spec = self.occupant(&seat_key, spec);
3621 let seat = self.seat(&seat_key, &spec.id);
3622 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3623 let text = if has_context(&spec, &seat, sessions) {
3624 prompt::final_vote(&viable, &language)
3625 } else {
3626 full.clone()
3627 };
3628 jobs.push(SeatJob {
3629 spec,
3630 seat,
3631 prompt: text,
3632 cwd: root.join(format!("judge-{}", j + 1)),
3633 timeout,
3634 allow_write: false,
3635 sessions,
3636 artifacts: artifacts.clone(),
3637 stem: format!("vote-judge-{}", j + 1),
3638 handover: Some(full),
3639 });
3640 seats_at.push(j);
3641 }
3642
3643 self.state.event(
3644 "vote",
3645 format!(
3646 "collecting {} final votes one by one, privately",
3647 jobs.len()
3648 ),
3649 );
3650 let allowed = viable.clone();
3651 let mut quota_losses = Vec::new();
3652 let cache = self.state.config.cache_dir();
3653 let ctx = WaveCtx {
3654 carry_seats: false,
3655 pauses: Some((&self.pause, &self.interrupt)),
3656 run: &run_id,
3657 node: "vote",
3658 prompts: &prompts,
3659 cache: cache.as_deref(),
3660 round: None,
3661 };
3662 let results = ask_json_wave::<FinalVote>(
3663 jobs,
3664 Arc::clone(&self.sem),
3665 self.state.config.graph.retries,
3666 &self.roles.judge_roster,
3667 &ctx,
3668 &mut quota_losses,
3669 &mut self.state,
3670 &move |v: &FinalVote| match v.label() {
3671 Some(c) if allowed.contains(&c) => Ok(()),
3672 other => bail!("vote {other:?} is not one of {allowed:?}"),
3673 },
3674 )
3675 .await;
3676 if self.state.resource_block.is_some() {
3677 return self.stop_for_resource();
3678 }
3679 self.state.quota.extend(quota_losses);
3680
3681 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3682 let agent_id = seat.agent.clone();
3683 self.state.seats.insert(seat.key.clone(), seat);
3684 let initial = self
3685 .state
3686 .judgements
3687 .get(j)
3688 .and_then(|r| r.ranking.first().copied());
3689 let mut record = VoteRecord {
3690 judge: j + 1,
3691 agent: agent_id,
3692 vote: None,
3693 reason: String::new(),
3694 changed: false,
3695 };
3696 match res {
3697 Ok((v, _)) => {
3698 record.vote = v.label();
3699 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3700 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3701 self.state.event(
3702 "vote",
3703 format!(
3704 "judge {} voted {}{}",
3705 j + 1,
3706 record.vote.unwrap_or('?'),
3707 if record.changed { " (changed)" } else { "" }
3708 ),
3709 );
3710 }
3711 Err(e) => {
3712 self.state
3713 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3714 }
3715 }
3716 self.state.votes.push(record);
3717 self.state.save()?;
3718 }
3719 Ok(())
3720 }
3721
3722 fn tally(&mut self) -> Result<()> {
3725 if self.state.tally.is_some() {
3726 return Ok(());
3727 }
3728 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3729 let tops: Vec<char> = self
3730 .state
3731 .judgements
3732 .iter()
3733 .filter_map(|j| j.ranking.first().copied())
3734 .collect();
3735 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3736
3737 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3740 let mut cast: Vec<char> = Vec::new();
3741 for (i, j) in self.state.judgements.iter().enumerate() {
3742 let vote = self
3743 .state
3744 .votes
3745 .iter()
3746 .find(|v| v.judge == i + 1)
3747 .and_then(|v| v.vote)
3748 .or_else(|| j.ranking.first().copied());
3749 if let Some(v) = vote {
3750 *first_choice.entry(v).or_insert(0) += 1;
3751 cast.push(v);
3752 }
3753 }
3754
3755 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3756 for j in &self.state.judgements {
3757 let n = j.ranking.len();
3758 for (pos, label) in j.ranking.iter().enumerate() {
3759 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3760 }
3761 }
3762
3763 let best = first_choice.values().copied().max().unwrap_or(0);
3764 let mut leaders: Vec<char> = first_choice
3765 .iter()
3766 .filter(|(_, v)| **v == best)
3767 .map(|(k, _)| *k)
3768 .collect();
3769 let mut tie_break = None;
3770 if leaders.len() > 1 {
3771 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3772 let borda_leaders: Vec<char> = leaders
3773 .iter()
3774 .copied()
3775 .filter(|l| borda[l] == top_borda)
3776 .collect();
3777 tie_break = Some(if borda_leaders.len() == 1 {
3778 format!(
3779 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3780 leaders.len()
3781 )
3782 } else {
3783 format!(
3784 "{} way tie on both first-choice votes and Borda points, broken by label order",
3785 leaders.len()
3786 )
3787 });
3788 leaders = borda_leaders;
3789 leaders.sort_unstable();
3790 }
3791 let winner = *leaders
3792 .first()
3793 .or(viable.first())
3794 .context("no candidate to declare a winner from")?;
3795
3796 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3797 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3798 let deliberated = !self.state.deliberation.is_empty();
3799
3800 let quota_seats: std::collections::BTreeSet<&str> =
3804 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3805 let mut present = 0usize;
3806 for (i, j) in self.state.judgements.iter().enumerate() {
3807 if quota_seats.contains(j.seat.as_str()) {
3808 continue;
3809 }
3810 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3811 let voted = self
3812 .state
3813 .votes
3814 .iter()
3815 .any(|v| v.judge == i + 1 && v.vote.is_some());
3816 if ranked || voted {
3817 present += 1;
3818 }
3819 }
3820 let needs_quorum = viable.len() > 1;
3826 let judges_total = if needs_quorum {
3827 self.roles.judges.len()
3828 } else {
3829 0
3830 };
3831 let quorum = if needs_quorum {
3832 judges_total / 2 + 1
3833 } else {
3834 0
3835 };
3836 let met_quorum = !needs_quorum || present >= quorum;
3837 let uncontested = (!needs_quorum).then(|| {
3838 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3839 });
3840
3841 self.state.event(
3842 "tally",
3843 match &uncontested {
3844 Some(reason) => format!("winner {winner} — {reason}"),
3845 None => format!(
3846 "winner {winner} — votes {} | initial {} | {} changed | \
3847 {present}/{judges_total} judges{}",
3848 first_choice
3849 .iter()
3850 .map(|(k, v)| format!("{k}:{v}"))
3851 .collect::<Vec<_>>()
3852 .join(" "),
3853 if unanimous_initial {
3854 "unanimous"
3855 } else {
3856 "split"
3857 },
3858 changed_votes,
3859 if met_quorum {
3860 String::new()
3861 } else {
3862 format!(" — below quorum ({quorum} required)")
3863 },
3864 ),
3865 },
3866 );
3867 if !met_quorum {
3868 self.state.event(
3869 "stall",
3870 format!(
3871 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3872 the run stops here, resumable"
3873 ),
3874 );
3875 }
3876 self.state.tally = Some(Tally {
3877 first_choice,
3878 borda,
3879 winner,
3880 rankings: tops.len(),
3881 unanimous_initial,
3882 deliberated,
3883 changed_votes,
3884 unanimous_final,
3885 tie_break,
3886 judges: judges_total,
3887 present,
3888 quorum,
3889 met_quorum,
3890 uncontested,
3891 });
3892 self.state.status = if met_quorum {
3893 RunStatus::Reviewing
3894 } else {
3895 RunStatus::Stalled
3896 };
3897 self.state.save()?;
3898 Ok(())
3899 }
3900
3901 #[allow(clippy::too_many_lines)]
3922 async fn recover_stall(&mut self) -> Result<bool> {
3923 let run_id = self.state.id.clone();
3928 let prompts = self.state.config.prompts.clone();
3929 let quota_seats: BTreeSet<&str> =
3934 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3935 let absent: Vec<String> = self
3936 .state
3937 .judgements
3938 .iter()
3939 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3940 .map(|j| j.seat.clone())
3941 .collect();
3942 if absent.is_empty() {
3943 return Ok(false);
3944 }
3945 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3946 if viable.len() <= 1 {
3947 return Ok(false);
3948 }
3949 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3950 let language = self.state.config.graph.language.clone();
3951 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3952 let sessions = self.state.config.graph.sessions;
3953 let artifacts = agent::artifacts_dir(&self.state.dir());
3954 let root = self.state.worktree_root();
3955 let base_short = short(&self.state.base_commit);
3956 let candidates: Vec<Candidate> = viable.clone();
3957
3958 let mut positions: Vec<usize> = absent
3960 .iter()
3961 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3962 .collect();
3963 if positions.is_empty() {
3964 return Ok(false);
3965 }
3966 positions.sort_unstable();
3967 positions.dedup();
3968
3969 let mut judge_jobs = Vec::new();
3971 for &j in &positions {
3972 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3973 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3974 let seat_key = format!("judge-{}", j + 1);
3975 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3976 let seat = self.seat(&seat_key, &spec.id);
3977 judge_jobs.push(SeatJob {
3978 spec,
3979 seat,
3980 prompt: prompt::judge(
3981 &self.state.instruction,
3982 &views,
3983 self.roles.judges.len(),
3984 &base_short,
3985 &language,
3986 ),
3987 cwd: root.join(seat_key),
3988 timeout,
3989 allow_write: false,
3990 sessions,
3991 artifacts: artifacts.clone(),
3992 stem: format!("judge-{}-recover", j + 1),
3993 handover: None,
3994 });
3995 }
3996
3997 let labels_for_check = labels.clone();
3998 let mut judge_losses = Vec::new();
3999 let retries = self.state.config.graph.retries;
4000 let cache = self.state.config.cache_dir();
4001 let ctx = WaveCtx {
4002 carry_seats: false,
4003 pauses: Some((&self.pause, &self.interrupt)),
4004 run: &run_id,
4005 node: "judge",
4006 prompts: &prompts,
4007 cache: cache.as_deref(),
4008 round: None,
4009 };
4010 let results = ask_json_wave::<Ranking>(
4011 judge_jobs,
4012 Arc::clone(&self.sem),
4013 retries,
4014 &self.roles.judge_roster,
4015 &ctx,
4016 &mut judge_losses,
4017 &mut self.state,
4018 &move |r: &Ranking| r.validate(&labels_for_check),
4019 )
4020 .await;
4021 if self.state.resource_block.is_some() {
4022 self.stop_for_resource()?;
4023 return Ok(false);
4024 }
4025
4026 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4028 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4029 let agent_id = seat.agent.clone();
4030 self.state.seats.insert(seat.key.clone(), seat);
4031 let record = &mut self.state.judgements[j];
4032 match res {
4033 Ok((ranking, out)) => {
4034 record.agent = agent_id;
4035 record.ranking = ranking.normalized();
4036 record.reasons = ranking.reasons;
4037 record.strengths = ranking.strengths;
4038 record.confidence = ranking.confidence;
4039 record.failed = None;
4040 record.duration_ms = out.duration_ms;
4041 recovered.insert(j);
4042 self.state.event(
4043 "recover",
4044 format!("judge {} ranked again after the limit", j + 1),
4045 );
4046 }
4047 Err(e) => {
4048 self.state
4049 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4050 }
4051 }
4052 }
4053
4054 let mut vote_jobs = Vec::new();
4056 let mut vote_pos: Vec<usize> = Vec::new();
4057 for &j in &recovered {
4058 let seat_key = format!("judge-{}", j + 1);
4059 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4060 let seat = self.seat(&seat_key, &spec.id);
4061 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4062 let text = if has_context(&spec, &seat, sessions) {
4063 prompt::final_vote(&labels, &language)
4064 } else {
4065 full.clone()
4066 };
4067 vote_jobs.push(SeatJob {
4068 spec,
4069 seat,
4070 prompt: text,
4071 cwd: root.join(seat_key),
4072 timeout,
4073 allow_write: false,
4074 sessions,
4075 artifacts: artifacts.clone(),
4076 stem: format!("vote-judge-{}-recover", j + 1),
4077 handover: Some(full),
4078 });
4079 vote_pos.push(j);
4080 }
4081 let allowed = labels.clone();
4082 let mut vote_losses = Vec::new();
4083 let vote_retries = self.state.config.graph.retries;
4084 let vote_cache = self.state.config.cache_dir();
4085 let ctx = WaveCtx {
4086 carry_seats: false,
4087 pauses: Some((&self.pause, &self.interrupt)),
4088 run: &run_id,
4089 node: "vote",
4090 prompts: &prompts,
4091 cache: vote_cache.as_deref(),
4092 round: None,
4093 };
4094 let votes = ask_json_wave::<FinalVote>(
4095 vote_jobs,
4096 Arc::clone(&self.sem),
4097 vote_retries,
4098 &self.roles.judge_roster,
4099 &ctx,
4100 &mut vote_losses,
4101 &mut self.state,
4102 &move |v: &FinalVote| match v.label() {
4103 Some(c) if allowed.contains(&c) => Ok(()),
4104 other => bail!("vote {other:?} is not one of {allowed:?}"),
4105 },
4106 )
4107 .await;
4108 if self.state.resource_block.is_some() {
4109 self.stop_for_resource()?;
4110 return Ok(false);
4111 }
4112 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4113 let agent_id = seat.agent.clone();
4114 self.state.seats.insert(seat.key.clone(), seat);
4115 match res {
4116 Ok((v, _)) => {
4117 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4118 rec.vote = v.label();
4119 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4120 } else {
4121 self.state.votes.push(VoteRecord {
4122 judge: j + 1,
4123 agent: agent_id,
4124 vote: v.label(),
4125 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4126 changed: false,
4127 });
4128 }
4129 self.state.event(
4130 "recover",
4131 format!("judge {} voted again after the limit", j + 1),
4132 );
4133 }
4134 Err(e) => {
4135 self.state
4136 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4137 }
4138 }
4139 }
4140
4141 let recovered_keys: BTreeSet<String> = recovered
4145 .iter()
4146 .map(|&j| format!("judge-{}", j + 1))
4147 .collect();
4148 self.state
4149 .quota
4150 .retain(|q| !recovered_keys.contains(&q.seat));
4151 for loss in judge_losses.into_iter().chain(vote_losses) {
4155 if recovered_keys.contains(&loss.seat) {
4156 continue;
4157 }
4158 self.state.quota.retain(|q| q.seat != loss.seat);
4159 self.state.quota.push(loss);
4160 }
4161
4162 self.state.tally = None;
4164 self.tally()?;
4165 Ok(self
4166 .state
4167 .tally
4168 .as_ref()
4169 .map(|t| t.met_quorum)
4170 .unwrap_or(false))
4171 }
4172
4173 async fn graft(&mut self) -> Result<()> {
4189 let Err(e) = self.graft_inner().await else {
4190 return Ok(());
4191 };
4192 if e.is::<crate::repo_lock::ResourceBlocked>() {
4193 return Err(e);
4194 }
4195 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4196 self.state.status = RunStatus::Blocked;
4197 self.state.event("graft", format!("blocked: {e:#}"));
4198 self.state.save()?;
4199 return Err(e);
4200 }
4201 if let Some(rec) = self.state.graft.clone()
4206 && !rec.outcome.settled()
4207 {
4208 if let Some(winner) = self.state.winner().cloned() {
4209 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4210 if re.is::<crate::repo_lock::ResourceBlocked>() {
4211 return Err(re);
4212 }
4213 self.state.status = RunStatus::Blocked;
4214 self.state.event("graft", format!("blocked: {re:#}"));
4215 self.state.save()?;
4216 return Err(re);
4217 }
4218 }
4219 if let Some(g) = self.state.graft.as_mut() {
4220 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4221 }
4222 }
4223 self.state.event("graft", format!("skipped: {e:#}"));
4224 if let Err(e) = self.state.save() {
4225 tracing::warn!("save after graft: {e:#}");
4226 }
4227 Ok(())
4228 }
4229
4230 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4233 let _coordination = self.coordinate(wt).await?;
4234 crate::recovery::restore_repair(wt, pre)
4235 .await
4236 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4237 }
4238
4239 fn graft_skipped(&mut self, why: &str) {
4241 let message = format!("skipped: {why}");
4242 if !self
4243 .state
4244 .events
4245 .iter()
4246 .any(|e| e.node == "graft" && e.message == message)
4247 {
4248 self.state.event("graft", message);
4249 }
4250 }
4251
4252 async fn graft_inner(&mut self) -> Result<()> {
4253 if let Some(rec) = self.state.graft.clone()
4256 && !rec.outcome.settled()
4257 {
4258 if self.state.reviews.is_empty()
4262 && let Some(winner) = self.state.winner().cloned()
4263 {
4264 self.graft_restore(&winner.worktree, &rec.pre).await?;
4265 }
4266 if rec.outcome == GraftOutcome::WaitingForRestore {
4267 self.state.graft = None;
4268 self.state.save()?;
4269 } else {
4270 if let Some(g) = self.state.graft.as_mut() {
4271 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4272 }
4273 self.state.event(
4274 "graft",
4275 "rolled back: the previous round was interrupted; the winner is untouched",
4276 );
4277 self.state.save()?;
4278 return Ok(());
4279 }
4280 }
4281 let plan = match graft::plan(&self.state) {
4282 Ok(plan) => plan,
4283 Err(skip) => {
4284 if skip != graft::Skip::AlreadyDone {
4285 self.graft_skipped(skip.reason());
4286 self.state.save()?;
4287 }
4288 return Ok(());
4289 }
4290 };
4291 let Some(winner) = self.state.winner().cloned() else {
4292 return Ok(());
4293 };
4294 let wt = winner.worktree.clone();
4295 if !git::is_clean(&wt).await? {
4296 self.graft_skipped("the winner's worktree has uncommitted changes");
4297 self.state.save()?;
4298 return Ok(());
4299 }
4300 let pre = git::rev_parse(&wt, "HEAD").await?;
4301 self.state.graft = Some(GraftRecord {
4304 pre: pre.clone(),
4305 post: None,
4306 lines: None,
4307 limit: None,
4308 agent: None,
4309 outcome: GraftOutcome::Running,
4310 requested: graft::requested(&plan),
4311 adoption: None,
4312 adoption_error: None,
4313 });
4314 self.state.event(
4315 "graft",
4316 format!(
4317 "porting strengths of candidate(s) {} onto {}",
4318 plan.strengths
4319 .iter()
4320 .map(|(l, _, _)| l.to_string())
4321 .collect::<Vec<_>>()
4322 .join(", "),
4323 plan.winner
4324 ),
4325 );
4326 self.state.save()?;
4327
4328 let fix_prompt = prompt::graft(
4329 &self.state.instruction,
4330 plan.winner,
4331 &plan.strengths,
4332 &self.state.config.graph.language,
4333 );
4334 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4335 let sessions = self.state.config.graph.sessions;
4336 let artifacts = agent::artifacts_dir(&self.state.dir());
4337 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4338 let (_, seat, out) = self
4339 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4340 SeatJob {
4341 prompt: fix_prompt.clone(),
4342 spec,
4343 seat,
4344 cwd: wt.clone(),
4345 timeout,
4346 allow_write: true,
4347 sessions,
4348 artifacts: artifacts.clone(),
4349 stem: "graft".to_owned(),
4350 handover: None,
4351 }
4352 })
4353 .await;
4354 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4355 self.state.graft.as_mut().expect("graft record").outcome =
4358 GraftOutcome::WaitingForRestore;
4359 return self.stop_for_resource();
4360 }
4361 let mut why: Option<String> = None;
4362 match &out {
4363 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4364 why = Some("the fixer produced nothing usable".to_owned());
4365 }
4366 AgentOutcome::Ok(_) => {}
4367 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4368 AgentOutcome::Quota(o) => {
4369 self.state.quota.push(QuotaLoss {
4370 seat: seat.key.clone(),
4371 node: "graft".to_owned(),
4372 at: Timestamp::now(),
4373 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4374 });
4375 why = Some("rate limited (quota)".to_owned());
4376 }
4377 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4378 why = Some(format!(
4379 "the fixer failed: {}",
4380 e.lines()
4381 .next()
4382 .unwrap_or("")
4383 .chars()
4384 .take(160)
4385 .collect::<String>()
4386 ));
4387 }
4388 }
4389 let agent_id = seat.agent.clone();
4390 self.state.seats.insert(seat.key.clone(), seat);
4391
4392 let mut post = None;
4393 let mut lines = None;
4394 let _coordination = self.coordinate(&wt).await?;
4395 if why.is_none() {
4396 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4397 if let Ok(r) = &rescued {
4398 self.state.note_withheld("graft", &r.withheld);
4399 }
4400 let head = git::rev_parse(&wt, "HEAD").await?;
4401 let dirty = match &rescued {
4405 Ok(r) => git::status_porcelain(&wt)
4406 .await
4407 .map(|st| {
4408 st.lines().any(|l| {
4409 !r.withheld
4410 .iter()
4411 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4412 })
4413 })
4414 .unwrap_or(true),
4415 Err(_) => true,
4416 };
4417 if dirty {
4418 why = Some("the fixer left changes that could not be committed".to_owned());
4419 } else if head == pre {
4420 why = Some("the fixer changed nothing".to_owned());
4421 } else {
4422 lines = git::changed_lines(&wt, &pre, &head).await;
4423 post = Some(head);
4424 }
4425 }
4426 let (adoption, adoption_error) = match &out {
4427 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4428 _ => (None, None),
4429 };
4430 let requested = graft::requested(&plan);
4431 let outcome = match why {
4432 None => {
4433 self.state.event(
4434 "graft",
4435 format!(
4436 "applied: {}",
4437 lines.map_or_else(
4438 || "changed lines unavailable".to_owned(),
4439 |n| format!("{n} changed lines")
4440 )
4441 ),
4442 );
4443 GraftOutcome::Applied
4444 }
4445 Some(why) => {
4446 crate::recovery::restore_repair(&wt, &pre)
4449 .await
4450 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4451 self.state.event(
4452 "graft",
4453 format!("rolled back: {why}; the winner is untouched"),
4454 );
4455 GraftOutcome::RolledBack(why)
4456 }
4457 };
4458 self.state.graft = Some(GraftRecord {
4459 pre,
4460 post,
4461 lines,
4462 limit: None,
4463 agent: Some(agent_id),
4464 outcome,
4465 requested,
4466 adoption,
4467 adoption_error,
4468 });
4469 self.state.save()?;
4470 Ok(())
4471 }
4472
4473 async fn fold_losers(&mut self) -> Result<()> {
4476 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4477 return Ok(());
4478 };
4479 if !self.state.candidate_preservation.is_empty() {
4480 bail!("candidate preservation incomplete; refusing cleanup");
4481 }
4482 let repo = self.state.repo.clone();
4483 let _coordination = self.coordinate(&repo).await?;
4484 let mut folded = Vec::new();
4485 for i in 0..self.state.candidates.len() {
4486 let c = &self.state.candidates[i];
4487 if c.label == winner || c.folded {
4488 continue;
4489 }
4490 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4491 remove_preserved_worktree(&repo, &wt).await?;
4492 if git::branch_exists(&repo, &branch).await?
4493 && !git::branch_delete(&repo, &branch).await?
4494 {
4495 bail!("candidate branch removal refused");
4496 }
4497 self.state.candidates[i].folded = true;
4498 self.state.save()?;
4499 folded.push(label.to_string());
4500 }
4501 let root = self.state.worktree_root();
4503 for j in 1..=self.roles.judges.len() {
4504 let wt = root.join(format!("judge-{j}"));
4505 if wt.exists() {
4506 remove_preserved_worktree(&repo, &wt).await?;
4507 }
4508 }
4509 if self.state.config.graph.advise {
4512 for k in 1..=self.state.config.graph.advisors {
4513 let wt = root.join(format!("advisor-{k}"));
4514 if wt.exists() {
4515 remove_preserved_worktree(&repo, &wt).await?;
4516 }
4517 }
4518 }
4519 if !folded.is_empty() {
4520 self.state
4521 .event("fold", format!("folded candidates {}", folded.join(", ")));
4522 self.state.save()?;
4523 }
4524 Ok(())
4525 }
4526
4527 async fn sync_to_base(&mut self) -> Result<()> {
4567 if self.state.status == RunStatus::AlreadyInBase {
4568 return Ok(());
4569 }
4570 let conflicted = self
4571 .state
4572 .base_sync
4573 .as_ref()
4574 .is_some_and(|s| s.conflict.is_some());
4575 let Some(winner) = self.state.winner().cloned() else {
4576 return Ok(());
4577 };
4578
4579 let repo = self.state.repo.clone();
4580 let remote = self.state.config.merge.remote.clone();
4581 let base_branch = self.state.base_branch.clone();
4582 let tracking = format!("{remote}/{base_branch}");
4583
4584 git::fetch(&repo, &remote, &base_branch).await.ok();
4585 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4590 return Ok(());
4591 };
4592
4593 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4594 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4595 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4596
4597 if (behind > 0 || conflicted || head == tip)
4606 && self
4607 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4608 .await?
4609 {
4610 return Ok(());
4611 }
4612 if conflicted {
4613 return Ok(());
4614 }
4615
4616 if behind == 0 {
4617 if let Some(from) = self
4626 .state
4627 .rebase_fixes
4628 .iter()
4629 .rev()
4630 .find_map(|r| r.from.clone())
4631 && from != head
4632 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4633 .await
4634 .is_ok_and(|o| o.ok())
4635 {
4636 git::sync_to_head(&winner.worktree).await?;
4637 }
4638 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4644 if let Some(why) = &conflict {
4645 self.state.status = RunStatus::Blocked;
4646 self.state.event("land", why.clone());
4647 }
4648 self.state.base_sync = Some(BaseSync {
4649 tip,
4650 behind: 0,
4651 attempts,
4652 conflict,
4653 already_in: None,
4654 });
4655 self.state.save()?;
4656 return Ok(());
4657 }
4658
4659 if attempts >= BASE_SYNC_ROUNDS {
4660 let why = format!(
4661 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4662 rebase(s); rebasing again would only race it",
4663 winner.branch
4664 );
4665 self.state.status = RunStatus::Blocked;
4666 self.state.base_sync = Some(BaseSync {
4667 tip,
4668 behind,
4669 attempts,
4670 conflict: Some(why.clone()),
4671 already_in: None,
4672 });
4673 self.state.event("land", why);
4674 self.state.save()?;
4675 return Ok(());
4676 }
4677
4678 self.state.event(
4679 "land",
4680 format!(
4681 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4682 winner.branch
4683 ),
4684 );
4685 self.state.save()?;
4686
4687 let branch_tracking = format!("{remote}/{}", winner.branch);
4693 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4694 let remote_tip = if self.state.task_pr_branch().is_none()
4695 && matches!(&fetched_branch, Ok(o) if o.ok())
4696 {
4697 git::rev_parse(&repo, &branch_tracking).await.ok()
4698 } else {
4699 None
4700 };
4701 if let Some(theirs) = &remote_tip
4705 && !git::is_ancestor(&repo, theirs, &head).await
4706 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4707 .await
4708 .is_ok_and(|missing| missing.is_empty())
4709 {
4710 let why = format!(
4711 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4712 them",
4713 short(theirs),
4714 winner.branch
4715 );
4716 self.state.status = RunStatus::Blocked;
4717 self.state.base_sync = Some(BaseSync {
4718 tip,
4719 behind,
4720 attempts,
4721 conflict: Some(why.clone()),
4722 already_in: None,
4723 });
4724 self.state.event("land", why);
4725 self.state.save()?;
4726 return Ok(());
4727 }
4728
4729 let scratch = self.state.dir().join("base-sync");
4730 let rebased = match crate::rebase::rebase_with_fixer(
4731 &mut self.state,
4732 &scratch,
4733 &winner.branch,
4734 &tracking,
4735 )
4736 .await
4737 {
4738 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4739 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4740 Err(e) => Err(e),
4741 };
4742 let attempts = attempts + 1;
4743 match rebased {
4744 Ok(None) => {
4745 git::sync_to_head(&winner.worktree).await?;
4749 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4750 let mut conflict = None;
4751 if let Some(pinned) = &remote_tip {
4752 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4753 match pushed {
4754 Ok(o) if o.ok() => self.state.event(
4755 "land",
4756 format!("pushed rebased {} to {remote}", winner.branch),
4757 ),
4758 Ok(o) => {
4759 conflict = Some(format!(
4760 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4761 winner.branch,
4762 short(pinned),
4763 o.stderr.chars().take(600).collect::<String>()
4764 ));
4765 }
4766 Err(e) => {
4767 conflict = Some(format!(
4768 "rebased {} locally but could not push it: {e:#}",
4769 winner.branch
4770 ));
4771 }
4772 }
4773 }
4774 if let Some(why) = &conflict {
4775 self.state.status = RunStatus::Blocked;
4776 self.state.event("land", why.clone());
4777 }
4778 self.state.base_sync = Some(BaseSync {
4779 tip: tip.clone(),
4780 behind: 0,
4781 attempts,
4782 conflict,
4783 already_in: None,
4784 });
4785 self.state
4786 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4787 }
4788 Ok(Some(conflict)) => {
4789 let why = format!(
4790 "{} conflicts with {tracking} and did not rebase: {}",
4791 winner.branch,
4792 conflict.chars().take(600).collect::<String>()
4793 );
4794 self.state.status = RunStatus::Blocked;
4795 self.state.base_sync = Some(BaseSync {
4796 tip,
4797 behind,
4798 attempts,
4799 conflict: Some(why.clone()),
4800 already_in: None,
4801 });
4802 self.state.event("land", why);
4803 }
4804 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4808 self.state.save()?;
4809 return Ok(());
4810 }
4811 Err(e) => {
4812 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4813 self.state.status = RunStatus::Blocked;
4814 self.state.base_sync = Some(BaseSync {
4815 tip,
4816 behind,
4817 attempts,
4818 conflict: Some(why.clone()),
4819 already_in: None,
4820 });
4821 self.state.event("land", why);
4822 }
4823 }
4824 self.state.save()?;
4825 Ok(())
4826 }
4827
4828 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4834 if self.state.task_pr_branch().is_some() {
4837 return None;
4838 }
4839 let repo = self.state.repo.clone();
4840 let remote = self.state.config.merge.remote.clone();
4841 let fetched = git::fetch(&repo, &remote, branch).await;
4842 if !matches!(&fetched, Ok(o) if o.ok()) {
4843 return None;
4844 }
4845 let branch_tracking = format!("{remote}/{branch}");
4846 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4847 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4848 return None;
4849 }
4850 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4851 .await
4852 .is_ok_and(|missing| missing.is_empty())
4853 {
4854 return Some(format!(
4855 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4856 them",
4857 short(&theirs)
4858 ));
4859 }
4860 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4861 Ok(o) if o.ok() => {
4862 self.state
4863 .event("land", format!("pushed rebased {branch} to {remote}"));
4864 None
4865 }
4866 Ok(o) => Some(format!(
4867 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4868 someone may have pushed): {}",
4869 short(&theirs),
4870 o.stderr.chars().take(600).collect::<String>()
4871 )),
4872 Err(e) => Some(format!(
4873 "{branch} is rebased locally but could not be pushed: {e:#}"
4874 )),
4875 }
4876 }
4877
4878 async fn settle_already_in(
4895 &mut self,
4896 branch: &str,
4897 tip: &str,
4898 head: &str,
4899 attempts: usize,
4900 behind: usize,
4901 ) -> Result<bool> {
4902 let repo = self.state.repo.clone();
4903 let remote = self.state.config.merge.remote.clone();
4904 let start = self.state.base_commit.clone();
4905 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4906 Ok(Some(e)) => e,
4907 Ok(None) => return Ok(false),
4908 Err(e) => {
4909 tracing::warn!("already-in-base check for {branch}: {e:#}");
4910 return Ok(false);
4911 }
4912 };
4913 let mut verified = vec![head.to_owned()];
4914 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4915 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4916 if matches!(&fetched, Ok(o) if o.ok())
4917 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4918 && theirs != head
4919 {
4920 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4921 Ok(Some(_)) => verified.push(theirs),
4922 _ => return Ok(false),
4923 }
4924 }
4925 let base_branch = self.state.base_branch.clone();
4926 let message = format!(
4927 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4928 land",
4929 evidence.names(),
4930 evidence.proof.as_str()
4931 );
4932 let closed =
4933 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4934 .await;
4935 match closed {
4936 Ok(Ok(url)) => self
4937 .state
4938 .event("land", format!("closed {url}: superseded on {base_branch}")),
4939 Ok(Err(why)) => self
4940 .state
4941 .event("land", format!("did not close a pull request: {why}")),
4942 Err(e) => {
4943 let why = format!(
4944 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4945 not be closed ({e:#}); resume to retry"
4946 );
4947 self.state.status = RunStatus::Blocked;
4948 self.state.base_sync = Some(BaseSync {
4949 tip: tip.to_owned(),
4950 behind,
4951 attempts,
4952 conflict: Some(why.clone()),
4953 already_in: None,
4954 });
4955 self.state.event("land", why);
4956 self.state.save()?;
4957 return Ok(true);
4958 }
4959 }
4960 self.state.status = RunStatus::AlreadyInBase;
4961 self.state.base_sync = Some(BaseSync {
4962 tip: tip.to_owned(),
4963 behind,
4964 attempts,
4965 conflict: None,
4966 already_in: Some(evidence),
4967 });
4968 self.state.event("land", message);
4969 self.state.save()?;
4970 self.settle_questions();
4971 Ok(true)
4972 }
4973
4974 fn landing_base(&self) -> String {
4984 self.state
4985 .base_sync
4986 .as_ref()
4987 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
4988 }
4989
4990 pub async fn fix_selected(
5023 &mut self,
5024 ids: &[String],
5025 reason: &str,
5026 allow_stale: bool,
5027 ) -> Result<()> {
5028 let reason = reason.trim();
5029 if reason.is_empty() {
5030 bail!("a fix request needs a reason — that is the operator's own record of why");
5031 }
5032 if ids.is_empty() {
5033 bail!("no finding id given");
5034 }
5035 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5036 bail!(
5037 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5038 has already concluded — can be given a targeted fix. A run still \
5039 in progress should simply be resumed; a `merged` run's branch has \
5040 already landed, so its answer is a fresh `magi review <branch>`, \
5041 not reopening this run's own record",
5042 self.state.id,
5043 self.state.status.as_str()
5044 );
5045 }
5046 let Some(winner) = self.state.winner().cloned() else {
5047 bail!("run {} has no winning candidate to fix", self.state.id);
5048 };
5049 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5050 bail!(
5051 "branch `{}` no longer exists; this run cannot be extended",
5052 winner.branch
5053 );
5054 }
5055 let home = crate::run::home();
5056 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5057 bail!(
5058 "run {} is currently being worked on by another magi process",
5059 self.state.id
5060 );
5061 }
5062 let _claim = FixClaim::acquire(&self.state.dir())?;
5068
5069 let mut seen = BTreeSet::new();
5073 let mut findings = Vec::new();
5074 let mut missing = Vec::new();
5075 for id in ids {
5076 if !seen.insert(id.clone()) {
5077 continue;
5078 }
5079 match self.state.finding(id) {
5080 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5081 id: f.id.clone(),
5082 severity: f.severity,
5083 reviewer_vote: rec.vote,
5084 round: round.round,
5085 round_head: round.head.clone(),
5086 reviewer: rec.reviewer,
5087 agent: rec.agent.clone(),
5088 file: f.file.clone(),
5089 line: f.line,
5090 title: f.title.clone(),
5091 detail: f.detail.clone(),
5092 outcome: OperatorFixOutcome::Pending,
5093 }),
5094 None => missing.push(id.clone()),
5095 }
5096 }
5097 if !missing.is_empty() {
5098 bail!(
5099 "unknown finding id(s): {}; nothing was changed",
5100 missing.join(", ")
5101 );
5102 }
5103
5104 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5105 let stale_details: Vec<(String, String)> = findings
5106 .iter()
5107 .filter(|f| f.round_head != head_at_request)
5108 .map(|f| (f.id.clone(), f.round_head.clone()))
5109 .collect();
5110 let stale = !stale_details.is_empty();
5111 if stale && !allow_stale {
5112 bail!(
5113 "the branch has moved since some finding(s) were raised — {} — now \
5114 at {}; pass --allow-stale to fix anyway, or re-run review first",
5115 stale_details
5116 .iter()
5117 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5118 .collect::<Vec<_>>()
5119 .join(", "),
5120 short(&head_at_request)
5121 );
5122 }
5123
5124 let request = OperatorFixRequest {
5125 requested_at: Timestamp::now(),
5126 reason: reason.to_owned(),
5127 findings,
5128 head_at_request: head_at_request.clone(),
5129 allow_stale,
5130 stale,
5131 fix: None,
5132 result_head: None,
5133 follow_up_review_run: None,
5134 };
5135 self.state.event(
5136 "fix",
5137 format!(
5138 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5139 request.findings.len(),
5140 request
5141 .findings
5142 .iter()
5143 .map(|f| f.id.as_str())
5144 .collect::<Vec<_>>()
5145 .join(", "),
5146 ),
5147 );
5148 self.state.operator_fixes.push(request);
5155 self.state.save()?;
5156 let request_index = self.state.operator_fixes.len() - 1;
5157
5158 let repo = self.state.repo.clone();
5167 let coordination = self.coordinate(&repo).await?;
5168 if winner.worktree.exists() {
5169 let dirty = git::git(
5172 &winner.worktree,
5173 &["status", "--porcelain", "--untracked-files=all"],
5174 )
5175 .await?;
5176 let only_withheld = dirty.lines().all(|l| {
5177 l.strip_prefix("?? ")
5178 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5179 });
5180 if !only_withheld {
5181 bail!(
5182 "`{}` has uncommitted changes; refusing to touch it — commit or \
5183 discard them first",
5184 winner.worktree.display()
5185 );
5186 }
5187 git::worktree_remove(&self.state.repo, &winner.worktree)
5188 .await
5189 .ok();
5190 }
5191 let fix_worktree = self.state.worktree_root().join("operator-fix");
5192 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5193 git::git(
5194 &self.state.repo,
5195 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5196 )
5197 .await
5198 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5199 if !git::is_clean(&fix_worktree).await? {
5200 git::worktree_remove(&self.state.repo, &fix_worktree)
5201 .await
5202 .ok();
5203 bail!(
5204 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5205 winner.branch
5206 );
5207 }
5208 let repo_for_setup = self.state.repo.clone();
5209 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5210 Ok(()) => {
5211 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5212 }
5213 Err(e) => Err(e),
5214 };
5215 if let Err(e) = setup {
5216 git::worktree_remove(&self.state.repo, &fix_worktree)
5217 .await
5218 .ok();
5219 return Err(e);
5220 }
5221
5222 let run_id = self.state.id.clone();
5223 let prompts = self.state.config.prompts.clone();
5224 let language = self.state.config.graph.language.clone();
5225 let sessions = self.state.config.graph.sessions;
5226 let artifacts = agent::artifacts_dir(&self.state.dir());
5227 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5228 .findings
5229 .iter()
5230 .map(|f| Finding {
5231 id: f.id.clone(),
5232 severity: f.severity,
5233 file: f.file.clone(),
5234 line: f.line,
5235 title: f.title.clone(),
5236 detail: f.detail.clone(),
5237 })
5238 .collect();
5239 let fix_prompt = prompt::operator_fix_in(
5240 &self.state.instruction,
5241 &finding_list,
5242 reason,
5243 &stale_details,
5244 &head_at_request,
5245 &language,
5246 &self.state.config.graph.github_language,
5247 );
5248 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5249 drop(coordination);
5250 let (job, seat, out) = self
5251 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5252 prompt: fix_prompt.clone(),
5253 spec,
5254 seat,
5255 cwd: fix_worktree.clone(),
5256 timeout,
5257 allow_write: true,
5258 sessions,
5259 artifacts: artifacts.clone(),
5260 stem: "operator-fix".to_owned(),
5261 handover: None,
5262 })
5263 .await;
5264 let agent_id = seat.agent.clone();
5265
5266 let mut fix = FixRecord {
5267 agent: agent_id,
5268 addressed: Vec::new(),
5269 rejected: Vec::new(),
5270 notes: String::new(),
5271 committed: false,
5272 failed: None,
5273 duration_ms: 0,
5274 continuation: None,
5275 };
5276 let mut final_seat = seat.clone();
5277 match out {
5278 AgentOutcome::Ok(o) => {
5279 fix.duration_ms = o.duration_ms;
5280 let parsed = verdict::extract_json::<FixReport>(&o.text);
5281 let incomplete_reason = match &parsed {
5282 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5283 "the reply parsed, but it reported a command whose own CLI \
5284 never confirmed an exit status"
5285 .to_owned(),
5286 ),
5287 Ok(_) => None,
5288 Err(e) => Some(e.to_string()),
5289 };
5290 match incomplete_reason {
5291 None => {
5292 let report = parsed.expect("checked Ok above");
5293 fix.addressed = report.addressed;
5294 fix.rejected = report.rejected;
5295 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5296 }
5297 Some(reason) => {
5298 let (resumed_seat, resolved, failure, cont) = self
5299 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5300 .await;
5301 fix.duration_ms += cont.cumulative_wait_ms;
5302 fix.continuation = Some(cont);
5303 final_seat = resumed_seat;
5304 match resolved {
5305 Some(report) => {
5306 fix.addressed = report.addressed;
5307 fix.rejected = report.rejected;
5308 fix.notes =
5309 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5310 }
5311 None => fix.failed = failure,
5312 }
5313 }
5314 }
5315 }
5316 AgentOutcome::Dropped(o) => {
5317 fix.duration_ms = o.duration_ms;
5318 let why = o
5319 .dropped
5320 .as_ref()
5321 .map(|d| d.why.as_str())
5322 .unwrap_or("the CLI ended the stream without delivering its answer");
5323 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5324 }
5325 AgentOutcome::Quota(o) => {
5326 self.state.quota.push(QuotaLoss {
5327 seat: final_seat.key.clone(),
5328 node: "fix".to_owned(),
5329 at: Timestamp::now(),
5330 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5331 });
5332 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5333 }
5334 AgentOutcome::Failed(e) => fix.failed = Some(e),
5335 AgentOutcome::ResourceBlocked(e) => {
5336 fix.failed = Some(format!("the fixer was not started: {e}"));
5337 }
5338 }
5339 if fix.continuation.is_none() {
5340 fix.continuation = Some(ContinuationRecord::not_needed());
5341 }
5342 self.state.seats.insert(final_seat.key.clone(), final_seat);
5343
5344 let rescue_message = format!(
5345 "magi: operator-selected fix ({}) (uncommitted work)",
5346 self.state.operator_fixes[request_index]
5347 .findings
5348 .iter()
5349 .map(|f| f.id.as_str())
5350 .collect::<Vec<_>>()
5351 .join(", ")
5352 );
5353 let repo = self.state.repo.clone();
5354 let coordination = self.coordinate(&repo).await?;
5355 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5356 self.state.note_withheld("fix", &r.withheld);
5357 }
5358 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5359 fix.committed = after != head_at_request;
5360 git::worktree_remove(&self.state.repo, &fix_worktree)
5361 .await
5362 .ok();
5363 drop(coordination);
5364
5365 self.state.event(
5366 "fix",
5367 match &fix.failed {
5368 Some(reason) => format!(
5369 "operator fix: adoption report was lost ({reason}); {}",
5370 if fix.committed {
5371 "committed"
5372 } else {
5373 "NO new commit"
5374 }
5375 ),
5376 None => format!(
5377 "operator fix: {} addressed, {} rejected, {}",
5378 fix.addressed.len(),
5379 fix.rejected.len(),
5380 if fix.committed {
5381 "committed"
5382 } else {
5383 "NO new commit"
5384 }
5385 ),
5386 },
5387 );
5388
5389 for f in &mut self.state.operator_fixes[request_index].findings {
5396 f.outcome = if fix.failed.is_some() {
5397 OperatorFixOutcome::Unreported
5398 } else if fix.addressed.contains(&f.id) {
5399 OperatorFixOutcome::Addressed
5400 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5401 OperatorFixOutcome::Rejected { why: r.why.clone() }
5402 } else {
5403 OperatorFixOutcome::Unreported
5404 };
5405 }
5406
5407 let committed = fix.committed;
5408 if committed {
5409 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5410 }
5411 self.state.operator_fixes[request_index].fix = Some(fix);
5412 self.state.save()?;
5415
5416 if committed {
5417 self.state.event(
5418 "fix",
5419 format!(
5420 "operator fix committed {}; opening a follow-up review-only run",
5421 short(&after)
5422 ),
5423 );
5424 let origin =
5427 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5428 match Self::review(
5429 &self.state.repo,
5430 &winner.branch,
5431 self.state.config.clone(),
5432 origin,
5433 )
5434 .await
5435 {
5436 Ok(mut follow_up) => {
5437 follow_up.state.event(
5438 "start",
5439 format!(
5440 "requested by an operator fix on run {} for finding(s) {}",
5441 self.state.id,
5442 self.state.operator_fixes[request_index]
5443 .findings
5444 .iter()
5445 .map(|f| f.id.as_str())
5446 .collect::<Vec<_>>()
5447 .join(", "),
5448 ),
5449 );
5450 follow_up.state.save()?;
5451 let follow_up_id = follow_up.state.id.clone();
5452 let adopted = match crate::direct::adopt(
5455 &follow_up.state,
5456 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5457 ) {
5458 Ok(a) => a,
5459 Err(e) => {
5460 self.state.event(
5463 "fix",
5464 format!(
5465 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5466 ),
5467 );
5468 self.state.save()?;
5469 return Ok(());
5470 }
5471 };
5472 let executed = follow_up.execute().await;
5473 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5474 if let Some(a) = adopted {
5475 a.finish(&follow_up.state, executed);
5476 }
5477 if let Some(e) = failure {
5478 self.state.event(
5479 "fix",
5480 format!(
5481 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5482 ),
5483 );
5484 }
5485 self.state.operator_fixes[request_index].follow_up_review_run =
5486 Some(follow_up_id);
5487 }
5488 Err(e) => {
5489 self.state.event(
5490 "fix",
5491 format!("committed the fix but could not open a follow-up review: {e:#}"),
5492 );
5493 }
5494 }
5495 self.state.save()?;
5496 }
5497
5498 Ok(())
5499 }
5500
5501 async fn ask_fixer(
5521 &mut self,
5522 winner: &Candidate,
5523 node: &'static str,
5524 round: Option<usize>,
5525 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5526 ) -> (SeatJob, SeatState, AgentOutcome) {
5527 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5528 self.ask_fixer_among(attempts, node, round, None, build)
5529 .await
5530 }
5531
5532 async fn ask_fixer_among(
5537 &mut self,
5538 attempts: Vec<(AgentSpec, String)>,
5539 node: &'static str,
5540 round: Option<usize>,
5541 restore: Option<(&Path, &str)>,
5542 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5543 ) -> (SeatJob, SeatState, AgentOutcome) {
5544 let run_id = self.state.id.clone();
5545 let prompts = self.state.config.prompts.clone();
5546 let cache = self.state.config.cache_dir();
5547 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5548 let mut last = None;
5549 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5550 let seat = self.seat(&key, &spec.id);
5551 let mut job = build(spec.clone(), seat);
5552 if i > 0 {
5553 job.stem = format!("{}-{}", job.stem, spec.id);
5554 }
5555 let ctx = WaveCtx {
5556 carry_seats: false,
5557 pauses: Some((&self.pause, &self.interrupt)),
5558 run: &run_id,
5559 node,
5560 prompts: &prompts,
5561 cache: cache.as_deref(),
5562 round,
5563 };
5564 let (seat, out) =
5565 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5566 let advances = match &out {
5569 AgentOutcome::Ok(o) => agent::output_advances(o),
5570 AgentOutcome::ResourceBlocked(_) => false,
5571 _ => true,
5572 };
5573 if let Some(next) = ids.get(i + 1)
5574 && advances
5575 {
5576 self.state.seats.insert(seat.key.clone(), seat.clone());
5577 let class =
5578 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5579 let reason = match &out {
5580 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5581 other => fail_reason(other),
5582 };
5583 if let Some((wt, rev)) = restore
5587 && let Err(e) = self.graft_restore(wt, rev).await
5588 {
5589 let why =
5590 format!("could not restore the worktree before the next fixer: {e:#}");
5591 tracing::warn!("{why}");
5592 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5593 AgentOutcome::ResourceBlocked(why)
5594 } else {
5595 AgentOutcome::Failed(why)
5596 };
5597 last = Some((job, seat, out));
5598 break;
5599 }
5600 record_handover(
5601 &mut self.state,
5602 node,
5603 &seat.key,
5604 &spec.id,
5605 next,
5606 &class,
5607 &reason,
5608 );
5609 continue;
5610 }
5611 last = Some((job, seat, out));
5612 break;
5613 }
5614 last.expect("the fixer chain always has an entry")
5615 }
5616
5617 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5628 use crate::recovery::{Mode, Repair};
5629 if !self.state.reviews.is_empty() {
5630 return Ok(());
5631 }
5632 let Some(mut rec) = self.state.recovery.clone() else {
5633 return Ok(());
5634 };
5635 if rec.mode != Mode::VerifyRepair {
5636 return Ok(());
5637 }
5638 let wt = winner.worktree.clone();
5639 let mut head = git::rev_parse(&wt, "HEAD").await?;
5640 match rec.repair.clone() {
5641 Repair::Done { .. } => return Ok(()),
5642 Repair::WaitingForRestore { before } => {
5643 self.graft_restore(&wt, &before).await?;
5644 head = before;
5645 rec.repair = Repair::NotStarted;
5646 self.state.recovery = Some(rec.clone());
5647 self.state.save()?;
5648 }
5649 Repair::Started { before } => {
5650 if let Err(e) = self.graft_restore(&wt, &before).await {
5654 if e.is::<crate::repo_lock::ResourceBlocked>() {
5655 return Err(e);
5656 }
5657 return self
5658 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5659 .await;
5660 }
5661 rec.repair = Repair::Done {
5662 before: before.clone(),
5663 after: before,
5664 committed: false,
5665 failed: Some("interrupted; rolled back".to_owned()),
5666 };
5667 self.state
5668 .event("recovery", "the repair was interrupted and rolled back");
5669 self.state.recovery = Some(rec);
5670 self.state.save()?;
5671 return Ok(());
5672 }
5673 Repair::NotStarted => {}
5674 }
5675 if crate::recovery::recheck(&mut rec, &head) {
5676 self.state.event("recovery", rec.summary());
5677 self.state.recovery = Some(rec);
5678 self.state.save()?;
5679 return Ok(());
5680 }
5681 rec.repair = Repair::Started {
5682 before: head.clone(),
5683 };
5684 self.state.recovery = Some(rec.clone());
5685 self.state.save()?;
5686
5687 let language = self.state.config.graph.language.clone();
5688 let sessions = self.state.config.graph.sessions;
5689 let artifacts = agent::artifacts_dir(&self.state.dir());
5690 let fix_prompt = prompt::recovery_fix(
5691 &self.state.instruction,
5692 &rec,
5693 self.state.config.graph.review_rounds,
5694 &language,
5695 );
5696 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5697 self.state.event(
5698 "recovery",
5699 format!(
5700 "repair: the fixer starts from the failed verification on {}",
5701 short(&head)
5702 ),
5703 );
5704 let (_job, seat, out) = self
5705 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5706 prompt: fix_prompt.clone(),
5707 spec,
5708 seat,
5709 cwd: wt.clone(),
5710 timeout,
5711 allow_write: true,
5712 sessions,
5713 artifacts: artifacts.clone(),
5714 stem: "fix-recovery".to_owned(),
5715 handover: None,
5716 })
5717 .await;
5718 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5719 rec.repair = Repair::WaitingForRestore { before: head };
5720 self.state.recovery = Some(rec);
5721 return self.stop_for_resource();
5722 }
5723 let failed = match &out {
5724 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5725 AgentOutcome::Quota(o) => {
5726 self.state.quota.push(QuotaLoss {
5727 seat: seat.key.clone(),
5728 node: "fix".to_owned(),
5729 at: Timestamp::now(),
5730 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5731 });
5732 Some("rate limited (quota); the fixer could not run".to_owned())
5733 }
5734 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5735 AgentOutcome::Failed(e) => Some(e.clone()),
5736 };
5737 self.state.seats.insert(seat.key.clone(), seat);
5738 let coordination = self.coordinate(&wt).await?;
5739 let failed = match failed {
5740 Some(why) => {
5742 if let Err(e) = git::restore_to(&wt, &head).await {
5743 return self
5744 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5745 .await;
5746 }
5747 Some(why)
5748 }
5749 None => {
5750 if let Ok(r) = git::rescue_commit(
5751 &wt,
5752 "magi: recovery repair of the failed verification (uncommitted work)",
5753 )
5754 .await
5755 {
5756 self.state.note_withheld("fix", &r.withheld);
5757 }
5758 None
5759 }
5760 };
5761 let after = git::rev_parse(&wt, "HEAD").await?;
5762 drop(coordination);
5763 self.state.event(
5764 "recovery",
5765 match &failed {
5766 Some(why) => format!("repair: nothing kept ({why})"),
5767 None if after != head => {
5768 format!("repair: committed {}", short(&after))
5769 }
5770 None => "repair: the fixer made no commit".to_owned(),
5771 },
5772 );
5773 rec.repair = Repair::Done {
5774 before: head.clone(),
5775 committed: after != head,
5776 after,
5777 failed,
5778 };
5779 self.state.recovery = Some(rec);
5780 self.state.save()?;
5781 Ok(())
5782 }
5783
5784 async fn stop_recovery(
5787 &mut self,
5788 mut rec: crate::recovery::RecoveryRecord,
5789 head: &str,
5790 why: String,
5791 ) -> Result<()> {
5792 rec.repair = crate::recovery::Repair::Started {
5794 before: head.to_owned(),
5795 };
5796 self.state.recovery = Some(rec);
5797 self.state
5798 .event("recovery", format!("repair could not be undone: {why}"));
5799 self.state.status = RunStatus::Blocked;
5800 self.state.save()?;
5801 bail!("recovery repair could not be rolled back: {why}")
5802 }
5803
5804 async fn review_loop(&mut self) -> Result<()> {
5805 if self
5810 .state
5811 .base_sync
5812 .as_ref()
5813 .is_some_and(|s| s.conflict.is_some())
5814 {
5815 return Ok(());
5816 }
5817 let run_id = self.state.id.clone();
5822 let prompts = self.state.config.prompts.clone();
5823 let Some(winner) = self.state.winner().cloned() else {
5824 return Ok(());
5825 };
5826 let max_rounds = self.state.config.graph.review_rounds;
5827 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5837 if status == RunStatus::Gating {
5840 self.record_contested_handoff();
5841 }
5842 self.state.status = status;
5843 self.state.save()?;
5844 return Ok(());
5845 }
5846 self.state.status = RunStatus::Reviewing;
5847 if self
5857 .state
5858 .reviews
5859 .last()
5860 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5861 {
5862 let shell = self.state.config.shell();
5863 return self
5864 .stop_reviewing(
5865 "the last round's own verification never resolved",
5866 &shell,
5867 &winner.worktree,
5868 )
5869 .await;
5870 }
5871
5872 self.recovery_repair(&winner).await?;
5873 if self.resource_halt()? {
5874 return Ok(());
5875 }
5876
5877 let repo = self.state.repo.clone();
5878 let root = self.state.worktree_root();
5879 let language = self.state.config.graph.language.clone();
5880 let sessions = self.state.config.graph.sessions;
5881 let artifacts = self.merge_repair_artifacts();
5882 let base = self.landing_base();
5883 let base_short = short(&base);
5884 let reviewers = self.roles.reviewers.clone();
5885 let shell = self.state.config.shell();
5886
5887 for round in (self.state.reviews.len() + 1)..=max_rounds {
5888 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5889 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5890 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5891 let prev_verification = self
5900 .state
5901 .reviews
5902 .last()
5903 .and_then(|r| r.verification_summary(&head));
5904 let prior_section = match (&self.state.recovery, round) {
5907 (Some(rec), 1) if self.state.reviews.is_empty() => {
5908 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5909 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5910 Some(crate::recovery::head_of(
5911 &git::diff(&winner.worktree, reviewed, "HEAD")
5912 .await
5913 .unwrap_or_default(),
5914 crate::recovery::SINCE_DIFF_MAX,
5915 ))
5916 }
5917 _ => None,
5918 };
5919 Some(prompt::recovery_review(rec, since.as_deref()))
5920 }
5921 _ => None,
5922 };
5923
5924 let mut jobs = Vec::new();
5928 let coordination = self.coordinate(&repo).await?;
5929 for (r, spec) in reviewers.iter().cloned().enumerate() {
5930 let wt = root.join(format!("review-{}", r + 1));
5931 if wt.exists() {
5934 git::reset_detached(&wt, &head).await?;
5935 } else {
5936 git::worktree_add_detached(&repo, &wt, &head).await?;
5937 }
5938 self.setup_seat_worktree(&repo, &wt, false).await?;
5939 let seat_key = format!("review-{}", r + 1);
5940 let spec = pick_start_spec(
5943 &self.roles.reviewer_roster,
5944 spec,
5945 self.state.seat_history.get(&seat_key),
5946 );
5947 let seat = self.seat(&seat_key, &spec.id);
5948 jobs.push(SeatJob {
5949 prompt: prompt::review(&prompt::ReviewCtx {
5950 instruction: &self.state.instruction,
5951 branch: &winner.branch,
5952 base_short: &base_short,
5953 stat: &stat,
5954 patch: &patch,
5955 verification: prev_verification.as_ref(),
5956 prior: prior_section.as_deref(),
5957 reviewers: reviewers.len(),
5958 round,
5959 rounds: max_rounds,
5960 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5963 lens: Lens::for_seat(r),
5964 language: &language,
5965 github_language: &self.state.config.graph.github_language,
5966 }),
5967 spec,
5968 seat,
5969 cwd: wt,
5970 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5971 allow_write: false,
5972 sessions,
5973 artifacts: artifacts.clone(),
5974 stem: format!("review-{round}-{}", r + 1),
5975 handover: None,
5976 });
5977 }
5978
5979 drop(coordination);
5980 self.state.event(
5981 "review",
5982 format!(
5983 "round {round}: {} reviewers on {}",
5984 jobs.len(),
5985 short(&head)
5986 ),
5987 );
5988 let mut quota_losses = Vec::new();
5989 let review_retries = self.state.config.graph.retries;
5990 let review_cache = self.state.config.cache_dir();
5991 let ctx = WaveCtx {
5992 carry_seats: true,
5993 pauses: Some((&self.pause, &self.interrupt)),
5994 run: &run_id,
5995 node: "review",
5996 prompts: &prompts,
5997 cache: review_cache.as_deref(),
5998 round: Some(round),
5999 };
6000 let results = ask_json_wave::<Review>(
6001 jobs,
6002 Arc::clone(&self.sem),
6003 review_retries,
6004 &self.roles.reviewer_roster,
6005 &ctx,
6006 &mut quota_losses,
6007 &mut self.state,
6008 &|_: &Review| Ok(()),
6009 )
6010 .await;
6011 if self.state.resource_block.is_some() {
6012 return self.stop_for_resource();
6013 }
6014 let round_quota_missing = quota_losses.len();
6018 self.state.quota.extend(quota_losses);
6019
6020 let mut records = Vec::new();
6021 let mut all_findings = Vec::new();
6022 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6023 let agent_id = seat.agent.clone();
6024 self.state.seats.insert(seat.key.clone(), seat);
6025 let mut record = ReviewRecord {
6026 reviewer: r + 1,
6027 agent: agent_id,
6028 summary: String::new(),
6029 findings: Vec::new(),
6030 vote: None,
6031 failed: None,
6032 duration_ms: 0,
6033 attempts,
6039 };
6040 match res {
6041 Ok((review, out)) => {
6042 record.summary =
6050 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6051 record.vote = Some(review.vote);
6052 record.duration_ms = out.duration_ms;
6053 for (n, mut f) in review.findings.into_iter().enumerate() {
6054 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6057 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6058 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6059 f.file = f
6065 .file
6066 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6067 all_findings.push(f.clone());
6068 record.findings.push(f);
6069 }
6070 self.state.event(
6071 "review",
6072 format!(
6073 "round {round}: reviewer {} voted {} with {} finding(s)",
6074 r + 1,
6075 review.vote.label(),
6076 record.findings.len()
6077 ),
6078 );
6079 }
6080 Err(e) => {
6081 record.failed = Some(e.to_string());
6082 self.state.event(
6083 "review",
6084 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6085 );
6086 }
6087 }
6088 records.push(record);
6089 }
6090
6091 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6098 let vote_split =
6099 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6100 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6101 if vote_split {
6102 self.state.event(
6103 "review",
6104 format!(
6105 "round {round}: votes split ({}) — one round of reconsideration",
6106 initial_votes
6107 .iter()
6108 .map(|v| v.label())
6109 .collect::<Vec<_>>()
6110 .join(", ")
6111 ),
6112 );
6113 let panel: Vec<ReviewSeatReport<'_>> = records
6116 .iter()
6117 .filter_map(|r| {
6118 r.vote.map(|vote| ReviewSeatReport {
6119 reviewer: r.reviewer,
6120 vote,
6121 summary: &r.summary,
6122 findings: &r.findings,
6123 })
6124 })
6125 .collect();
6126
6127 let mut jobs = Vec::new();
6128 let mut seats_at = Vec::new();
6129 for (r, spec) in reviewers.iter().cloned().enumerate() {
6130 if records[r].vote.is_none() {
6134 continue;
6135 }
6136 let wt = root.join(format!("review-{}", r + 1));
6137 let seat_key = format!("review-{}", r + 1);
6138 let spec = self.occupant(&seat_key, spec);
6139 let seat = self.seat(&seat_key, &spec.id);
6140 let build = |with_patch: bool| {
6148 prompt::review_reconsider(&ReviewReconsiderCtx {
6149 instruction: &self.state.instruction,
6150 reviewer: r + 1,
6151 lens: Lens::for_seat(r),
6152 panel: &panel,
6153 patch: with_patch.then_some(ReviewPatch {
6154 branch: &winner.branch,
6155 base_short: &base_short,
6156 stat: &stat,
6157 patch: &patch,
6158 }),
6159 round,
6160 rounds: max_rounds,
6161 language: &language,
6162 })
6163 };
6164 let full = build(true);
6165 let prompt = if has_context(&spec, &seat, sessions) {
6166 build(false)
6167 } else {
6168 full.clone()
6169 };
6170 jobs.push(SeatJob {
6171 prompt,
6172 spec,
6173 seat,
6174 cwd: wt,
6175 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6176 allow_write: false,
6177 sessions,
6178 artifacts: artifacts.clone(),
6179 stem: format!("review-{round}-reconsider-{}", r + 1),
6180 handover: Some(full),
6181 });
6182 seats_at.push(r);
6183 }
6184
6185 let mut recon_quota_losses = Vec::new();
6186 let recon_cache = self.state.config.cache_dir();
6187 let recon_ctx = WaveCtx {
6188 carry_seats: true,
6189 pauses: Some((&self.pause, &self.interrupt)),
6190 run: &run_id,
6191 node: "review",
6192 prompts: &prompts,
6193 cache: recon_cache.as_deref(),
6194 round: Some(round),
6195 };
6196 let recon_results = ask_json_wave::<ReviewRevote>(
6197 jobs,
6198 Arc::clone(&self.sem),
6199 review_retries,
6200 &self.roles.reviewer_roster,
6201 &recon_ctx,
6202 &mut recon_quota_losses,
6203 &mut self.state,
6204 &|_: &ReviewRevote| Ok(()),
6205 )
6206 .await;
6207 if self.state.resource_block.is_some() {
6208 return self.stop_for_resource();
6209 }
6210 self.state.quota.extend(recon_quota_losses);
6211
6212 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6213 let agent_id = seat.agent.clone();
6214 self.state.seats.insert(seat.key.clone(), seat);
6215 let mut rec = ReviewRevoteRecord {
6216 reviewer: r + 1,
6217 agent: agent_id,
6218 vote: None,
6219 reason: String::new(),
6220 failed: None,
6221 };
6222 match res {
6223 Ok((rv, _)) => {
6224 rec.vote = Some(rv.vote);
6225 rec.reason =
6226 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6227 self.state.event(
6228 "review",
6229 format!(
6230 "round {round}: reviewer {} revoted {}",
6231 r + 1,
6232 rv.vote.label()
6233 ),
6234 );
6235 }
6236 Err(e) => {
6237 rec.failed = Some(e.to_string());
6238 self.state.event(
6239 "review",
6240 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6241 );
6242 }
6243 }
6244 reconsideration.push(rec);
6245 }
6246 } else if initial_votes.len() > 1 {
6247 self.state.event(
6248 "review",
6249 format!(
6250 "round {round}: votes agreed ({}) — no reconsideration",
6251 initial_votes[0].label()
6252 ),
6253 );
6254 }
6255
6256 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6257 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6258 let defer_e2e =
6269 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6270 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6271 let reason =
6272 format!("{blocking} blocking finding(s) already required a fix this round");
6273 self.state.event(
6274 "verify",
6275 format!(
6276 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6277 {}); it will run once a round has none left",
6278 short(&head)
6279 ),
6280 );
6281 (Vec::new(), false, true, Some(reason))
6282 } else {
6283 let e2e_commands = self.state.config.verify.e2e.clone();
6284 let cache_dir = self.state.config.cache_dir();
6285 let context = format!("round {round}");
6286 let wait = self.repository_wait();
6287 let (e2e, verify_retried) = with_cache_lease(
6288 &mut self.state,
6289 cache_dir.as_deref(),
6290 "e2e",
6291 "e2e",
6292 &winner.worktree,
6293 &head,
6294 verify_timeout,
6295 &context,
6296 &wait,
6297 |state, budget| {
6298 let shell = shell.clone();
6299 let e2e_commands = e2e_commands.clone();
6300 let worktree = winner.worktree.clone();
6301 let context = context.clone();
6302 async move {
6303 run_e2e_with_retry(
6304 state,
6305 &shell,
6306 &e2e_commands,
6307 &worktree,
6308 budget,
6309 &context,
6310 )
6311 .await
6312 }
6313 },
6314 )
6315 .await?;
6316 (e2e, verify_retried, false, None)
6317 };
6318
6319 let expected = records.len();
6320 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6321 let incomplete = answered < expected;
6322 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6323 let policy = self.state.config.graph.incomplete_review;
6324 let clean = round_is_clean(
6325 blocking,
6326 e2e_ok,
6327 answered,
6328 expected,
6329 round_quota_missing,
6330 policy,
6331 );
6332
6333 let mut round_record = ReviewRound {
6334 round,
6335 head: head.clone(),
6336 verified_head: None,
6337 verified_at: None,
6338 reviews: records,
6339 e2e,
6340 verify_retried,
6341 e2e_deferred,
6342 e2e_defer_reason,
6343 fix: None,
6344 blocking,
6345 answered,
6346 expected,
6347 clean,
6348 progressed: false,
6349 vote_split,
6350 reconsideration,
6351 verdict: None,
6352 };
6353 round_record.verdict = ReviewVote::worst(
6357 round_record
6358 .final_votes()
6359 .into_iter()
6360 .map(|(_, _, vote)| vote),
6361 );
6362 if !matches!(
6373 round_record.e2e_status(),
6374 E2eStatus::Deferred | E2eStatus::NotConfigured
6375 ) {
6376 round_record.verified_head = Some(head.clone());
6377 round_record.verified_at = Some(Timestamp::now());
6378 }
6379 let this_round_verification = round_record.verification_summary(&head);
6380
6381 if incomplete {
6382 let missing: Vec<String> = round_record
6383 .reviews
6384 .iter()
6385 .filter(|r| r.failed.is_some())
6386 .map(|r| format!("review-{}", r.reviewer))
6387 .collect();
6388 self.state.event(
6389 "review",
6390 format!(
6391 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6392 missing.join(", ")
6393 ),
6394 );
6395 }
6396
6397 if clean {
6398 self.state.event(
6399 "review",
6400 if incomplete && policy == IncompleteReviewPolicy::Warn {
6401 format!(
6402 "round {round}: clean (warn policy, incomplete panel) — no \
6403 blocking findings from the seats that answered, verification green"
6404 )
6405 } else if incomplete {
6406 format!(
6407 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6408 quorum) — no blocking findings from the seats that answered, \
6409 verification green",
6410 expected - answered
6411 )
6412 } else {
6413 format!("round {round}: clean — no blocking findings, verification green")
6414 },
6415 );
6416 self.state.reviews.push(round_record);
6417 self.state.status = RunStatus::Gating;
6418 self.state.save()?;
6419 return Ok(());
6420 }
6421
6422 if incomplete && blocking == 0 && e2e_ok {
6430 self.state.reviews.push(round_record);
6431 self.state.save()?;
6432 if round == max_rounds {
6433 self.state.status = RunStatus::Blocked;
6434 self.state.event(
6435 "review",
6436 format!(
6437 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6438 refusing to call it clean",
6439 expected - answered
6440 ),
6441 );
6442 return Ok(());
6443 }
6444 continue;
6445 }
6446
6447 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6459 self.state.reviews.push(round_record);
6460 return self
6461 .stop_reviewing(
6462 "the round's own verification could not run",
6463 &shell,
6464 &winner.worktree,
6465 )
6466 .await;
6467 }
6468
6469 if round == max_rounds {
6470 self.state.reviews.push(round_record);
6471 return self
6472 .stop_reviewing(
6473 &format!(
6474 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6475 ),
6476 &shell,
6477 &winner.worktree,
6478 )
6479 .await;
6480 }
6481
6482 let blocking_findings: Vec<_> = all_findings
6485 .iter()
6486 .filter(|f| f.severity.blocks())
6487 .cloned()
6488 .collect();
6489 let fix_prompt = prompt::fix_in(
6490 &self.state.instruction,
6491 &blocking_findings,
6492 this_round_verification.as_ref(),
6493 round,
6494 max_rounds,
6495 &language,
6496 &self.state.config.graph.github_language,
6497 );
6498 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6499 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6500 let (job, seat, out) = self
6501 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6502 prompt: fix_prompt.clone(),
6503 spec,
6504 seat,
6505 cwd: winner.worktree.clone(),
6506 timeout,
6507 allow_write: true,
6508 sessions,
6509 artifacts: artifacts.clone(),
6510 stem: format!("fix-{round}"),
6511 handover: None,
6512 })
6513 .await;
6514 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6517 return self.stop_for_resource();
6518 }
6519 let agent_id = seat.agent.clone();
6520
6521 let mut fix = FixRecord {
6522 agent: agent_id,
6523 addressed: Vec::new(),
6524 rejected: Vec::new(),
6525 notes: String::new(),
6526 committed: false,
6527 failed: None,
6528 duration_ms: 0,
6529 continuation: None,
6530 };
6531 let mut continuation = ContinuationRecord::not_needed();
6532 let mut final_seat = seat.clone();
6533 match out {
6534 AgentOutcome::Ok(o) => {
6535 fix.duration_ms = o.duration_ms;
6536 let parsed = verdict::extract_json::<FixReport>(&o.text);
6537 let incomplete_reason = match &parsed {
6544 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6545 "the reply parsed, but it reported a command whose own CLI never \
6546 confirmed an exit status"
6547 .to_owned(),
6548 ),
6549 Ok(_) => None,
6550 Err(e) => Some(e.to_string()),
6551 };
6552 match incomplete_reason {
6553 None => {
6554 let report = parsed.expect("checked Ok above");
6555 fix.addressed = report.addressed;
6556 fix.rejected = report.rejected;
6557 fix.notes =
6558 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6559 }
6560 Some(reason) => {
6561 let (resumed_seat, resolved, failure, cont) = self
6562 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6563 .await;
6564 fix.duration_ms += cont.cumulative_wait_ms;
6565 continuation = cont;
6566 final_seat = resumed_seat;
6567 match resolved {
6568 Some(report) => {
6569 fix.addressed = report.addressed;
6570 fix.rejected = report.rejected;
6571 fix.notes = blind::sanitize_prose(
6572 &report.notes,
6573 &self.state.config.blind,
6574 );
6575 }
6576 None => fix.failed = failure,
6577 }
6578 }
6579 }
6580 }
6581 AgentOutcome::Dropped(o) => {
6583 fix.duration_ms = o.duration_ms;
6584 let why = o
6585 .dropped
6586 .as_ref()
6587 .map(|d| d.why.as_str())
6588 .unwrap_or("the CLI ended the stream without delivering its answer");
6589 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6590 }
6591 AgentOutcome::Quota(o) => {
6592 self.state.quota.push(QuotaLoss {
6593 seat: final_seat.key.clone(),
6594 node: "fix".to_owned(),
6595 at: Timestamp::now(),
6596 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6597 });
6598 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6599 }
6600 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6601 }
6602 fix.continuation = Some(continuation);
6603 self.state.seats.insert(final_seat.key.clone(), final_seat);
6604 let coordination = self.coordinate(&winner.worktree).await?;
6605 if let Ok(r) = git::rescue_commit(
6606 &winner.worktree,
6607 &format!("magi: review round {round} fixes (uncommitted work)"),
6608 )
6609 .await
6610 {
6611 self.state.note_withheld("fix", &r.withheld);
6612 }
6613 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6614 fix.committed = after != before;
6615 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6623 drop(coordination);
6624 let progressed = diff_after != patch;
6625 let commit_note = if fix.committed {
6626 "committed"
6627 } else {
6628 "NO new commit"
6629 };
6630 let tree_note = if progressed {
6631 "changed vs base"
6632 } else {
6633 "unchanged vs base"
6634 };
6635 self.state.event(
6636 "fix",
6637 match &fix.failed {
6638 Some(reason) => {
6644 format!(
6645 "round {round}: fixer's adoption report was lost ({reason}); \
6646 {commit_note}, tree {tree_note}"
6647 )
6648 }
6649 None => format!(
6650 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6651 {tree_note}{}",
6652 fix.addressed.len(),
6653 fix.rejected.len(),
6654 if continuation.outcome == ContinuationOutcome::Resumed {
6655 format!(
6656 " (adoption report recovered after {} continuation(s))",
6657 continuation.attempts
6658 )
6659 } else {
6660 String::new()
6661 },
6662 ),
6663 },
6664 );
6665 round_record.fix = Some(fix);
6666 round_record.progressed = progressed;
6667 self.state.reviews.push(round_record);
6668 self.state.save()?;
6669
6670 if matches!(
6683 continuation.outcome,
6684 ContinuationOutcome::Exhausted
6685 | ContinuationOutcome::QuotaLost
6686 | ContinuationOutcome::NoSession
6687 ) {
6688 return self
6689 .stop_reviewing(
6690 "the fixer's adoption report never came back, even after resuming its \
6691 own seat; refusing to start another round against the same worktree \
6692 while that is unresolved",
6693 &shell,
6694 &winner.worktree,
6695 )
6696 .await;
6697 }
6698
6699 let streak = self
6700 .state
6701 .reviews
6702 .iter()
6703 .rev()
6704 .take_while(|r| !r.progressed)
6705 .count();
6706 if streak >= STAGNANT_LIMIT {
6707 return self
6708 .stop_reviewing(
6709 &format!(
6710 "the tree has not moved against base for {streak} round(s) in a row"
6711 ),
6712 &shell,
6713 &winner.worktree,
6714 )
6715 .await;
6716 }
6717 }
6718 Ok(())
6719 }
6720
6721 fn record_contested_handoff(&mut self) {
6755 if self.state.contested_handoff.is_some() {
6756 return;
6757 }
6758 let Some(contested) = self
6759 .state
6760 .reviews
6761 .last()
6762 .and_then(ReviewRound::contested_handoff)
6763 else {
6764 return;
6765 };
6766 self.state.event(
6767 "review",
6768 format!(
6769 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6770 wait for the owner's approval",
6771 contested.findings.len(),
6772 contested.rejecters.len()
6773 ),
6774 );
6775 self.state.contested_handoff = Some(contested);
6776 }
6777
6778 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6779 let round_idx = self.state.reviews.len() - 1;
6780 let needs_catchup_run = matches!(
6788 self.state.reviews[round_idx].e2e_status(),
6789 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6790 );
6791 if needs_catchup_run {
6792 let round = self.state.reviews[round_idx].round;
6793 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6794 let commands = self.state.config.verify.e2e.clone();
6795 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6796 let cache_dir = self.state.config.cache_dir();
6797 let context = format!(
6798 "round {round}: verification unresolved, catching up before the final decision"
6799 );
6800 let wait = self.repository_wait();
6801 let (outcomes, verify_retried) = with_cache_lease(
6802 &mut self.state,
6803 cache_dir.as_deref(),
6804 "e2e",
6805 "e2e",
6806 worktree,
6807 &attempted_head,
6808 timeout,
6809 &context,
6810 &wait,
6811 |state, budget| {
6812 let shell = shell.to_vec();
6813 let commands = commands.clone();
6814 let context = context.clone();
6815 async move {
6816 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6817 .await
6818 }
6819 },
6820 )
6821 .await?;
6822 let last = &mut self.state.reviews[round_idx];
6823 last.e2e = outcomes;
6824 last.verify_retried = verify_retried;
6825 last.verified_head = Some(attempted_head);
6832 last.verified_at = Some(Timestamp::now());
6833 if verify_inconclusive(&last.e2e) {
6834 self.state.save()?;
6841 return Ok(());
6842 }
6843 last.e2e_deferred = false;
6844 }
6845 let last = &self.state.reviews[round_idx];
6846 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6847
6848 match last.e2e_status() {
6849 E2eStatus::Failed => {
6850 let red: Vec<String> = last
6851 .e2e
6852 .iter()
6853 .filter(|o| !o.ok())
6854 .map(|o| {
6855 format!(
6856 "`{}` -> {:?}\n{}",
6857 o.command,
6858 o.code,
6859 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6860 )
6861 })
6862 .collect();
6863 self.state
6864 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6865 self.state.status = RunStatus::Blocked;
6866 }
6867 E2eStatus::ResourceBlocked => {
6872 self.state.event(
6873 "review",
6874 format!(
6875 "{why}; e2e could not run (shared build cache unavailable); not \
6876 deciding yet"
6877 ),
6878 );
6879 }
6880 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6881 self.state.event(
6882 "review",
6883 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6884 );
6885 self.record_contested_handoff();
6886 self.state.status = RunStatus::Gating;
6887 }
6888 }
6889 self.state.save()?;
6890 Ok(())
6891 }
6892
6893 async fn gate(&mut self) -> Result<()> {
6896 if self.state.status == RunStatus::Failed
6908 || self
6909 .state
6910 .base_sync
6911 .as_ref()
6912 .is_some_and(|s| s.conflict.is_some())
6913 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6914 != Some(RunStatus::Gating)
6915 {
6916 return Ok(());
6917 }
6918 if self.state.gate_ran {
6919 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6930 self.state.status = RunStatus::Blocked;
6931 self.state.save()?;
6932 }
6933 return Ok(());
6934 }
6935 let Some(winner) = self.state.winner().cloned() else {
6936 return Ok(());
6937 };
6938 self.state.status = RunStatus::Gating;
6939 let mut outcomes = self.run_gate(&winner).await?;
6940 loop {
6941 if verify_inconclusive(&outcomes) {
6952 self.state.save()?;
6953 return Ok(());
6954 }
6955 if outcomes.iter().all(CommandOutcome::ok) {
6956 break;
6957 }
6958 match self.gate_fix_round(&winner, &outcomes).await? {
6959 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6960 GateFix::Stop => break,
6961 GateFix::Defer => {
6962 self.state.save()?;
6963 return Ok(());
6964 }
6965 }
6966 }
6967 let passed = outcomes.iter().all(CommandOutcome::ok);
6968 self.state.gate = outcomes;
6969 self.state.gate_ran = true;
6970 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6971 if !passed {
6972 self.state.status = RunStatus::Blocked;
6973 let spent = self.state.gate_fixes.len();
6974 self.state.event(
6975 "gate",
6976 if spent == 0 {
6977 "gate failed; not merging".to_owned()
6978 } else {
6979 format!("gate failed after {spent} gate-fix round(s); not merging")
6980 },
6981 );
6982 }
6983 self.state.save()?;
6984 Ok(())
6985 }
6986
6987 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
6997 let commands = state.config.verify.pre_gate.clone();
6998 if commands.is_empty() {
6999 return;
7000 }
7001 let shell = state.config.shell();
7002 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7003 let (outcomes, _) = run_commands(
7004 state,
7005 "pre_gate",
7006 "pre_gate",
7007 0,
7008 &shell,
7009 &commands,
7010 &winner.worktree,
7011 timeout,
7012 )
7013 .await;
7014 for o in &outcomes {
7015 if !o.ok() {
7016 tracing::warn!(
7017 "pre_gate `{}` failed ({:?}); the gate decides",
7018 o.command,
7019 o.code
7020 );
7021 }
7022 state.event(
7023 "pre_gate",
7024 format!(
7025 "`{}` -> {}",
7026 o.command,
7027 if o.ok() {
7028 "pass".to_owned()
7029 } else {
7030 format!(
7031 "FAIL ({:?})\n{}",
7032 o.code,
7033 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7034 )
7035 }
7036 ),
7037 );
7038 }
7039 state.pre_gate = outcomes;
7040 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7041 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7042 Ok(head) => {
7043 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7044 state.pre_gate_commit = Some(head);
7045 }
7046 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7047 },
7048 Ok(false) => {}
7049 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7050 }
7051 if let Err(e) = state.save() {
7052 tracing::warn!("could not persist the pre_gate record: {e:#}");
7053 }
7054 }
7055
7056 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7059 let wait = self.repository_wait();
7060 run_landing_gate(&mut self.state, winner, &wait).await
7061 }
7062
7063 async fn gate_fix_round(
7076 &mut self,
7077 winner: &Candidate,
7078 outcomes: &[CommandOutcome],
7079 ) -> Result<GateFix> {
7080 let cap = self.state.config.graph.gate_fix_rounds;
7081 let spent = self.state.gate_fixes.len();
7082 if spent >= cap {
7083 if cap > 0 {
7084 self.state.event(
7085 "gate",
7086 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7087 );
7088 }
7089 return Ok(GateFix::Stop);
7090 }
7091 if !gate_fixable(outcomes) {
7092 self.state.event(
7093 "gate",
7094 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7095 command or similar); not spending a fix round on it",
7096 );
7097 return Ok(GateFix::Stop);
7098 }
7099 let min_free = self.state.config.disk.min_free_bytes;
7100 if min_free > 0 {
7101 match crate::disk::free_bytes(&winner.worktree) {
7102 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7103 Ok(free) => {
7104 self.state.event(
7105 "gate",
7106 format!(
7107 "only {free} bytes free ({min_free} required by `[disk] \
7108 min_free_bytes`); not spending a fix round on a failure the disk \
7109 may explain"
7110 ),
7111 );
7112 return Ok(GateFix::Stop);
7113 }
7114 Err(e) => {
7115 self.state.event(
7116 "gate",
7117 format!("free disk space could not be measured ({e:#}); no fix round"),
7118 );
7119 return Ok(GateFix::Stop);
7120 }
7121 }
7122 }
7123
7124 let attempt = spent + 1;
7125 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7126 let base = self.landing_base();
7127 let fix_prompt = prompt::gate_fix_in(
7128 &self.state.instruction,
7129 &failed,
7130 attempt,
7131 cap,
7132 &self.state.config.graph.language,
7133 &self.state.config.graph.github_language,
7134 );
7135 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7136 let sessions = self.state.config.graph.sessions;
7137 let artifacts = self.merge_repair_artifacts();
7138 self.state.event(
7139 "gate",
7140 format!("gate failed; gate-fix round {attempt} of {cap}"),
7141 );
7142 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7143 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7144 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7147 head: before.clone(),
7148 passed: false,
7149 outcomes: Vec::new(),
7150 });
7151 self.state.save()?;
7152 let (_, seat, out) = self
7153 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7154 prompt: fix_prompt.clone(),
7155 spec,
7156 seat,
7157 cwd: winner.worktree.clone(),
7158 timeout,
7159 allow_write: true,
7160 sessions,
7161 artifacts: artifacts.clone(),
7162 stem: format!("gate-fix-{attempt}"),
7163 handover: None,
7164 })
7165 .await;
7166 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7169 return Ok(GateFix::Defer);
7170 }
7171 let mut record = GateFixRecord {
7172 agent: seat.agent.clone(),
7173 failed,
7174 notes: String::new(),
7175 committed: false,
7176 error: None,
7177 };
7178 match out {
7179 AgentOutcome::Ok(o) => {
7180 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7183 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7184 }
7185 }
7186 AgentOutcome::Dropped(_) => {
7187 record.error = Some("the CLI dropped the stream".to_owned());
7188 }
7189 AgentOutcome::Quota(o) => {
7190 self.state.quota.push(QuotaLoss {
7191 seat: seat.key.clone(),
7192 node: "gate-fix".to_owned(),
7193 at: Timestamp::now(),
7194 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7195 });
7196 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7197 }
7198 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7199 }
7200 self.state.seats.insert(seat.key.clone(), seat);
7201 let coordination = self.coordinate(&winner.worktree).await?;
7202 if let Ok(r) = git::rescue_commit(
7203 &winner.worktree,
7204 &format!("magi: gate fix {attempt} (uncommitted work)"),
7205 )
7206 .await
7207 {
7208 self.state.note_withheld("gate-fix", &r.withheld);
7209 }
7210 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7211 record.committed = after != before;
7212 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7213 let note = record.error.clone();
7214 self.state.gate_fixes.push(record);
7215 self.state.save()?;
7216 if !changed {
7217 self.state.event(
7218 "gate",
7219 match note {
7220 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7221 None => format!("gate-fix round {attempt}: the tree did not change"),
7222 },
7223 );
7224 return Ok(GateFix::Stop);
7225 }
7226 self.state.event(
7227 "gate",
7228 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7229 );
7230
7231 drop(coordination);
7232 let wait = self.repository_wait();
7233 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7234 }
7235
7236 async fn guarded_pr_message(
7243 &mut self,
7244 winner: &Candidate,
7245 facts: Option<&BranchFacts>,
7246 posting: bool,
7247 ) -> Result<Option<PrMessage>> {
7248 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7249 let head = match final_head.as_deref() {
7250 Some(h) => crate::pr_outcome::FinalHead::At(h),
7251 None => crate::pr_outcome::FinalHead::Unknown,
7252 };
7253 let summarized = if posting {
7255 Some(self.summarize_change(winner, final_head.as_deref()).await)
7256 } else {
7257 None
7258 };
7259 if self.state.resource_block.is_some() {
7264 self.state.event(
7265 "pr-summary",
7266 "a shared resource was busy; nothing was published, resume to summarize again",
7267 );
7268 return Ok(None);
7269 }
7270 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7271 anyhow::bail!(
7272 "the final commit moved during public description generation; retry before publishing"
7273 );
7274 }
7275 let note = match (&summarized, &self.state.change_summary) {
7276 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7277 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7278 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7279 (None, _) => ChangeNote::NotAsked,
7280 };
7281 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7282 if !posting {
7283 let identity = crate::scrub::Identity::current();
7284 return Ok(Some(PrMessage {
7285 title: crate::scrub::scrub(&pr.title, &identity),
7286 body: crate::scrub::scrub(&pr.body, &identity),
7287 }));
7288 }
7289 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7292 if let Some(known) = self
7293 .state
7294 .github_text
7295 .clone()
7296 .filter(|g| g.fingerprint == raw_fp)
7297 {
7298 return self.settle_withheld(winner, &pr, known).await;
7299 }
7300 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7301 let mut violations =
7302 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7303 let identity = crate::scrub::Identity::current();
7304 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7305 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7306 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7307 {
7308 violations.push(crate::github_text::Violation::SensitiveData);
7309 }
7310 let has_language = violations
7314 .iter()
7315 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7316 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7317 self.state.event(
7318 "github-text",
7319 if has_language {
7320 format!("description rejected: {violations:?}; requesting one rewrite")
7321 } else {
7322 format!("description rejected: {violations:?}; asking the owner")
7323 },
7324 );
7325 let mut rewritten = false;
7326 let mut rejected_rewrite = None;
7327 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7328 {
7329 let output = text;
7330 if let Some(title) = summary_title(&output) {
7331 let raw_body = summary_without_title(&output);
7334 let clean_title = crate::scrub::scrub(&title, &identity);
7335 let mut body = crate::scrub::scrub(&raw_body, &identity);
7336 let redacted = clean_title != title || body != raw_body;
7337 let title = clean_title;
7338 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7339 if !complete {
7340 self.state.event(
7341 "github-text",
7342 "description rewrite omitted required findings",
7343 );
7344 }
7345 if !body.trim().is_empty() {
7346 body.push_str("\n\n");
7347 body.push_str(&public_pr_evidence(&self.state, head));
7348 }
7349 let marker = format!("magi:run/{}", self.state.id);
7350 if !body.trim().is_empty() && !body.contains(&marker) {
7351 body.push_str(&format!("\n\n{marker}"));
7352 }
7353 let rewrite = PrMessage {
7354 title: title.clone(),
7355 body: body.clone(),
7356 };
7357 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7358 let mut failed = crate::github_text::check_for(
7359 &self.state.config.graph,
7360 &title,
7361 &body,
7362 redecision,
7363 );
7364 if (!crate::github_text::shareable(&title)
7365 || !crate::github_text::shareable(&body))
7366 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7367 {
7368 failed.push(crate::github_text::Violation::SensitiveData);
7369 }
7370 if complete
7371 && !title.trim().is_empty()
7372 && !body.trim().is_empty()
7373 && !failed.is_empty()
7374 {
7375 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7376 title: title.clone(),
7377 body: body.clone(),
7378 failed: failed
7379 .iter()
7380 .map(|v| crate::github_text::category(*v).to_owned())
7381 .collect(),
7382 });
7383 }
7384 if complete
7385 && !title.trim().is_empty()
7386 && !raw_body.trim().is_empty()
7387 && failed.is_empty()
7388 {
7389 decision = redecision;
7390 pr = PrMessage { title, body };
7391 rewritten = true;
7392 self.state
7393 .event("github-text", "description rewrite passed");
7394 if redacted {
7395 self.state.event(
7396 "github-text",
7397 "sensitive data removed from description rewrite",
7398 );
7399 }
7400 }
7401 }
7402 }
7403 if !rewritten {
7404 self.state.event(
7405 "github-text",
7406 "description rewrite unavailable or rejected; asking the owner",
7407 );
7408 if let Some(w) =
7409 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7410 {
7411 self.retire_github_text_question("the withheld text changed");
7414 let record = crate::run::GithubTextAsk {
7415 rewrite: rejected_rewrite,
7416 fingerprint: raw_fp,
7417 title: w.title,
7418 body: w.body,
7419 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7420 question: None,
7421 asks: 0,
7422 resolved: false,
7423 chosen_title: None,
7424 choice: None,
7425 };
7426 return self.settle_withheld(winner, &pr, record).await;
7427 }
7428 }
7429 }
7430 let (title, body) =
7431 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7432 Ok(Some(PrMessage { title, body }))
7433 }
7434
7435 async fn summarize_change(
7446 &mut self,
7447 winner: &Candidate,
7448 head: Option<&str>,
7449 ) -> std::result::Result<(), String> {
7450 let Some(head) = head else {
7451 return Err("the final commit could not be read".to_owned());
7452 };
7453 let remote = self.state.config.merge.remote.clone();
7454 let tracking = format!("{remote}/{}", self.state.base_branch);
7455 if !git::rev_exists(&winner.worktree, &tracking).await {
7456 return Err(format!("the remote base `{tracking}` could not be read"));
7457 }
7458 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7459 Ok(b) if !b.is_empty() => b,
7460 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7461 };
7462 let evidence = crate::pr_outcome::summary_evidence(
7463 &self.state,
7464 crate::pr_outcome::FinalHead::At(head),
7465 );
7466 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7467 if self
7468 .state
7469 .change_summary
7470 .as_ref()
7471 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7472 {
7473 return Ok(());
7474 }
7475 if self.state.change_summary.take().is_some() {
7476 self.state.event(
7477 "pr-summary",
7478 "the final diff or evidence changed; the earlier change summary was discarded",
7479 );
7480 }
7481 if let Err(e) = write_artifact(
7483 &self.state,
7484 &format!("public-description-{fingerprint}.md"),
7485 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7486 ) {
7487 tracing::warn!("could not save public description evidence: {e:#}");
7488 }
7489 let chain = match agent::pick_chain(
7490 &self.state.config.agents,
7491 self.state.config.roles.synthesizer.as_ref(),
7492 &agent::installed,
7493 "synthesizer",
7494 ) {
7495 Ok(chain) => chain,
7496 Err(e) => {
7497 tracing::warn!("no agent for the change summary: {e:#}");
7498 self.state.event("pr-summary", "no summary agent available");
7499 return Err("no agent was available to summarize it".to_owned());
7500 }
7501 };
7502 let subjects = git::subjects(&winner.worktree, &base, head)
7503 .await
7504 .unwrap_or_default();
7505 let stat = git::diff_stat(&winner.worktree, &base, head)
7506 .await
7507 .unwrap_or_default();
7508 let stat: String = stat.chars().take(6_000).collect();
7509 let mut prompt = crate::prompt::change_summary_in(
7510 &base,
7511 head,
7512 &subjects,
7513 stat.trim(),
7514 &self.state.config.graph.github_language,
7515 );
7516 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7517 let sessions = self.state.config.graph.sessions;
7518 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7519 let prompts = self.state.config.prompts.clone();
7520 let cache = self.state.config.cache_dir();
7521 let run = self.state.id.clone();
7522 for (i, spec) in chain.into_iter().enumerate() {
7523 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7524 let job = SeatJob {
7525 spec: spec.clone(),
7526 seat,
7527 cwd: winner.worktree.clone(),
7528 prompt: prompt.clone(),
7529 timeout,
7530 allow_write: false,
7531 sessions,
7532 artifacts: agent::artifacts_dir(&self.state.dir()),
7533 stem: if i == 0 {
7534 "change-summary".to_owned()
7535 } else {
7536 format!("change-summary-{}", spec.id)
7537 },
7538 handover: None,
7539 };
7540 let ctx = WaveCtx {
7541 run: &run,
7542 node: "pr-summary",
7543 prompts: &prompts,
7544 cache: cache.as_deref(),
7545 round: None,
7546 carry_seats: false,
7547 pauses: Some((&self.pause, &self.interrupt)),
7548 };
7549 let (seat, outcome) =
7550 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7551 self.state.seats.insert(seat.key.clone(), seat);
7552 let text = match outcome {
7553 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7554 AgentOutcome::ResourceBlocked(why) => {
7555 if self.state.resource_block.is_none() {
7558 self.state.resource_block = Some(crate::run::ResourceBlock {
7559 node: "pr-summary".to_owned(),
7560 kind: "shared resource".to_owned(),
7561 detail: why,
7562 at: Timestamp::now(),
7563 });
7564 }
7565 return Err("a shared resource was busy".to_owned());
7566 }
7567 _ => continue,
7568 };
7569 let blind = &self.state.config.blind;
7570 let Some(title) = summary_title(&text) else {
7571 continue;
7572 };
7573 let body: String = summary_without_title(&text)
7574 .lines()
7575 .filter(|l| !l.contains("magi:run/"))
7576 .collect::<Vec<_>>()
7577 .join("\n");
7578 let body = blind::sanitize_prose(body.trim(), blind);
7579 let title = blind::sanitize_prose(&title, blind);
7580 if body.trim().is_empty() || title.trim().is_empty() {
7581 continue;
7582 }
7583 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7584 continue;
7585 }
7586 if git::rev_parse(&winner.worktree, "HEAD")
7588 .await
7589 .ok()
7590 .as_deref()
7591 != Some(head)
7592 {
7593 return Err("the branch moved while the summary was written".to_owned());
7594 }
7595 if git::merge_base(&winner.worktree, &tracking, head)
7596 .await
7597 .ok()
7598 .as_deref()
7599 != Some(base.as_str())
7600 || crate::pr_outcome::summary_evidence(
7601 &self.state,
7602 crate::pr_outcome::FinalHead::At(head),
7603 ) != evidence
7604 {
7605 return Err("the summary evidence changed while it was written".to_owned());
7606 }
7607 self.state.change_summary = Some(crate::run::ChangeSummary {
7608 head: head.to_owned(),
7609 base,
7610 evidence: fingerprint,
7611 title,
7612 body,
7613 });
7614 self.state.event(
7615 "pr-summary",
7616 format!("change summary written by {} for the final diff", spec.id),
7617 );
7618 if let Err(e) = self.state.save() {
7619 tracing::warn!("could not save the change summary: {e:#}");
7620 }
7621 return Ok(());
7622 }
7623 self.state
7624 .event("pr-summary", "no agent produced a change summary");
7625 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7626 }
7627
7628 async fn rewrite_pr_text(
7636 &mut self,
7637 winner: &Candidate,
7638 pr: &PrMessage,
7639 violations: &[crate::github_text::Violation],
7640 ) -> Option<String> {
7641 let prompt = format!(
7642 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7643 pr.title, pr.body
7644 );
7645 let sessions = self.state.config.graph.sessions;
7646 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7647 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7649 let key = format!("impl-{}", winner.label);
7650 if let Some(spec) = self
7651 .state
7652 .config
7653 .agents
7654 .iter()
7655 .find(|a| a.id == winner.agent)
7656 .cloned()
7657 && let Some(existing) = self.state.seats.get(&key)
7658 && existing.agent == spec.id
7659 && agent::has_session(spec.kind, existing, sessions)
7660 {
7661 attempts.push((spec, existing.clone(), true));
7662 } else {
7663 let chain = agent::pick_chain(
7664 &self.state.config.agents,
7665 self.state.config.roles.language_judge.as_ref(),
7666 &agent::installed,
7667 "language_judge",
7668 );
7669 match chain {
7670 Ok(chain) => {
7671 for spec in chain {
7672 let seat = crate::fixer::seat_for(
7673 &mut self.state,
7674 "github-text-rewrite",
7675 &spec.id,
7676 );
7677 attempts.push((spec, seat, false));
7678 }
7679 }
7680 Err(e) => {
7681 tracing::warn!("no agent for the description rewrite: {e:#}");
7682 self.state
7683 .event("github-text", "no rewrite agent available");
7684 return None;
7685 }
7686 }
7687 }
7688 let prompts = self.state.config.prompts.clone();
7689 let cache = self.state.config.cache_dir();
7690 let run = self.state.id.clone();
7691 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7692 let job = SeatJob {
7693 spec: spec.clone(),
7694 seat,
7695 cwd: winner.worktree.clone(),
7696 prompt: prompt.clone(),
7697 timeout: retry_budget(timeout_full, resumed),
7698 allow_write: false,
7699 sessions,
7700 artifacts: agent::artifacts_dir(&self.state.dir()),
7701 stem: if i == 0 {
7702 "github-text-rewrite".to_owned()
7703 } else {
7704 format!("github-text-rewrite-{}", spec.id)
7705 },
7706 handover: None,
7707 };
7708 let ctx = WaveCtx {
7709 run: &run,
7710 node: "github-text",
7711 prompts: &prompts,
7712 cache: cache.as_deref(),
7713 round: None,
7714 carry_seats: false,
7715 pauses: Some((&self.pause, &self.interrupt)),
7716 };
7717 let (seat, outcome) =
7718 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7719 self.state.seats.insert(seat.key.clone(), seat);
7720 match outcome {
7721 AgentOutcome::Ok(o)
7722 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7723 {
7724 return Some(o.text);
7725 }
7726 AgentOutcome::ResourceBlocked(_) => return None,
7727 _ => {}
7728 }
7729 }
7730 None
7731 }
7732
7733 fn retire_github_text_question(&mut self, why: &str) {
7735 let Some(id) = self
7736 .state
7737 .github_text
7738 .as_ref()
7739 .and_then(|g| g.question.clone())
7740 else {
7741 return;
7742 };
7743 let _ = ask::Questions::open().update(&id, |q| {
7744 q.abandon(why);
7745 Ok(())
7746 });
7747 }
7748
7749 async fn settle_withheld(
7753 &mut self,
7754 winner: &Candidate,
7755 pr: &PrMessage,
7756 mut g: crate::run::GithubTextAsk,
7757 ) -> Result<Option<PrMessage>> {
7758 use crate::github_text as gt;
7759 let store = ask::Questions::open();
7760 let timeout = self.state.config.graph.answer_timeout;
7761 if !g.resolved {
7762 let mut standing = g
7763 .question
7764 .as_ref()
7765 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7766 if let Some(q) = standing.as_ref()
7767 && q.status.open()
7768 && gt::expired(q, timeout)
7769 {
7770 let why = format!("no answer within {}s of asking", timeout.max(1));
7771 standing = store
7772 .update(&q.id, |q| {
7773 q.abandon(&why);
7774 Ok(())
7775 })
7776 .ok()
7777 .map(|(q, ())| q);
7778 }
7779 let mut ask_again: Option<Option<Vec<String>>> = None;
7780 match standing {
7781 None if g.asks <= 1 => {
7784 g.asks = 0;
7785 ask_again = Some(None);
7786 }
7787 None => {
7788 g.resolved = true;
7789 g.chosen_title = None;
7790 g.choice = Some(GithubTextChoice::Fallback);
7791 }
7792 Some(q) if q.status.open() => {
7793 self.park_on_github_text(&g)?;
7794 return Ok(None);
7795 }
7796 Some(q) => match gt::read_reply(&q) {
7797 gt::Reply::Fallback => {
7798 g.resolved = true;
7799 g.chosen_title = None;
7800 g.choice = Some(GithubTextChoice::Fallback);
7801 }
7802 gt::Reply::Original => {
7803 g.resolved = true;
7804 g.chosen_title = None;
7805 g.choice = Some(GithubTextChoice::Original);
7806 }
7807 gt::Reply::Rewrite => {
7808 g.resolved = true;
7809 g.chosen_title = None;
7810 g.choice = Some(if g.rewrite.is_some() {
7811 GithubTextChoice::Rewrite
7812 } else {
7813 GithubTextChoice::Fallback
7814 });
7815 }
7816 gt::Reply::Title(text) => {
7817 let first = text
7818 .lines()
7819 .map(str::trim)
7820 .find(|l| !l.is_empty())
7821 .unwrap_or("");
7822 let decision = if first.is_empty() {
7823 None
7824 } else {
7825 let candidate = PrMessage {
7826 title: first.to_owned(),
7827 body: String::new(),
7828 };
7829 self.judge_pr_language(&winner.worktree, &candidate).await
7830 };
7831 match gt::vet_title(
7832 &text,
7833 decision,
7834 self.state.config.graph.github_english(),
7835 ) {
7836 Ok(title) => {
7837 g.resolved = true;
7838 g.chosen_title = Some(title);
7839 g.choice = Some(GithubTextChoice::MyTitle);
7840 }
7841 Err(cats) if g.asks < 2 => {
7842 ask_again = Some(Some(
7843 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7844 ))
7845 }
7846 Err(_) => {
7847 g.resolved = true;
7848 g.chosen_title = None;
7849 g.choice = Some(GithubTextChoice::Fallback);
7850 }
7851 }
7852 }
7853 },
7854 }
7855 if let Some(rejected) = ask_again {
7856 let w = gt::Withheld {
7859 title: g.title,
7860 body: g.body,
7861 categories: g.categories.clone(),
7862 };
7863 let artifact = match crate::run::write_artifact(
7866 &self.state,
7867 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7868 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7869 ) {
7870 Ok(p) => Some(p),
7871 Err(e) => {
7872 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7873 None
7874 }
7875 };
7876 let (detail, comparison) = gt::question_content(
7877 &w,
7878 &pr.title,
7879 &pr.body,
7880 &self.state.id,
7881 rejected.as_deref(),
7882 artifact.as_deref(),
7883 g.rewrite.as_ref(),
7884 );
7885 let mut q = ask::Question::new(
7886 self.state.id.clone(),
7887 gt::ASK_NODE.to_owned(),
7888 gt::ASK_SEAT.to_owned(),
7889 gt::question_summary(&self.state.config.graph.language, &w),
7890 detail,
7891 gt::question_choices(&w, g.rewrite.as_ref()),
7892 );
7893 q.posting_comparison = Some(comparison);
7894 q.answer_timeout = timeout;
7895 g.question = Some(q.id.clone());
7898 g.asks += 1;
7899 self.state.github_text = Some(g.clone());
7900 self.state.event(
7901 "github-text",
7902 format!("asking the owner about the withheld text ({})", q.short()),
7903 );
7904 self.state.save()?;
7905 store
7906 .put_executable(&mut q)
7907 .context("file the posting-gate question")?;
7908 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7909 tracing::warn!(
7910 "could not notify about posting-gate question {}: {e:#}",
7911 q.short()
7912 );
7913 }
7914 self.park_on_github_text(&g)?;
7915 return Ok(None);
7916 }
7917 }
7918 let resolved = g.chosen_title.clone();
7919 let original = g.choice == Some(GithubTextChoice::Original);
7920 self.state.event(
7922 "github-text",
7923 if original {
7924 format!(
7925 "posting the original text for: {}",
7926 match (g.title, g.body) {
7927 (true, true) => "title, description",
7928 (true, false) => "title",
7929 _ => "description",
7930 }
7931 )
7932 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7933 "posting the saved rewrite title and description".to_owned()
7934 } else if resolved.is_some() {
7935 "posting the owner's replacement title".to_owned()
7936 } else {
7937 "posting the neutral text for the withheld fields".to_owned()
7938 },
7939 );
7940 self.state.github_text = Some(g.clone());
7941 self.state.save()?;
7942 if g.choice == Some(GithubTextChoice::Rewrite)
7943 && let Some(r) = g.rewrite.as_ref()
7944 {
7945 return Ok(Some(PrMessage {
7946 title: r.title.clone(),
7947 body: r.body.clone(),
7948 }));
7949 }
7950 let w = gt::Withheld {
7951 title: g.title,
7952 body: g.body,
7953 categories: g.categories.clone(),
7954 };
7955 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7956 if let Some(t) = resolved {
7957 title = t;
7958 }
7959 let english = gt::LanguageDecision {
7961 title_english: true,
7962 body_english: true,
7963 body_complete: true,
7964 };
7965 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7966 if original {
7967 if g.title {
7970 title = pr.title.clone();
7971 }
7972 if g.body {
7973 body = pr.body.clone();
7974 }
7975 }
7976 Ok(Some(PrMessage { title, body }))
7977 }
7978
7979 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
7980 self.state.github_text = Some(g.clone());
7981 self.state.end_pass_parked(ParkCause::Question);
7982 self.state.parked = true;
7983 self.state.event(
7984 "github-text",
7985 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
7986 );
7987 self.state.save()
7988 }
7989
7990 async fn judge_pr_language(
7993 &mut self,
7994 cwd: &Path,
7995 pr: &PrMessage,
7996 ) -> Option<crate::github_text::LanguageDecision> {
7997 self.state.config.roles.language_judge.as_ref()?;
7998 if !self.state.config.graph.language_gate() {
7999 return None;
8000 }
8001 let decision =
8002 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8003 self.state.event(
8004 "github-text",
8005 if decision.is_some() {
8006 "language decided by the language judge"
8007 } else {
8008 "language judge unavailable; using built-in heuristics"
8009 },
8010 );
8011 decision
8012 }
8013
8014 async fn merge(&mut self) -> Result<()> {
8015 if self
8030 .state
8031 .base_sync
8032 .as_ref()
8033 .is_some_and(|s| s.conflict.is_some())
8034 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8035 != Some(RunStatus::Gating)
8036 || !self.state.gate_allows_progress()
8044 {
8045 return Ok(());
8046 }
8047 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8056 return Ok(());
8057 }
8058 let Some(winner) = self.state.winner().cloned() else {
8059 return Ok(());
8060 };
8061 let repo = self.state.repo.clone();
8062 let base = self.state.base_branch.clone();
8063 let mode = self.state.config.merge.mode;
8064 let style = self.state.config.merge.style;
8065 let facts = if is_review_run(&self.state) {
8066 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8067 let start = review_base(
8068 &repo,
8069 &self.state.config.merge.remote,
8070 &base,
8071 &self.state.base_commit,
8072 &winner.branch,
8073 )
8074 .await;
8075 branch_facts(&repo, &start, &winner.branch).await
8076 } else {
8077 None
8078 };
8079 let leaked = if is_review_run(&self.state) {
8082 leaked_subjects(&self.state, &winner.branch).await
8083 } else {
8084 Some(Vec::new())
8085 };
8086 let Some(pr) = self
8090 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8091 .await?
8092 else {
8093 return Ok(());
8094 };
8095 let message = pr.commit_message();
8096
8097 let outcome = match mode {
8098 MergeMode::None => MergeOutcome {
8099 mode,
8100 ok: true,
8101 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8102 empty: false,
8103 },
8104 MergeMode::Pr | MergeMode::Local
8105 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8106 {
8107 MergeOutcome {
8108 mode,
8109 ok: false,
8110 detail: empty_candidate_detail(&self.state, &base),
8111 empty: true,
8112 }
8113 }
8114 MergeMode::Local => {
8115 let on = git::current_branch(&repo).await?;
8116 if on.as_deref() != Some(base.as_str()) {
8117 MergeOutcome {
8118 mode,
8119 ok: false,
8120 detail: format!(
8121 "{} has {} checked out, not the base branch {base}",
8122 repo.display(),
8123 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8124 ),
8125 empty: false,
8126 }
8127 } else if !git::is_clean(&repo).await? {
8128 MergeOutcome {
8129 mode,
8130 ok: false,
8131 detail: format!("{} is dirty; refusing to merge", repo.display()),
8132 empty: false,
8133 }
8134 } else {
8135 let out = match style {
8136 MergeStyle::Merge => {
8137 git::merge_no_ff(&repo, &winner.branch, &message).await?
8138 }
8139 MergeStyle::Squash => {
8140 git::merge_squash(&repo, &winner.branch, &message).await?
8141 }
8142 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8143 };
8144 MergeOutcome {
8145 mode,
8146 ok: out.ok(),
8147 detail: if out.ok() { out.stdout } else { out.stderr },
8148 empty: false,
8149 }
8150 }
8151 }
8152 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8153 self.state.pr_publication_pending = true;
8154 self.state.save()?;
8155 let out =
8156 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8157 self.state.pr_publication_pending = out.is_err();
8158 MergeOutcome {
8159 mode,
8160 ok: out.is_ok(),
8161 detail: match out {
8162 Ok(url) => url,
8163 Err(e) => format!("{e:#}"),
8164 },
8165 empty: false,
8166 }
8167 }
8168 MergeMode::Pr => {
8169 let remote = self.state.config.merge.remote.clone();
8170 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8171 if !pushed.ok() {
8172 MergeOutcome {
8173 mode,
8174 ok: false,
8175 detail: pushed.stderr,
8176 empty: false,
8177 }
8178 } else {
8179 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8185 let out = match pr_merge_plan(found) {
8186 PrPlan::Create => {
8187 gh_pr_create(
8188 &winner.worktree,
8189 &base,
8190 &winner.branch,
8191 &pr.title,
8192 &pr.body,
8193 )
8194 .await
8195 }
8196 PrPlan::Adopt { url, title } => {
8197 self.state
8198 .event("merge", format!("Pr: adopted open pull request {url}"));
8199 if self.state.github_text.as_ref().is_some_and(|g| {
8200 g.resolved
8201 && g.choice == Some(GithubTextChoice::Rewrite)
8202 && g.rewrite.is_some()
8203 }) {
8204 land::set_pr_text_verbatim(
8207 &winner.worktree,
8208 &url,
8209 &pr.title,
8210 &pr.body,
8211 )
8212 .await
8213 .map(|()| url)
8214 } else {
8215 if title != pr.title
8216 && (!is_review_run(&self.state)
8217 || leaked
8218 .as_deref()
8219 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8220 && let Err(e) =
8221 if self.state.github_text.as_ref().is_some_and(|g| {
8222 g.resolved
8223 && g.title
8224 && g.choice == Some(GithubTextChoice::Original)
8225 }) {
8226 land::set_pr_title_verbatim(
8228 &winner.worktree,
8229 &url,
8230 &pr.title,
8231 )
8232 .await
8233 } else {
8234 land::set_pr_title(
8235 &mut self.state,
8236 &winner.worktree,
8237 &url,
8238 &pr.title,
8239 )
8240 .await
8241 }
8242 {
8243 tracing::warn!("could not refresh title of {url}: {e:#}");
8244 self.state
8245 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8246 }
8247 Ok(url)
8248 }
8249 }
8250 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8251 };
8252 match out {
8253 Ok(url) => MergeOutcome {
8254 mode,
8255 ok: true,
8256 detail: url,
8257 empty: false,
8258 },
8259 Err(e) => MergeOutcome {
8260 mode,
8261 ok: false,
8262 detail: e.to_string(),
8263 empty: false,
8264 },
8265 }
8266 }
8267 }
8268 };
8269
8270 self.state.status = match (mode, outcome.ok) {
8271 (MergeMode::None, _) => RunStatus::Ready,
8272 (_, true) if outcome.empty => RunStatus::Ready,
8273 (MergeMode::Pr, true) => RunStatus::Ready,
8274 (_, true) => RunStatus::Merged,
8275 (_, false) => RunStatus::Blocked,
8276 };
8277 self.state.event(
8278 "merge",
8279 format!(
8280 "{:?}: {}",
8281 mode,
8282 outcome.detail.lines().next().unwrap_or("")
8283 ),
8284 );
8285 self.state.merge = Some(outcome);
8286 self.state.save()?;
8287
8288 if self.state.config.graph.land
8294 && mode == MergeMode::Pr
8295 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8296 {
8297 self.run_land().await?;
8298 }
8299 self.settle_questions();
8304 Ok(())
8305 }
8306
8307 async fn run_land(&mut self) -> Result<()> {
8318 crate::pr_head::ensure_owner(&self.state)?;
8319 let url = self
8320 .state
8321 .pr
8322 .as_ref()
8323 .map(|pr| pr.url.clone())
8324 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8325 .unwrap_or_default();
8326 let url = url.lines().next().unwrap_or("").trim().to_owned();
8327 if !url.starts_with("http") {
8328 return Ok(());
8329 }
8330 loop {
8333 match land::land(&mut self.state, &url).await {
8334 Ok(_)
8335 if self
8336 .state
8337 .merge_repairs
8338 .last()
8339 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8340 {
8341 if self.repair_merge_review().await? {
8342 continue;
8343 }
8344 }
8345 Ok(pr) if self.state.parked => {
8346 let _ = pr;
8350 }
8351 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8352 self.state.status = RunStatus::Superseded;
8353 self.state.save()?;
8354 }
8355 Ok(pr) => {
8356 self.state.status = match pr.state {
8357 land::PrLifecycle::Merged => RunStatus::Merged,
8358 _ => RunStatus::Blocked,
8359 };
8360 if bump::should_release_bump(self.state.status)
8367 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8368 {
8369 self.state
8375 .event("bump", format!("release bump skipped: {e:#}"));
8376 }
8377 if self.state.status == RunStatus::Merged {
8380 crate::followup::after_merge(&mut self.state, &pr.url).await;
8381 }
8382 self.state.save()?;
8383 }
8384 Err(e) => {
8385 if e.is::<crate::repo_lock::ResourceBlocked>() {
8386 return Err(e);
8387 }
8388 self.state.status = RunStatus::Blocked;
8389 self.state.event("land", format!("gave up: {e}"));
8390 self.state.save()?;
8391 }
8392 }
8393 break;
8394 }
8395 Ok(())
8396 }
8397
8398 async fn repair_merge_review(&mut self) -> Result<bool> {
8401 use crate::run::MergeRepairPhase as Phase;
8402 crate::pr_head::ensure_owner(&self.state)?;
8403 let i = self.state.merge_repairs.len() - 1;
8404 let request = self.state.merge_repairs[i].clone();
8405 if request.phase == Phase::Fixing {
8406 let restored = match self.state.winner().cloned() {
8407 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8408 None => Err(anyhow::anyhow!("repair has no winning branch")),
8409 };
8410 let why = match restored {
8411 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8412 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8413 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8414 };
8415 self.hold_merge_repair(i, why)?;
8416 return Ok(false);
8417 }
8418 let failure = if i >= self.state.config.graph.land_rounds
8419 || self.state.config.graph.review_rounds == 0
8420 {
8421 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8422 } else {
8423 None
8424 };
8425 if let Some(why) = failure {
8426 self.hold_merge_repair(i, why)?;
8427 return Ok(false);
8428 }
8429 if request.phase == Phase::Requested {
8430 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8431 Ok(pr) => pr,
8432 Err(e) => {
8433 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8434 return Ok(false);
8435 }
8436 };
8437 self.state.merge_repairs[i].phase = Phase::Fixing;
8438 self.state.land_approval = None;
8439 self.state.land_armed_head = None;
8440 self.state.save()?;
8441 let evidence = serde_json::to_string_pretty(&request)?;
8444 let reason = format!(
8445 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8446 );
8447 let budget = self.state.config.graph.land_rounds;
8448 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8449 match result {
8450 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8451 Ok(land::Fixed::Failed(why)) => {
8452 self.hold_merge_repair(
8453 i,
8454 format!(
8455 "repair failed: {why}; inspect the branch and run an explicit review"
8456 ),
8457 )?;
8458 return Ok(false);
8459 }
8460 Err(e) => {
8461 self.hold_merge_repair(
8462 i,
8463 format!(
8464 "repair failed: {e:#}; inspect the branch and run an explicit review"
8465 ),
8466 )?;
8467 return Ok(false);
8468 }
8469 }
8470 self.begin_merge_repair_review(i)?;
8474 }
8475 self.review_loop().await?;
8476 self.gate().await?;
8477 if self.resource_halt()? || self.state.parked {
8478 return Ok(false);
8479 }
8480 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8481 != Some(RunStatus::Gating)
8482 || !self.state.gate_allows_progress()
8483 {
8484 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8485 return Ok(false);
8486 }
8487 let winner = self
8490 .state
8491 .winner()
8492 .cloned()
8493 .context("repair has no winning branch")?;
8494 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8495 if self.state.gate_head.as_deref() != Some(after.as_str())
8496 || !git::is_clean(&winner.worktree).await?
8497 {
8498 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8499 return Ok(false);
8500 }
8501 let expected = self
8502 .state
8503 .pr_pushed_head
8504 .as_deref()
8505 .unwrap_or(&request.head)
8506 .to_owned();
8507 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8508 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8511 let refspec = format!("{after}:refs/heads/{public}");
8512 let push = git::git_raw(
8513 &winner.worktree,
8514 &["push", &lease, &self.state.config.merge.remote, &refspec],
8515 )
8516 .await;
8517 match push {
8518 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8519 Ok(out) => {
8520 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8521 return Ok(false);
8522 }
8523 Err(e) => {
8524 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8525 return Ok(false);
8526 }
8527 }
8528 self.state.merge_repairs[i].phase = Phase::Complete;
8529 self.state.status = RunStatus::Landing;
8530 self.state.event(
8531 "repair",
8532 "new review and gate completed; fresh owner merge decision required",
8533 );
8534 self.state.save()?;
8535 Ok(true)
8536 }
8537
8538 fn merge_repair_artifacts(&self) -> PathBuf {
8539 let artifacts = agent::artifacts_dir(&self.state.dir());
8540 match self
8541 .state
8542 .merge_repairs
8543 .last()
8544 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8545 {
8546 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8547 None => artifacts,
8548 }
8549 }
8550
8551 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8552 self.state.reviews.clear();
8553 self.state.gate.clear();
8554 self.state.gate_ran = false;
8555 self.state.gate_head = None;
8556 self.state.gate_fixes.clear();
8557 self.state.gate_repair_e2e = None;
8558 self.state.pre_gate.clear();
8559 self.state.pre_gate_commit = None;
8560 self.state.contested_handoff = None;
8561 self.state.land_reruns = None;
8562 self.state.land_approval = None;
8563 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8564 self.state.status = RunStatus::Reviewing;
8565 self.state.save()
8566 }
8567
8568 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8569 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8570 self.state.merge_repairs[index].failure = Some(why.clone());
8571 self.state.status = RunStatus::Blocked;
8572 self.state.event("repair", why);
8573 self.state.save()
8574 }
8575
8576 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8580 if let Some(existing) = self.state.seats.get(key)
8581 && existing.agent == agent
8582 {
8583 return existing.clone();
8584 }
8585 let fresh = if self.state.seats.contains_key(key) {
8588 handover_seat(key, agent, self.state.next_seat_seed())
8589 } else {
8590 SeatState::new(key, agent, self.state.seed)
8591 };
8592 self.state.seats.insert(key.to_owned(), fresh.clone());
8593 fresh
8594 }
8595
8596 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8602 match self.state.seats.get(key) {
8603 Some(s) if s.agent != spec.id => {
8604 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8605 }
8606 _ => spec,
8607 }
8608 }
8609
8610 fn view(&self, c: &Candidate) -> CandidateView {
8612 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8613 .unwrap_or_default();
8614 let (patch, _) = blind::sanitize_patch(
8615 &format!("candidate {} patch", c.label),
8616 &raw,
8617 &self.state.config.blind,
8618 );
8619 CandidateView {
8620 label: c.label,
8621 branch: c.branch.clone(),
8622 summary: c.summary.clone(),
8623 stat: c.stat.clone(),
8624 patch,
8625 }
8626 }
8627
8628 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8630 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8631 prompt::judge(
8632 "(see above)",
8633 &views,
8634 self.roles.judges.len(),
8635 base_short,
8636 "en",
8637 )
8638 }
8639
8640 fn vote_prompt_full(
8646 &self,
8647 j: usize,
8648 labels: &[char],
8649 language: &str,
8650 candidates: &[Candidate],
8651 base_short: &str,
8652 ) -> String {
8653 let mut text = format!(
8654 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8655 prompt::final_vote(labels, language),
8656 self.state.instruction,
8657 self.candidate_block(candidates, base_short)
8658 );
8659 if let Some(own) = self
8660 .state
8661 .judgements
8662 .get(j)
8663 .filter(|r| !r.ranking.is_empty())
8664 {
8665 let reasons = own
8666 .reasons
8667 .iter()
8668 .map(|(k, v)| format!("- {k}: {v}"))
8669 .collect::<Vec<_>>()
8670 .join("\n");
8671 text.push_str(&format!(
8672 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8673 own.ranking.iter().collect::<String>(),
8674 if reasons.is_empty() {
8675 ""
8676 } else {
8677 ", because:\n"
8678 }
8679 ));
8680 }
8681 if !self.state.deliberation.is_empty() {
8682 text.push_str("\n# What was argued before this vote\n");
8683 for t in self.transcript(&[], j) {
8684 text.push_str(&format!(
8685 "\n## {}{}\n\n{}\n",
8686 t.who,
8687 if t.is_self { " (you)" } else { "" },
8688 t.body.trim()
8689 ));
8690 }
8691 }
8692 text
8693 }
8694
8695 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8702 let mut turns = Vec::new();
8703 for j in &self.state.judgements {
8704 if j.ranking.is_empty() {
8705 continue;
8706 }
8707 let reasons = j
8708 .reasons
8709 .iter()
8710 .map(|(k, v)| format!("- {k}: {v}"))
8711 .collect::<Vec<_>>()
8712 .join("\n");
8713 turns.push(Turn {
8714 who: format!("Judge {} (opening ranking)", j.judge),
8715 is_self: j.judge == self_idx + 1,
8716 body: format!(
8717 "Ranked {}{}{reasons}",
8718 j.ranking.iter().collect::<String>(),
8719 if reasons.is_empty() {
8720 ""
8721 } else {
8722 ", because:\n"
8723 }
8724 ),
8725 });
8726 }
8727 for t in self
8728 .state
8729 .deliberation
8730 .iter()
8731 .flat_map(|r| r.turns.iter())
8732 .chain(current)
8733 {
8734 turns.push(Turn {
8735 who: format!("Judge {}", t.judge),
8736 is_self: t.judge == self_idx + 1,
8737 body: t.body.clone(),
8738 });
8739 }
8740 turns
8741 }
8742}
8743
8744fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8746 agent::has_session(spec.kind, seat, sessions)
8747}
8748
8749fn next_untried_in_roster<'a>(
8770 roster: &'a [AgentSpec],
8771 start: usize,
8772 tried: &BTreeSet<String>,
8773) -> Option<&'a AgentSpec> {
8774 roster
8775 .get(start + 1..)?
8776 .iter()
8777 .find(|s| !tried.contains(&s.id))
8778}
8779
8780fn pick_successor<'a>(
8801 roster: &'a [AgentSpec],
8802 start: usize,
8803 tried: &BTreeSet<String>,
8804 carried: Option<&BTreeSet<String>>,
8805 others: &BTreeSet<String>,
8806) -> Option<&'a AgentSpec> {
8807 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8808 roster
8809 .get(start + 1..)
8810 .and_then(|tail| {
8811 tail.iter()
8812 .filter(free)
8813 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8814 })
8815 .or_else(|| {
8816 carried
8817 .is_some()
8818 .then(|| roster.iter().find(free))
8819 .flatten()
8820 })
8821 .or_else(|| match carried {
8822 Some(c) => next_for_seat(roster, start, tried, c),
8823 None => next_untried_in_roster(roster, start, tried),
8824 })
8825}
8826
8827fn next_for_seat<'a>(
8837 roster: &'a [AgentSpec],
8838 start: usize,
8839 round_tried: &BTreeSet<String>,
8840 carried_failed: &BTreeSet<String>,
8841) -> Option<&'a AgentSpec> {
8842 roster
8843 .get(start + 1..)?
8844 .iter()
8845 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8846 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8847}
8848
8849fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8855 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8856 return spec;
8857 };
8858 let ok = |id: &str| !h.failed.contains(id);
8859 if let Some(last) = h.last_ok.as_deref()
8860 && ok(last)
8861 && let Some(s) = roster.iter().find(|s| s.id == last)
8862 {
8863 return s.clone();
8864 }
8865 if ok(&spec.id) {
8866 return spec;
8867 }
8868 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8869 roster
8870 .get(from + 1..)
8871 .into_iter()
8872 .flatten()
8873 .chain(roster.iter())
8874 .find(|s| ok(&s.id))
8875 .cloned()
8876 .unwrap_or(spec)
8877}
8878
8879pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8883 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8884}
8885
8886const TIMED_OUT: &str = "timed out";
8889
8890impl FailClass {
8891 fn of(out: &AgentOutcome) -> Option<Self> {
8893 match out {
8894 AgentOutcome::Ok(_) => None,
8895 AgentOutcome::Quota(_) => Some(Self::Quota),
8896 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8897 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8898 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8899 AgentOutcome::ResourceBlocked(_) => None,
8902 }
8903 }
8904
8905 fn stem_word(&self) -> &'static str {
8907 match self {
8908 Self::Quota => "quota",
8909 _ => "handover",
8910 }
8911 }
8912}
8913
8914fn failure_signature(msg: &str) -> String {
8918 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8919 if let Some(code) = line
8922 .strip_prefix("process exited with no usable output (exit code ")
8923 .and_then(|s| s.strip_suffix(')'))
8924 .and_then(|s| s.parse::<i32>().ok())
8925 {
8926 line = format!("exited with some({code}) and no usable output");
8927 } else if matches!(
8928 line.as_str(),
8929 "process exited with no usable output (no exit code reported)"
8930 | "process exited with no usable output (exit code unavailable)"
8931 ) {
8932 line = "exited with none and no usable output".to_owned();
8933 }
8934 let mut out = Vec::new();
8935 for word in line.split_whitespace() {
8936 if word.contains('/') || word.contains('\\') {
8937 out.push("<path>".to_owned());
8938 continue;
8939 }
8940 let mut w = String::new();
8941 let mut in_digits = false;
8942 for c in word.chars() {
8943 if c.is_ascii_digit() {
8944 if !in_digits {
8945 w.push('#');
8946 }
8947 in_digits = true;
8948 } else {
8949 in_digits = false;
8950 w.push(c);
8951 }
8952 }
8953 out.push(w);
8954 }
8955 out.join(" ").chars().take(120).collect()
8956}
8957
8958fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8965 match cur {
8966 FailClass::Quota | FailClass::Timeout => true,
8967 FailClass::Other(_) => prev != Some(cur),
8968 }
8969}
8970
8971fn fail_reason(out: &AgentOutcome) -> String {
8973 match out {
8974 AgentOutcome::Ok(_) => String::new(),
8975 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
8976 AgentOutcome::Dropped(o) => format!(
8977 "the CLI dropped the stream ({})",
8978 o.dropped
8979 .as_ref()
8980 .map(|d| d.why.as_str())
8981 .unwrap_or("it ended without delivering its answer")
8982 ),
8983 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
8984 e.lines().next().unwrap_or("").chars().take(160).collect()
8985 }
8986 }
8987}
8988
8989pub(crate) fn record_handover(
8992 state: &mut RunState,
8993 node: &str,
8994 seat: &str,
8995 from: &str,
8996 to: &str,
8997 class: &FailClass,
8998 reason: &str,
8999) {
9000 let message = if *class == FailClass::Quota {
9001 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9002 } else {
9003 format!("{seat}: handed over {from} -> {to} ({reason})")
9004 };
9005 state.event(node, message);
9006 state.handovers.push(Handover {
9007 at: Timestamp::now(),
9008 node: node.to_owned(),
9009 seat: seat.to_owned(),
9010 from: from.to_owned(),
9011 to: to.to_owned(),
9012 reason: reason.to_owned(),
9013 });
9014}
9015
9016fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9030 commands.iter().any(|c| c.exit_code.is_none())
9031}
9032
9033fn verified_noop_claim(
9046 usable: bool,
9047 commands: &[agent::CommandEvidence],
9048 text: &str,
9049) -> Option<String> {
9050 (usable && !has_unconfirmed_command(commands))
9051 .then(|| verdict::verified_noop(text))
9052 .flatten()
9053}
9054
9055fn short(commit: &str) -> String {
9056 commit.chars().take(7).collect()
9057}
9058
9059fn make_executable(path: &Path) -> Result<()> {
9060 #[cfg(unix)]
9061 {
9062 use std::os::unix::fs::PermissionsExt as _;
9063 let mut perms = std::fs::metadata(path)?.permissions();
9064 perms.set_mode(0o755);
9065 std::fs::set_permissions(path, perms)?;
9066 }
9067 #[cfg(not(unix))]
9068 {
9069 let _ = path;
9070 }
9071 Ok(())
9072}
9073
9074struct WaveCtx<'a> {
9081 run: &'a str,
9084 node: &'a str,
9086 prompts: &'a Prompts,
9087 cache: Option<&'a Path>,
9089 round: Option<usize>,
9092 carry_seats: bool,
9096 pauses: Option<(&'a Pause, &'a Pause)>,
9099}
9100
9101fn repository_wait(
9103 state: &RunState,
9104 pauses: Option<(&Pause, &Pause)>,
9105) -> crate::repo_lock::WaitPolicy {
9106 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9107 let inherited = crate::repo_lock::WaitPolicy::current();
9108 crate::repo_lock::WaitPolicy::new(
9109 Duration::from_secs(state.config.graph.timeout_resource_wait),
9110 move || {
9111 inherited.cancelled()
9112 || pauses
9113 .as_ref()
9114 .is_some_and(|(a, b)| a.parked() || b.parked())
9115 },
9116 )
9117}
9118
9119fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9120 if error.is::<crate::repo_lock::ResourceBlocked>() {
9121 state.resource_block = Some(crate::run::ResourceBlock {
9122 node: "repository".into(),
9123 kind: if error.to_string().contains("disk pressure") {
9124 "disk pressure"
9125 } else {
9126 "repository coordination"
9127 }
9128 .into(),
9129 detail: error.to_string(),
9130 at: Timestamp::now(),
9131 });
9132 state.event("resource", error.to_string());
9133 }
9134}
9135
9136async fn coordinate_operation(
9137 state: &mut RunState,
9138 cwd: &Path,
9139 wait: &crate::repo_lock::WaitPolicy,
9140) -> Result<Option<crate::repo_lock::SeatLease>> {
9141 let result =
9142 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9143 .await;
9144 if let Err(e) = &result {
9145 record_repository_block(state, e);
9146 }
9147 result
9148}
9149
9150async fn run_one(
9152 job: SeatJob,
9153 sem: Arc<Semaphore>,
9154 ctx: &WaveCtx<'_>,
9155 state: &mut RunState,
9156 attempt: usize,
9157) -> (SeatState, AgentOutcome) {
9158 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9159 .await
9160 .pop()
9161 .expect("one job in, one result out");
9162 (seat, out)
9163}
9164
9165async fn wave(
9171 jobs: Vec<SeatJob>,
9172 sem: Arc<Semaphore>,
9173 ctx: &WaveCtx<'_>,
9174 state: &mut RunState,
9175 attempt: usize,
9176) -> Vec<(usize, SeatState, AgentOutcome)> {
9177 let WaveCtx {
9178 run,
9179 node,
9180 prompts,
9181 cache,
9182 round,
9183 carry_seats: _,
9184 pauses,
9185 } = *ctx;
9186 for job in &jobs {
9187 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9188 }
9189 if let Err(e) = state.save() {
9190 tracing::warn!("could not persist in-progress seats: {e:#}");
9195 }
9196 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9212 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9216 let cache_guard = if let Some(cache_dir) = cache {
9217 if jobs_had_a_writer {
9218 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9219 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9220 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9221 {
9222 Ok(Some(g)) => Some(g),
9223 Ok(None) => {
9226 state.resource_block = Some(crate::run::ResourceBlock {
9227 node: node.to_owned(),
9228 kind: "shared build cache".to_owned(),
9229 detail: "the run is parking".to_owned(),
9230 at: Timestamp::now(),
9231 });
9232 return block_wave(
9233 jobs,
9234 state,
9235 node,
9236 attempt,
9237 "parked while waiting for the shared build cache",
9238 );
9239 }
9240 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9243 Err(e) => {
9244 let why = format!("{e:#}");
9248 state.resource_block = Some(crate::run::ResourceBlock {
9249 node: node.to_owned(),
9250 kind: "shared build cache".to_owned(),
9251 detail: why.clone(),
9252 at: Timestamp::now(),
9253 });
9254 state.event(
9255 "resource",
9256 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9257 );
9258 return block_wave(jobs, state, node, attempt, &why);
9259 }
9260 }
9261 } else {
9262 None
9263 }
9264 } else {
9265 None
9266 };
9267 state.seats_spawning(node, attempt);
9270 if let Err(e) = state.save() {
9271 tracing::warn!("could not persist the start of a wave: {e:#}");
9272 }
9273 let slot_limit = resource_limit
9277 + jobs
9278 .iter()
9279 .map(|j| j.timeout)
9280 .max()
9281 .unwrap_or_default()
9282 .saturating_mul(jobs.len() as u32);
9283 let mut set = tokio::task::JoinSet::new();
9284 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9287 let overlay = prompts.overlay(node);
9288 let repository_wait = repository_wait(state, pauses);
9289 for (i, mut job) in jobs.into_iter().enumerate() {
9290 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9291 if cache.is_some() {
9292 job.prompt.push('\n');
9293 job.prompt
9294 .push_str(&prompt::build_cache_note(node, job.allow_write));
9295 }
9296 let sem = Arc::clone(&sem);
9297 let run = run.to_owned();
9298 let node = node.to_owned();
9299 let attachments = if node == "implement" {
9302 state.attachments.clone()
9303 } else {
9304 Vec::new()
9305 };
9306 let cache = cache
9317 .filter(|_| job.allow_write && cache_guard.is_some())
9318 .map(Path::to_path_buf);
9319 let spawned_tx = spawned_tx.clone();
9320 let repository_wait = repository_wait.clone();
9321 let coordination = state.config.coordination.clone();
9322 let fetched_base = state.fetched_base.clone();
9323 set.spawn(async move {
9324 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9328 Ok(p) => p,
9329 Err(_) => {
9330 return (
9331 i,
9332 job.seat,
9333 AgentOutcome::ResourceBlocked(format!(
9334 "no parallel slot became free within {}s",
9335 slot_limit.as_secs()
9336 )),
9337 false,
9338 );
9339 }
9340 };
9341 spawned_tx.send(job.seat.key.clone()).ok();
9342 let mut seat = job.seat;
9343 let out = repository_wait
9344 .scope(agent::invoke(
9345 &job.spec,
9346 &mut seat,
9347 &Invocation {
9348 scope: crate::agent::InvocationScope::Repository,
9349 coordination: coordination.as_ref(),
9350 fetched_base: fetched_base.as_ref(),
9351 cwd: &job.cwd,
9352 prompt: &job.prompt,
9353 timeout: job.timeout,
9354 allow_write: job.allow_write,
9355 unsandboxed: false,
9356 sessions: job.sessions,
9357 artifacts: &job.artifacts,
9358 stem: &job.stem,
9359 run: &run,
9360 node: &node,
9361 cache_dir: cache.as_deref(),
9362 attachments: &attachments,
9363 writable: &[],
9364 },
9365 ))
9366 .await;
9367 let interrupted = out
9368 .as_ref()
9369 .err()
9370 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9371 let out = match out {
9372 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9373 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9374 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9382 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9383 Ok(o) => AgentOutcome::Failed(format!(
9384 "process exited with no usable output ({})",
9385 crate::notices::exit_status(o.exit_code)
9386 )),
9387 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9388 AgentOutcome::ResourceBlocked(e.to_string())
9389 }
9390 Err(e) => AgentOutcome::Failed(e.to_string()),
9391 };
9392 (i, seat, out, interrupted)
9393 });
9394 }
9395 drop(spawned_tx);
9396 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9397 loop {
9398 let joined = tokio::select! {
9399 Some(key) = spawned_rx.recv() => {
9400 state.seat_spawned(&key);
9402 if let Err(e) = state.save() {
9403 tracing::warn!("could not persist a seat's start: {e:#}");
9404 }
9405 continue;
9406 }
9407 joined = set.join_next() => joined,
9408 };
9409 let Some(joined) = joined else {
9410 break;
9411 };
9412 let (i, seat, out, interrupted) = match joined {
9413 Ok(v) => v,
9414 Err(e) => {
9418 tracing::error!("agent task panicked: {e}");
9419 continue;
9420 }
9421 };
9422 state.seat_finished(&seat.key);
9423 if interrupted
9424 && node == "implement"
9425 && let Some(label) = state
9426 .candidates
9427 .iter()
9428 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9429 .map(|candidate| candidate.label)
9430 {
9431 state.candidate_preservation.insert(
9432 label,
9433 "interrupted implementation awaiting persistence".into(),
9434 );
9435 state.interrupted_implementations.insert(label);
9436 }
9437 if let AgentOutcome::ResourceBlocked(why) = &out
9438 && state.resource_block.is_none()
9439 {
9440 state.resource_block = Some(crate::run::ResourceBlock {
9441 node: node.to_owned(),
9442 kind: if why.contains("disk pressure") {
9443 "disk pressure"
9444 } else if why.contains("repository coordination") {
9445 "repository coordination"
9446 } else {
9447 "parallel slot"
9448 }
9449 .to_owned(),
9450 detail: why.clone(),
9451 at: Timestamp::now(),
9452 });
9453 let activity = if interrupted {
9454 "agent execution interrupted"
9455 } else {
9456 "no agent was started"
9457 };
9458 state.event("resource", format!("{node}: {activity} - {why}"));
9459 }
9460 record_jobs(state, node, round, &seat.key, &out);
9461 if let Err(e) = state.save() {
9462 tracing::warn!("could not persist a seat's completion: {e:#}");
9463 }
9464 if collected.len() <= i {
9465 collected.resize_with(i + 1, || None);
9466 }
9467 collected[i] = Some((i, seat, out));
9468 }
9469 if state
9475 .active
9476 .values()
9477 .any(|a| a.node == node && a.attempt == attempt)
9478 {
9479 state
9480 .active
9481 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9482 if let Err(e) = state.save() {
9483 tracing::warn!("could not persist the end of a wave: {e:#}");
9484 }
9485 }
9486 if let Some(cache_dir) = cache
9493 && jobs_had_a_writer
9494 {
9495 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9496 }
9497 if let Some(guard) = cache_guard {
9498 guard.release();
9499 }
9500 collected.into_iter().flatten().collect()
9501}
9502
9503fn record_jobs(
9514 state: &mut RunState,
9515 node: &str,
9516 round: Option<usize>,
9517 seat: &str,
9518 out: &AgentOutcome,
9519) {
9520 let commands: &[agent::CommandEvidence] = match out {
9521 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9522 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9523 };
9524 let checked_at = Timestamp::now();
9525 for c in commands {
9526 state.jobs.push(JobRecord {
9527 node: node.to_owned(),
9528 round,
9529 seat: seat.to_owned(),
9530 id: c.id.clone(),
9531 description: c.description.clone(),
9532 checked_at,
9533 status: match c.exit_code {
9534 Some(0) => JobStatus::Completed,
9535 Some(_) => JobStatus::Failed,
9536 None => JobStatus::Unknown,
9537 },
9538 exit_code: c.exit_code,
9539 result_summary: c.result_summary.clone(),
9540 source: c.source.clone(),
9541 });
9542 }
9543}
9544
9545fn round_is_clean(
9566 blocking: usize,
9567 e2e_ok: bool,
9568 answered: usize,
9569 expected: usize,
9570 quota_missing: usize,
9571 policy: IncompleteReviewPolicy,
9572) -> bool {
9573 if blocking != 0 || !e2e_ok {
9574 return false;
9575 }
9576 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9577 return true;
9578 }
9579 answered > 0 && expected - answered <= quota_missing
9580}
9581
9582pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9606 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9607 return Some(RunStatus::Gating);
9608 }
9609 let last = reviews.last()?;
9610 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9611 if reviews.len() < max_rounds && !stagnant {
9612 return None;
9613 }
9614 if last.incomplete() && last.blocking == 0 {
9615 return Some(RunStatus::Blocked);
9616 }
9617 if last.e2e_status() == E2eStatus::ResourceBlocked {
9618 return None;
9619 }
9620 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9621 RunStatus::Gating
9622 } else {
9623 RunStatus::Blocked
9624 })
9625}
9626
9627pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9642 if nudged {
9643 (full / 4).max(Duration::from_secs(120)).min(full)
9644 } else {
9645 full
9646 }
9647}
9648
9649#[allow(clippy::too_many_arguments)]
9678async fn ask_json_wave<T>(
9679 jobs: Vec<SeatJob>,
9680 sem: Arc<Semaphore>,
9681 retries: usize,
9682 roster: &[AgentSpec],
9683 ctx: &WaveCtx<'_>,
9684 losses: &mut Vec<QuotaLoss>,
9685 state: &mut RunState,
9686 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9687) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9688where
9689 T: serde::de::DeserializeOwned + Send + 'static,
9690{
9691 ask_wave_with(
9692 jobs,
9693 sem,
9694 retries,
9695 roster,
9696 ctx,
9697 losses,
9698 state,
9699 &|text: &str| {
9700 let v = verdict::extract_json::<T>(text)?;
9701 validate(&v)?;
9702 Ok(v)
9703 },
9704 )
9705 .await
9706}
9707
9708#[allow(clippy::too_many_arguments)]
9717async fn ask_wave_with<T>(
9718 jobs: Vec<SeatJob>,
9719 sem: Arc<Semaphore>,
9720 retries: usize,
9721 roster: &[AgentSpec],
9722 ctx: &WaveCtx<'_>,
9723 losses: &mut Vec<QuotaLoss>,
9724 state: &mut RunState,
9725 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9726) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9727where
9728 T: Send + 'static,
9729{
9730 let n = jobs.len();
9731 let originals: Vec<SeatJob> = jobs;
9732 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9733 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9734 let mut nudges: Vec<usize> = vec![0; n];
9741 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9745 let mut tried: Vec<BTreeSet<String>> = specs
9746 .iter()
9747 .map(|s| BTreeSet::from([s.id.clone()]))
9748 .collect();
9749 let starts: Vec<usize> = specs
9750 .iter()
9751 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9752 .collect();
9753 let carry = ctx.carry_seats && !roster.is_empty();
9754 let carried: Vec<BTreeSet<String>> = originals
9757 .iter()
9758 .map(|j| {
9759 state
9760 .seat_history
9761 .get(&j.seat.key)
9762 .filter(|_| carry)
9763 .map(|h| h.failed.clone())
9764 .unwrap_or_default()
9765 })
9766 .collect();
9767 let mut prev: Vec<Option<FailClass>> = originals
9768 .iter()
9769 .map(|j| {
9770 state
9771 .seat_history
9772 .get(&j.seat.key)
9773 .filter(|_| carry)
9774 .and_then(|h| h.last_fail.clone())
9775 })
9776 .collect();
9777 let next_agent =
9778 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9779 let others: BTreeSet<String> = specs
9780 .iter()
9781 .enumerate()
9782 .filter(|(j, _)| *j != i)
9783 .map(|(_, s)| s.id.clone())
9784 .collect();
9785 pick_successor(
9786 roster,
9787 starts[i],
9788 tried,
9789 carry.then(|| &carried[i]),
9790 &others,
9791 )
9792 .cloned()
9793 };
9794 let mut fresh: Vec<Option<String>> = vec![None; n];
9795 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9796 let mut pending: Vec<usize> = (0..n).collect();
9797
9798 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9803 for round in 0..max_rounds {
9804 if pending.is_empty() {
9805 break;
9806 }
9807 let mut batch = Vec::with_capacity(pending.len());
9808 let mut renudged: Vec<&str> = Vec::new();
9809 for &i in &pending {
9810 let src = &originals[i];
9811 let full: &str = match &src.handover {
9816 Some(h) if specs[i].id != src.spec.id => h,
9817 _ => &src.prompt,
9818 };
9819 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9822 (
9823 full.to_owned(),
9824 src.timeout,
9825 format!("{}-{word}-{}", src.stem, specs[i].id),
9826 )
9827 } else if nudges[i] == 0 {
9828 (full.to_owned(), src.timeout, src.stem.clone())
9829 } else {
9830 renudged.push(src.seat.key.as_str());
9831 let why = done[i]
9832 .as_ref()
9833 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9834 .unwrap_or_else(|| "no parsable answer".to_owned());
9835 let nudge = prompt::nudge(&why);
9836 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9837 let prompt = if nudged {
9838 nudge
9839 } else {
9840 format!("{full}\n\n---\n\n{nudge}")
9841 };
9842 (
9843 prompt,
9844 retry_budget(src.timeout, nudged),
9845 format!("{}-retry{}", src.stem, nudges[i]),
9846 )
9847 };
9848 batch.push(SeatJob {
9849 spec: specs[i].clone(),
9850 seat: seats[i].clone(),
9851 cwd: src.cwd.clone(),
9852 prompt,
9853 timeout,
9854 allow_write: src.allow_write,
9855 sessions: src.sessions,
9856 artifacts: src.artifacts.clone(),
9857 stem,
9858 handover: None,
9859 });
9860 }
9861
9862 if !renudged.is_empty() {
9863 state.event(
9864 ctx.node,
9865 format!("retry {round}: re-asking {}", renudged.join(", ")),
9866 );
9867 }
9868 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9869 let mut still = Vec::new();
9870 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9871 seats[i] = seat;
9872 if let AgentOutcome::ResourceBlocked(why) = &out {
9875 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9876 continue;
9877 }
9878 let class = FailClass::of(&out);
9879 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9882 && nudges[i] < retries
9883 && !roster.is_empty();
9884 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9885 let next = should_hand_over(prev[i].as_ref(), &cur)
9886 .then(|| next_agent(i, &tried[i], &specs))
9887 .flatten();
9888 if carry {
9889 let h = state
9890 .seat_history
9891 .entry(originals[i].seat.key.clone())
9892 .or_default();
9893 h.failed.insert(specs[i].id.clone());
9894 h.last_fail = Some(cur.clone());
9895 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9896 h.last_ok = None;
9897 }
9898 if let Err(e) = state.save() {
9901 tracing::warn!("could not persist a seat's failure history: {e:#}");
9902 }
9903 }
9904 if let Some(next) = next {
9905 record_handover(
9906 state,
9907 ctx.node,
9908 &originals[i].seat.key,
9909 &specs[i].id,
9910 &next.id,
9911 &cur,
9912 &fail_reason(&out),
9913 );
9914 tried[i].insert(next.id.clone());
9915 prev[i] = Some(cur.clone());
9916 seats[i] =
9917 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9918 specs[i] = next;
9919 fresh[i] = Some(cur.stem_word().to_owned());
9920 nudges[i] = 0;
9921 done[i] = Some(Err(anyhow::anyhow!(
9922 "handed over after: {}",
9923 fail_reason(&out)
9924 )));
9925 still.push(i);
9926 continue;
9927 }
9928 }
9929 if carry
9930 && !nudge_first
9931 && let Some(cur) = class.clone()
9932 {
9933 let h = state
9935 .seat_history
9936 .entry(originals[i].seat.key.clone())
9937 .or_default();
9938 h.failed.insert(specs[i].id.clone());
9939 h.last_fail = Some(cur);
9940 }
9941 let parsed = match out {
9942 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9943 AgentOutcome::Quota(o) => {
9944 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9945 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9946 }
9947 AgentOutcome::Dropped(o) => {
9951 let why = o
9952 .dropped
9953 .as_ref()
9954 .map(|d| d.why.as_str())
9955 .unwrap_or("the CLI ended the stream without delivering its answer");
9956 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9957 }
9958 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9959 Err(anyhow::anyhow!(e))
9960 }
9961 };
9962 let quota = class == Some(FailClass::Quota);
9963 let failed = parsed.is_err();
9964 done[i] = Some(parsed);
9965 if carry && !failed {
9966 let h = state
9967 .seat_history
9968 .entry(originals[i].seat.key.clone())
9969 .or_default();
9970 h.failed.remove(&specs[i].id);
9971 h.last_ok = Some(specs[i].id.clone());
9972 h.last_fail = None;
9973 }
9974 let agent_failure = class.is_some()
9982 && !nudge_first
9983 && !roster.is_empty()
9984 && next_agent(i, &tried[i], &specs).is_some();
9985 if failed && !quota && !agent_failure && nudges[i] < retries {
9986 nudges[i] += 1;
9987 still.push(i);
9988 }
9989 }
9990 pending = still;
9991 }
9992
9993 for (i, q) in last_quota.into_iter().enumerate() {
9996 if let Some(reset) = q {
9997 losses.push(QuotaLoss {
9998 seat: originals[i].seat.key.clone(),
9999 node: ctx.node.to_owned(),
10000 at: Timestamp::now(),
10001 reset,
10002 });
10003 }
10004 }
10005
10006 seats
10007 .into_iter()
10008 .zip(done)
10009 .zip(nudges)
10010 .map(|((seat, res), attempts)| {
10011 (
10012 seat,
10013 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10014 attempts,
10015 )
10016 })
10017 .collect()
10018}
10019
10020fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10022 results
10023 .iter()
10024 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10025}
10026
10027fn block_wave(
10031 jobs: Vec<SeatJob>,
10032 state: &mut RunState,
10033 node: &str,
10034 attempt: usize,
10035 why: &str,
10036) -> Vec<(usize, SeatState, AgentOutcome)> {
10037 state.resource_wait = None;
10038 state
10039 .active
10040 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10041 if let Err(e) = state.save() {
10042 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10043 }
10044 jobs.into_iter()
10045 .enumerate()
10046 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10047 .collect()
10048}
10049
10050async fn acquire_cache_lease(
10063 state: &mut RunState,
10064 cache_dir: &Path,
10065 owner: &crate::cache::Owner,
10066 budget: Duration,
10067 context: &str,
10068 cancel: &(dyn Fn() -> bool + Sync),
10069) -> Result<Option<crate::cache::Guard>> {
10070 let home = crate::run::home();
10071 let started = Instant::now();
10072 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10073 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10074 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10075 Err(e) => {
10076 state.event(
10077 "verify",
10078 format!("{context}: could not check the shared build cache: {e:#}"),
10079 );
10080 if let Err(e2) = state.save() {
10081 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10082 }
10083 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10084 }
10085 };
10086 state.event(
10087 "verify",
10088 format!(
10089 "{context}: waiting for the shared build cache at {} ({})",
10090 cache_dir.display(),
10091 busy.describe()
10092 ),
10093 );
10094 state.resource_wait = Some(crate::run::ResourceWait {
10097 node: context.to_owned(),
10098 kind: "shared build cache".to_owned(),
10099 holder: busy.describe(),
10100 since: Timestamp::now(),
10101 limit_secs: budget.as_secs(),
10102 });
10103 if let Err(e) = state.save() {
10104 tracing::warn!("could not persist a cache wait: {e:#}");
10105 }
10106 let remaining = budget.saturating_sub(started.elapsed());
10107 let waited = crate::cache::wait_for_unless(
10108 &home,
10109 cache_dir,
10110 owner,
10111 remaining,
10112 Duration::from_secs(5),
10113 cancel,
10114 )
10115 .await;
10116 state.resource_wait = None;
10117 match waited {
10118 Ok(g) => Ok(g),
10119 Err(e) => {
10120 state.event("verify", format!("{context}: {e:#}"));
10121 if let Err(e2) = state.save() {
10122 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10123 }
10124 Err(e)
10125 }
10126 }
10127}
10128
10129#[derive(Debug)]
10134struct CacheCheckError(String);
10135
10136impl std::fmt::Display for CacheCheckError {
10137 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10138 f.write_str(&self.0)
10139 }
10140}
10141
10142impl std::error::Error for CacheCheckError {}
10143
10144#[allow(clippy::too_many_arguments)]
10165async fn with_cache_lease<'s, F, Fut>(
10166 state: &'s mut RunState,
10167 cache_dir: Option<&Path>,
10168 node: &str,
10169 seat: &str,
10170 worktree: &Path,
10171 head: &str,
10172 budget: Duration,
10173 context: &str,
10174 wait: &crate::repo_lock::WaitPolicy,
10175 body: F,
10176) -> Result<(Vec<CommandOutcome>, bool)>
10177where
10178 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10179 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10180{
10181 let started = Instant::now();
10184 let home = crate::run::home();
10185 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10186 let guard = if let Some(cache_dir) = cache_dir {
10187 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10188 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10189 wait.cancelled()
10190 })
10191 .await
10192 .and_then(|g| {
10193 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10194 }) {
10195 Ok(g) => Some(g),
10196 Err(e) => {
10197 return Ok((
10198 vec![CommandOutcome {
10199 command: "(waiting for the shared build cache)".into(),
10200 code: None,
10201 output_tail: e.to_string(),
10202 duration_ms: started.elapsed().as_millis() as u64,
10203 resource_blocked: true,
10204 }],
10205 false,
10206 ));
10207 }
10208 }
10209 } else {
10210 None
10211 };
10212 let _coordination = coordinate_operation(state, worktree, wait).await?;
10213 if let Some(cache_dir) = cache_dir {
10214 let identity = crate::cache::Identity::new(worktree, head);
10215 if let Err(e) = crate::cache::ensure_fresh(
10216 &home,
10217 cache_dir,
10218 &identity,
10219 &state.config.verify.cargo_manifests(),
10220 ) {
10221 state.event(
10222 "verify",
10223 format!(
10224 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10225 worktree.display(),
10226 short(head)
10227 ),
10228 );
10229 if let Some(guard) = guard {
10230 guard.release();
10231 }
10232 return Ok((
10233 vec![CommandOutcome {
10234 command: "(confirming the shared build cache is fresh)".into(),
10235 code: None,
10236 output_tail: e.to_string(),
10237 duration_ms: started.elapsed().as_millis() as u64,
10238 resource_blocked: true,
10239 }],
10240 false,
10241 ));
10242 }
10243 }
10244 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10245 if !timed_out_pids.is_empty() {
10246 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10247 }
10248 drop(_coordination);
10251 if let Some(guard) = guard {
10252 guard.release();
10253 }
10254 Ok((outcomes, retried))
10255}
10256
10257async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10269 wait_for_pids_with(
10270 pids,
10271 crate::proc::pid_alive,
10272 LEASE_RELEASE_POLL,
10273 LEASE_RELEASE_MAX_WAIT,
10274 )
10275 .await;
10276}
10277
10278async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10284 pids: &[u32],
10285 alive: F,
10286 poll: Duration,
10287 max_wait: Duration,
10288) {
10289 let deadline = Instant::now() + max_wait;
10290 loop {
10291 if pids.iter().all(|&pid| !alive(pid)) {
10292 return;
10293 }
10294 if Instant::now() >= deadline {
10295 return;
10296 }
10297 tokio::time::sleep(poll).await;
10298 }
10299}
10300
10301fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10308 outcomes.iter().any(|o| o.resource_blocked)
10309}
10310
10311pub(crate) enum GateFix {
10313 Retry,
10315 Stop,
10318 Defer,
10321}
10322
10323fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10331 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10332 red.peek().is_some()
10333 && red.all(|o| {
10334 !o.resource_blocked
10335 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10336 && !o.output_tail.trim().is_empty()
10337 })
10338}
10339
10340fn e2e_outcome_label(o: &CommandOutcome) -> String {
10344 if o.ok() {
10345 return "pass".to_owned();
10346 }
10347 let reason = if o.build_failed() {
10348 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10349 } else {
10350 format!("FAIL ({:?})", o.code)
10351 };
10352 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10353}
10354
10355async fn run_e2e_with_retry(
10363 state: &mut RunState,
10364 shell: &[String],
10365 commands: &[String],
10366 worktree: &Path,
10367 timeout: Duration,
10368 context: &str,
10369) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10370 let (mut e2e, mut timed_out_pids) = run_commands(
10371 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10372 )
10373 .await;
10374 for o in &e2e {
10375 state.event(
10376 "verify",
10377 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10378 );
10379 }
10380 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10384 if verify_retried {
10385 state.event(
10386 "verify",
10387 format!(
10388 "{context}: verify could not build/link, not a test result — retrying once \
10389 before concluding"
10390 ),
10391 );
10392 let retried = run_commands(
10393 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10394 )
10395 .await;
10396 e2e = retried.0;
10397 timed_out_pids.extend(retried.1);
10400 for o in &e2e {
10401 state.event(
10402 "verify",
10403 format!(
10404 "{context}: retry `{}` -> {}",
10405 o.command,
10406 e2e_outcome_label(o)
10407 ),
10408 );
10409 }
10410 }
10411 (e2e, verify_retried, timed_out_pids)
10412}
10413
10414#[allow(clippy::too_many_arguments)]
10430async fn run_commands(
10431 state: &mut RunState,
10432 node: &str,
10433 task: &str,
10434 attempt: usize,
10435 shell: &[String],
10436 commands: &[String],
10437 cwd: &Path,
10438 timeout: Duration,
10439) -> (Vec<CommandOutcome>, Vec<u32>) {
10440 if commands.is_empty() {
10441 return (Vec::new(), Vec::new());
10446 }
10447 let mut out = Vec::new();
10448 let mut timed_out_pids = Vec::new();
10449 let total = commands.len();
10450 for (idx, command) in commands.iter().enumerate() {
10451 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10452 if let Err(e) = state.save() {
10453 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10454 }
10455 let started = Instant::now();
10456 let mut cmd = tokio::process::Command::new(&shell[0]);
10457 cmd.quiet();
10458 cmd.args(&shell[1..])
10459 .arg(command)
10460 .current_dir(cwd)
10461 .stdin(std::process::Stdio::null())
10462 .stdout(std::process::Stdio::piped())
10463 .stderr(std::process::Stdio::piped())
10464 .kill_on_drop(true);
10465 let mut outputs = Vec::new();
10466 let protected = (|| -> Result<()> {
10467 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10468 outputs.push(guard);
10469 }
10470 if let Some(cache) = state.config.cache_dir() {
10471 outputs.push(crate::retention::OutputLease::cache(
10472 &crate::run::home(),
10473 &cache,
10474 false,
10475 )?);
10476 }
10477 Ok(())
10478 })();
10479 if let Err(e) = protected {
10480 out.push(CommandOutcome {
10481 command: command.clone(),
10482 code: None,
10483 output_tail: format!("output ownership unavailable: {e:#}"),
10484 duration_ms: 0,
10485 resource_blocked: true,
10486 });
10487 continue;
10488 }
10489 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10490 let (code, body) = match spawned {
10491 Ok(mut running) => {
10492 let pid = running.child().id();
10493 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10494 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10495 let mut paths = vec![cwd.to_owned()];
10496 if let Some(cache) = state.config.cache_dir() {
10497 paths.push(cache);
10498 }
10499 let mut pressure = None;
10500 let status = tokio::select! {
10501 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10502 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10503 pressure = Some(reason); None
10504 }
10505 };
10506 let (code, failure) = match status {
10507 Some(Ok(Ok(status))) => (
10508 status.code(),
10509 status
10510 .code()
10511 .is_none()
10512 .then(|| format!("command terminated: {status}")),
10513 ),
10514 Some(Ok(Err(error))) => {
10515 (None, Some(format!("failed waiting for command: {error}")))
10516 }
10517 Some(Err(_)) => {
10518 if !running.coordinated()
10522 && let Some(pid) = pid
10523 {
10524 timed_out_pids.push(pid);
10525 }
10526 (
10527 None,
10528 Some(format!("timed out after {}s", timeout.as_secs())),
10529 )
10530 }
10531 None => (None, None),
10532 };
10533 running.finish().await;
10534 if let Some(reason) = pressure {
10535 state.resource_block = Some(crate::run::ResourceBlock {
10536 node: node.to_owned(),
10537 kind: "disk pressure".to_owned(),
10538 detail: reason.clone(),
10539 at: Timestamp::now(),
10540 });
10541 out.push(CommandOutcome {
10542 command: command.clone(),
10543 code: None,
10544 output_tail: reason,
10545 duration_ms: started.elapsed().as_millis() as u64,
10546 resource_blocked: true,
10547 });
10548 break;
10549 }
10550 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10551 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10552 if let Some(failure) = failure {
10553 body.push('\n');
10554 body.push_str(&failure);
10555 }
10556 (code, body)
10557 }
10558 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10559 };
10560 out.push(CommandOutcome {
10561 command: command.clone(),
10562 code,
10563 output_tail: tail(&body, OUTPUT_TAIL),
10564 duration_ms: started.elapsed().as_millis() as u64,
10565 resource_blocked: false,
10566 });
10567 }
10568 state.task_finished(task);
10569 if let Err(e) = state.save() {
10570 tracing::warn!("could not persist the end of {task}: {e:#}");
10571 }
10572 (out, timed_out_pids)
10573}
10574
10575fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10587 let repo = repo.display();
10588 match style {
10589 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10590 MergeStyle::Squash => {
10591 let subject = message
10594 .lines()
10595 .next()
10596 .unwrap_or(branch)
10597 .replace(['\\', '"', '$', '`'], "");
10598 format!(
10599 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10600 )
10601 }
10602 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10603 }
10604}
10605
10606const PR_TITLE_MAX: usize = 240;
10617
10618fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10629 let line = queue::first_line(text)?;
10630 let chars: Vec<char> = line.chars().collect();
10631 let (kept, mark) = if chars.len() <= max {
10632 (line.to_owned(), "")
10633 } else if let Some(end) = sentence_end(&chars, max) {
10634 (chars[..end].iter().collect::<String>(), "")
10635 } else {
10636 let room = max.saturating_sub(3);
10637 let cut = if chars[room].is_whitespace() {
10640 room
10641 } else {
10642 chars[..room]
10643 .iter()
10644 .rposition(|c| c.is_whitespace())
10645 .unwrap_or(room)
10646 };
10647 let head: String = chars[..cut].iter().collect();
10648 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10649 (head.to_owned(), "...")
10650 };
10651 if kept.is_empty() {
10652 return None;
10653 }
10654 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10655 return None;
10656 }
10657 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10658 return None;
10659 }
10660 Some(format!("{kept}{mark}"))
10661}
10662
10663fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10667 const MIN: usize = 20;
10668 for i in MIN..max.min(chars.len()) {
10669 if !matches!(chars[i], '.' | '!' | '?') {
10670 continue;
10671 }
10672 let Some(&next) = chars.get(i + 1) else {
10673 continue;
10674 };
10675 if !next.is_whitespace() {
10676 continue;
10677 }
10678 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10679 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10680 continue;
10681 }
10682 let word: String = chars[..i]
10683 .iter()
10684 .rev()
10685 .take_while(|c| !c.is_whitespace())
10686 .collect::<Vec<_>>()
10687 .into_iter()
10688 .rev()
10689 .collect();
10690 let word = word.to_ascii_lowercase();
10691 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10692 continue;
10693 }
10694 let end = chars[..i]
10695 .iter()
10696 .rposition(|c| !c.is_whitespace())
10697 .map_or(i, |p| p + 1);
10698 return Some(end);
10699 }
10700 None
10701}
10702
10703struct PrMessage {
10708 title: String,
10709 body: String,
10710}
10711
10712impl PrMessage {
10713 fn commit_message(&self) -> String {
10717 format!("{}\n\n{}", self.title, self.body)
10718 }
10719}
10720
10721fn title_marker(line: &str) -> Option<&str> {
10723 let line = line.trim();
10724 let head = line.get(..6)?;
10725 head.eq_ignore_ascii_case("title:")
10726 .then(|| line[6..].trim())
10727}
10728
10729fn summary_title(summary: &str) -> Option<String> {
10734 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10735 let raw = title_marker(first)?;
10736 if raw.is_empty() {
10737 return None;
10738 }
10739 let title = queue::title_from(raw, PR_TITLE_MAX);
10740 let lower = title.to_ascii_lowercase();
10741 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10742 return None;
10743 }
10744 Some(title)
10745}
10746
10747const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10749
10750pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10752
10753fn is_review_run(state: &RunState) -> bool {
10757 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10758}
10759
10760fn review_title(state: &RunState) -> Option<String> {
10765 english_subject(
10766 state.reviewed_commits.as_ref()?.first()?,
10767 state.config.graph.github_english(),
10768 )
10769}
10770
10771fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10774 let raw = raw.trim();
10775 if raw.is_empty() {
10776 return None;
10777 }
10778 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10779 return None;
10780 }
10781 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10782 return None;
10783 }
10784 let title = queue::title_from(raw, PR_TITLE_MAX);
10785 let lower = title.to_ascii_lowercase();
10786 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10787 return None;
10788 }
10789 Some(title)
10790}
10791
10792#[derive(Debug, Clone, PartialEq, Eq)]
10795struct BranchFacts {
10796 commits: Vec<(String, String)>,
10798 stat: String,
10800}
10801
10802const STAT_FILE_LINES: usize = 25;
10804const COMMITS_MAX_CHARS: usize = 20_000;
10806
10807async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10810 let commits = git::commit_log(repo, base, branch).await.ok()?;
10811 if commits.is_empty() {
10812 return None;
10813 }
10814 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10815 let lines: Vec<&str> = stat.lines().collect();
10816 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10817 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10818 let more = format!(" ... {omitted} more file(s)");
10819 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10820 kept.push(&more);
10821 kept.push(lines[lines.len() - 1]);
10822 kept.join("\n")
10823 } else {
10824 lines.join("\n")
10825 };
10826 Some(BranchFacts { commits, stat })
10827}
10828
10829fn markdown_safe(text: &str) -> String {
10832 text.replace("</details>", "</details>")
10833 .replace("\x60\x60\x60", "~~~")
10834}
10835
10836fn neutral_title(state: &RunState, winner: char) -> String {
10837 format!(
10838 "chore: land candidate {} of run {}",
10839 winner.to_ascii_uppercase(),
10840 state.id
10841 )
10842}
10843
10844pub(crate) async fn repair_landing_gate(
10847 state: &mut RunState,
10848 winner: &Candidate,
10849 outcomes: &[CommandOutcome],
10850) -> Result<GateFix> {
10851 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10852 return Ok(GateFix::Stop);
10853 }
10854 let mut runner = Runner {
10855 roles: state.config.resolve_roles()?,
10856 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10857 state: state.clone(),
10858 pause: Pause::new(),
10859 interrupt: Pause::new(),
10860 };
10861 let result = runner.gate_fix_round(winner, outcomes).await;
10862 *state = runner.state;
10863 result
10864}
10865
10866pub(crate) async fn check_gate_repair_e2e(
10869 state: &mut RunState,
10870 winner: &Candidate,
10871 wait: &crate::repo_lock::WaitPolicy,
10872) -> Result<GateFix> {
10873 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10874 return Ok(GateFix::Retry);
10875 }
10876 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10877 if state
10878 .gate_repair_e2e
10879 .as_ref()
10880 .is_some_and(|r| r.head == head && r.passed)
10881 {
10882 return Ok(GateFix::Retry);
10883 }
10884 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10885 head: head.clone(),
10886 passed: false,
10887 outcomes: Vec::new(),
10888 });
10889 state.save()?;
10890 let commands = state.config.verify.e2e.clone();
10891 let outcomes = if commands.is_empty() {
10892 Vec::new()
10893 } else {
10894 let shell = state.config.shell();
10895 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10896 let cache_dir = state.config.cache_dir();
10897 let (outcomes, _) = with_cache_lease(
10898 state,
10899 cache_dir.as_deref(),
10900 "e2e",
10901 "e2e",
10902 &winner.worktree,
10903 &head,
10904 timeout,
10905 "gate repair e2e",
10906 wait,
10907 |state, budget| async move {
10908 run_e2e_with_retry(
10909 state,
10910 &shell,
10911 &commands,
10912 &winner.worktree,
10913 budget,
10914 "gate repair e2e",
10915 )
10916 .await
10917 },
10918 )
10919 .await?;
10920 outcomes
10921 };
10922 let deferred = verify_inconclusive(&outcomes);
10923 let passed = outcomes.iter().all(CommandOutcome::ok)
10924 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10925 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10926 head,
10927 passed,
10928 outcomes,
10929 });
10930 state.save()?;
10931 if deferred {
10932 Ok(GateFix::Defer)
10933 } else if !passed {
10934 state.event(
10935 "gate",
10936 "verify.e2e failed after gate repair; nothing published",
10937 );
10938 Ok(GateFix::Stop)
10939 } else {
10940 Ok(GateFix::Retry)
10941 }
10942}
10943
10944pub(crate) async fn run_landing_gate(
10946 state: &mut RunState,
10947 winner: &Candidate,
10948 wait: &crate::repo_lock::WaitPolicy,
10949) -> Result<Vec<CommandOutcome>> {
10950 let shell = state.config.shell();
10951 let gate_commands = state.config.verify.gate.clone();
10952 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10953 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10954 Vec::new()
10955 } else {
10956 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10957 let cache_dir = state.config.cache_dir();
10958 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10959 let (outcomes, _) = with_cache_lease(
10960 state,
10961 cache_dir.as_deref(),
10962 "gate",
10963 "gate",
10964 &winner.worktree,
10965 &head,
10966 timeout,
10967 "final gate",
10968 wait,
10969 |state, budget| {
10970 let shell = shell.clone();
10971 let gate_commands = gate_commands.clone();
10972 let winner = winner.clone();
10973 async move {
10974 Runner::run_pre_gate(state, &winner).await;
10975 let (outcomes, timed_out_pids) = run_commands(
10976 state,
10977 "gate",
10978 "gate",
10979 0,
10980 &shell,
10981 &gate_commands,
10982 &winner.worktree,
10983 budget,
10984 )
10985 .await;
10986 (outcomes, false, timed_out_pids)
10987 }
10988 },
10989 )
10990 .await?;
10991 outcomes
10992 };
10993 if outcomes.is_empty() {
10994 state.event("gate", "no gate commands configured; not tested");
10999 }
11000 for o in &outcomes {
11001 state.event(
11002 "gate",
11003 format!(
11004 "`{}` -> {}",
11005 o.command,
11006 if o.ok() {
11007 "pass".to_owned()
11008 } else {
11009 format!(
11010 "FAIL ({:?})\n{}",
11011 o.code,
11012 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11013 )
11014 }
11015 ),
11016 );
11017 }
11018 Ok(outcomes)
11019}
11020
11021pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11027 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11028 ""
11029 } else {
11030 pr_title
11031 }
11032}
11033
11034pub fn landing_subject_source(state: &RunState) -> String {
11038 if is_review_run(state) {
11039 let winner = state.candidates.first().map_or('A', |c| c.label);
11040 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11041 }
11042 state.instruction.clone()
11043}
11044
11045fn summary_without_title(summary: &str) -> String {
11048 let mut lines = summary.trim().lines().peekable();
11049 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11050 lines.next();
11051 }
11052 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11053}
11054
11055#[cfg(test)]
11067fn pr_message(state: &RunState, winner: char) -> PrMessage {
11068 pr_message_with(state, winner, None)
11069}
11070
11071#[cfg(test)]
11074fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11075 let raw = pr_message_raw(
11076 state,
11077 winner,
11078 facts,
11079 crate::pr_outcome::FinalHead::Unreported,
11080 ChangeNote::NotAsked,
11081 );
11082 let id = crate::scrub::Identity::current();
11083 PrMessage {
11084 title: crate::scrub::scrub(&raw.title, &id),
11085 body: crate::scrub::scrub(&raw.body, &id),
11086 }
11087}
11088
11089#[derive(Debug, Clone, Copy)]
11091enum ChangeNote<'a> {
11092 NotAsked,
11094 Summary(&'a crate::run::ChangeSummary),
11096 Unavailable(&'a str),
11098}
11099
11100fn pr_message_raw(
11101 state: &RunState,
11102 winner: char,
11103 facts: Option<&BranchFacts>,
11104 head: crate::pr_outcome::FinalHead<'_>,
11105 change: ChangeNote<'_>,
11106) -> PrMessage {
11107 if !matches!(change, ChangeNote::NotAsked) {
11108 let (title, mut body) = match change {
11109 ChangeNote::Summary(c) => (
11110 c.title.clone(),
11111 format!("## Summary\n\n{}\n\n", c.body.trim()),
11112 ),
11113 ChangeNote::Unavailable(why) => (
11114 neutral_title(state, winner),
11115 format!(
11116 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11117 ),
11118 ),
11119 ChangeNote::NotAsked => unreachable!(),
11120 };
11121 body.push_str(&public_pr_evidence(state, head));
11122 body.push_str(&format!(
11123 "\n---\nmagi:run/{} magi:candidate-{}\n",
11124 state.id,
11125 winner.to_ascii_lowercase()
11126 ));
11127 return PrMessage { title, body };
11128 }
11129 pr_preview(state, winner, facts, head)
11130}
11131
11132fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11135 let mut body = crate::pr_outcome::review_rounds(state);
11136 body.push_str(&crate::pr_outcome::final_verification(state, head));
11137 let open = state.last_round_findings();
11138 if !open.is_empty() {
11139 body.push_str("## Open review findings\n\n");
11140 for f in open {
11141 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11142 }
11143 body.push('\n');
11144 }
11145 if let Some(c) = state.change_summary.as_ref()
11146 && head == crate::pr_outcome::FinalHead::At(&c.head)
11147 {
11148 body.push_str(&format!(
11149 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11150 c.base, c.head
11151 ));
11152 }
11153 body
11154}
11155
11156fn pr_preview(
11158 state: &RunState,
11159 winner: char,
11160 facts: Option<&BranchFacts>,
11161 head: crate::pr_outcome::FinalHead<'_>,
11162) -> PrMessage {
11163 let summary = state
11164 .candidates
11165 .iter()
11166 .find(|c| c.label == winner)
11167 .map(|c| c.summary.as_str())
11168 .unwrap_or_default();
11169 let review = is_review_run(state);
11172 let title = if review {
11173 facts
11174 .and_then(|f| {
11175 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11176 })
11177 .or_else(|| review_title(state))
11178 .or_else(|| {
11179 state
11180 .candidates
11181 .iter()
11182 .find(|c| c.label == winner)
11183 .filter(|c| !c.branch.starts_with("magi/"))
11184 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11185 })
11186 .unwrap_or_else(|| neutral_title(state, winner))
11187 } else {
11188 summary_title(summary).unwrap_or_else(|| {
11189 english_title(
11190 &state.instruction,
11191 PR_TITLE_MAX,
11192 state.config.graph.github_english(),
11193 )
11194 .unwrap_or_else(|| neutral_title(state, winner))
11195 })
11196 };
11197
11198 let mut body = String::new();
11199 let what = summary_without_title(summary);
11200 if !what.is_empty() {
11201 body.push_str("## Summary\n\n");
11202 body.push_str(&what);
11203 body.push_str("\n\n");
11204 }
11205
11206 body.push_str(&crate::pr_outcome::review_rounds(state));
11208 body.push_str(&crate::pr_outcome::final_verification(state, head));
11209 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11210
11211 let open = state.open_findings();
11212 if !open.is_empty() {
11213 body.push_str("## Open review findings\n\n");
11214 for f in &open {
11215 body.push_str(&format!(
11216 "- `{}` [{:?}] {}\n",
11217 f.id, f.severity, "Context retained in local review records."
11218 ));
11219 }
11220 body.push('\n');
11221 }
11222
11223 if let Some(fix) = fix
11224 && !fix.rejected.is_empty()
11225 {
11226 body.push_str("## Declined by the fixer\n\n");
11227 for r in &fix.rejected {
11228 body.push_str(&format!(
11229 "- `{}`: {}\n",
11230 r.id, "Reason retained in local review records."
11231 ));
11232 }
11233 body.push('\n');
11234 }
11235
11236 if review {
11237 body.push_str("## Commits under review\n\n");
11239 if let Some(facts) = facts {
11240 let mut left = COMMITS_MAX_CHARS;
11241 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11242 let mut entry = format!("- {}\n", markdown_safe(subject));
11243 for l in markdown_safe(text).lines() {
11244 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11245 }
11246 if left == 0 {
11247 body.push_str(&format!(
11248 "- ... {} more commit(s)\n",
11249 facts.commits.len() - i
11250 ));
11251 break;
11252 }
11253 if entry.len() > left {
11254 let mut end = left;
11257 while !entry.is_char_boundary(end) {
11258 end -= 1;
11259 }
11260 entry.truncate(end);
11261 entry.push_str("\n ... (truncated)\n");
11262 left = 0;
11263 } else {
11264 left -= entry.len();
11265 }
11266 body.push_str(&entry);
11267 }
11268 if !facts.stat.trim().is_empty() {
11269 body.push_str(&format!(
11270 "\n## Diff stat\n\n```\n{}\n```\n",
11271 markdown_safe(facts.stat.trim())
11272 ));
11273 }
11274 } else {
11275 match &state.reviewed_commits {
11276 Some(subjects) => {
11277 for s in subjects {
11278 body.push_str(&format!("- {}\n", s.trim()));
11279 }
11280 }
11281 None => {
11282 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11285 body.push_str(rest.trim());
11286 body.push('\n');
11287 }
11288 }
11289 }
11290 } else {
11291 let task = state.instruction.trim();
11292 let task = if task.is_empty() {
11293 "(empty task)"
11294 } else {
11295 task
11296 };
11297 body.push_str(&format!(
11298 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11299 task.replace("</details>", "</details>")
11300 ));
11301 }
11302
11303 body.push_str(&format!(
11304 "\n---\nmagi:run/{} magi:candidate-{}\n",
11305 state.id,
11306 winner.to_ascii_lowercase()
11307 ));
11308
11309 PrMessage { title, body }
11310}
11311
11312fn seeded_instruction(state: &RunState) -> String {
11315 let mut inputs = Vec::new();
11316 if let Some(start) = &state.sources.start_from {
11317 inputs.push(format!(
11318 "Start from `{}` at `{}` (including ancestry).",
11319 start.token, start.sha
11320 ));
11321 }
11322 for pick in &state.sources.pick_commits {
11323 inputs.push(format!(
11324 "Pick `{}` at `{}` (single commit).",
11325 pick.token, pick.sha
11326 ));
11327 }
11328 if inputs.is_empty() {
11329 state.instruction.clone()
11330 } else {
11331 format!(
11332 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11333 state.instruction,
11334 inputs.join("\n")
11335 )
11336 }
11337}
11338
11339async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11343 let base = &state.base_branch;
11344 let mut against = base.clone();
11345 if mode == MergeMode::Pr {
11346 let remote = &state.config.merge.remote;
11350 let tracking = format!("{remote}/{base}");
11351 let fetched = git::fetch(repo, remote, base).await;
11352 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11353 against = tracking;
11354 } else {
11355 return false;
11356 }
11357 }
11358 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11359}
11360
11361fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11364 let mut detail = format!(
11365 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11366 nothing to open a pull request for"
11367 );
11368 detail.push_str(
11369 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11370 );
11371 detail
11372}
11373
11374#[derive(Debug, PartialEq, Eq)]
11377enum PrPlan {
11378 Create,
11379 Adopt { url: String, title: String },
11380 Stop(String),
11381}
11382
11383fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11386 match found {
11387 Ok(land::OpenPr::None) => PrPlan::Create,
11388 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11389 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11390 "several open pull requests exist for this branch, not picking one: {}",
11391 urls.join(" ")
11392 )),
11393 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11394 }
11395}
11396
11397pub(crate) async fn gh_pr_create(
11399 cwd: &Path,
11400 base: &str,
11401 head: &str,
11402 title: &str,
11403 body: &str,
11404) -> Result<String> {
11405 let out = tokio::process::Command::new("gh")
11406 .args([
11407 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11408 ])
11409 .current_dir(cwd)
11410 .quiet()
11411 .stdin(std::process::Stdio::null())
11412 .output()
11413 .await
11414 .context("spawn gh")?;
11415 if out.status.success() {
11416 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11417 } else {
11418 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11419 }
11420}
11421
11422async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11424 match std::fs::symlink_metadata(path) {
11425 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11426 git::worktree_remove_clean(repo, path).await?;
11429 return Ok(());
11430 }
11431 Err(e) => return Err(e.into()),
11432 Ok(_) => {}
11433 }
11434 if !git::is_clean(path).await? {
11435 bail!(
11436 "worktree has uncommitted work; refusing removal: {}",
11437 path.display()
11438 );
11439 }
11440 if !git::worktree_remove_clean(repo, path).await? {
11441 bail!("worktree removal refused: {}", path.display());
11442 }
11443 Ok(())
11444}
11445
11446pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11455 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11456 bail!("refusing fold: {reason}");
11457 }
11458 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11459 bail!("candidate preservation incomplete; refusing cleanup");
11460 }
11461 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11465 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11466 let _driver = crate::driver::maintenance(home, &state.id)?;
11467 let mut output_guards = Vec::new();
11468 for entry in std::fs::read_dir(state.worktree_root())
11469 .into_iter()
11470 .flatten()
11471 .flatten()
11472 {
11473 if entry.path().join(".git").is_file() {
11474 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11475 }
11476 }
11477 let repo = state.repo.clone();
11478 let wait = repository_wait(state, None);
11481 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11482 Ok(lease) => lease,
11483 Err(e) => {
11484 state.save_under(home)?;
11485 return Err(e);
11486 }
11487 };
11488 let _source_outputs = if state.status == RunStatus::Merged
11492 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11493 {
11494 verify_merged_fold_sources(state, drop_winner, home).await?
11495 } else {
11496 Vec::new()
11497 };
11498 let root = state.worktree_root();
11499 let winner = state.tally.as_ref().map(|t| t.winner);
11500 let mut removed = Vec::new();
11501
11502 for i in 0..state.candidates.len() {
11503 let c = state.candidates[i].clone();
11504 let is_winner = Some(c.label) == winner;
11505 if is_winner && !drop_winner {
11506 continue;
11507 }
11508 let existed = c.worktree.exists();
11509 remove_preserved_worktree(&repo, &c.worktree).await?;
11512 if existed {
11513 removed.push(c.worktree.to_string_lossy().into_owned());
11514 }
11515 let handed_over = state.released_branches.contains(&c.branch);
11518 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11519 if !git::branch_delete(&repo, &c.branch).await? {
11520 bail!("candidate branch removal refused");
11521 }
11522 removed.push(c.branch.clone());
11523 }
11524 state.candidates[i].folded = true;
11525 }
11526
11527 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11528 let path = name.path();
11529 let keep = !drop_winner
11530 && winner.is_some_and(|w| {
11531 path.file_name()
11532 .is_some_and(|n| n == format!("cand-{w}").as_str())
11533 });
11534 if keep {
11535 continue;
11536 }
11537 remove_preserved_worktree(&repo, &path).await?;
11538 removed.push(path.to_string_lossy().into_owned());
11539 }
11540
11541 remove_if_empty(&root);
11550
11551 if state.enabled_worktree_config && drop_winner {
11552 git::release_worktree_config(&repo).await.ok();
11556 state.enabled_worktree_config = false;
11557 }
11558 state.save_under(home)?;
11559 Ok(removed)
11560}
11561
11562async fn verify_merged_fold_sources(
11563 state: &RunState,
11564 drop_winner: bool,
11565 home: &Path,
11566) -> Result<Vec<std::fs::File>> {
11567 let winner = state.tally.as_ref().map(|t| t.winner);
11568 let mut heads = BTreeSet::new();
11569 for candidate in &state.candidates {
11570 if Some(candidate.label) == winner && !drop_winner {
11571 continue;
11572 }
11573 match std::fs::symlink_metadata(&candidate.worktree) {
11574 Ok(_) => anyhow::ensure!(
11575 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11576 "candidate recovery source is outside the inventoried run root: {}",
11577 candidate.worktree.display()
11578 ),
11579 Err(error) if error.kind() == std::io::ErrorKind::NotFound && candidate.folded => {}
11580 Err(error) => {
11581 return Err(error).context(format!(
11582 "cannot inspect candidate recovery source: {}",
11583 candidate.worktree.display()
11584 ));
11585 }
11586 }
11587 if git::branch_exists(&state.repo, &candidate.branch).await? {
11588 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11589 heads.extend(
11590 git::git(
11591 &state.repo,
11592 &["reflog", "show", "--format=%H", &candidate.branch],
11593 )
11594 .await?
11595 .lines()
11596 .map(str::to_owned),
11597 );
11598 }
11599 }
11600 let mut cargo_guards = Vec::new();
11601 let entries = match std::fs::read_dir(state.worktree_root()) {
11602 Ok(entries) => Some(entries),
11603 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11604 Err(error) => return Err(error.into()),
11605 };
11606 for entry in entries.into_iter().flatten() {
11607 let path = entry?.path();
11608 let metadata = std::fs::symlink_metadata(&path)?;
11609 anyhow::ensure!(
11610 metadata.is_dir() && !metadata.file_type().is_symlink(),
11611 "non-directory or symlink recovery source retained: {}",
11612 path.display()
11613 );
11614 if !drop_winner
11615 && winner.is_some_and(|w| {
11616 path.file_name()
11617 .is_some_and(|name| name == format!("cand-{w}").as_str())
11618 })
11619 {
11620 continue;
11621 }
11622 anyhow::ensure!(
11623 path.join(".git").is_file(),
11624 "unregistered recovery source retained: {}",
11625 path.display()
11626 );
11627 anyhow::ensure!(
11628 git::git(
11629 &path,
11630 &[
11631 "status",
11632 "--porcelain",
11633 "--untracked-files=all",
11634 "--ignored",
11635 "--",
11636 ".",
11637 ":(exclude)target"
11638 ]
11639 )
11640 .await?
11641 .is_empty(),
11642 "source changes or ignored recovery files retained: {}",
11643 path.display()
11644 );
11645 match std::fs::symlink_metadata(path.join("target")) {
11646 Ok(_) => {
11647 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11648 }
11649 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11650 Err(error) => return Err(error.into()),
11651 }
11652 for checkpoint in [
11653 "rebase-merge",
11654 "rebase-apply",
11655 "sequencer",
11656 "CHERRY_PICK_HEAD",
11657 "MERGE_HEAD",
11658 "REBASE_HEAD",
11659 "REVERT_HEAD",
11660 "BISECT_LOG",
11661 "AUTO_MERGE",
11662 "MERGE_MSG",
11663 ] {
11664 let location = git::git(
11665 &path,
11666 &[
11667 "rev-parse",
11668 "--path-format=absolute",
11669 "--git-path",
11670 checkpoint,
11671 ],
11672 )
11673 .await?;
11674 anyhow::ensure!(
11675 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11676 if e.kind() == std::io::ErrorKind::NotFound),
11677 "Git recovery checkpoint retained: {checkpoint}"
11678 );
11679 }
11680 heads.insert(git::rev_parse(&path, "HEAD").await?);
11681 heads.extend(
11682 git::git(&path, &["reflog", "show", "--format=%H", "HEAD"])
11683 .await?
11684 .lines()
11685 .map(str::to_owned),
11686 );
11687 if let Ok(original) = git::rev_parse(&path, "ORIG_HEAD").await {
11688 heads.insert(original);
11689 }
11690 }
11691 if !heads.is_empty() {
11692 let reference = format!(
11696 "refs/remotes/{}/{}",
11697 state.config.merge.remote, state.base_branch
11698 );
11699 let base = git::rev_parse(&state.repo, &reference)
11700 .await
11701 .context("cannot prove merged sources against the fetched base")?;
11702 let mut squash_winner = None;
11703 let mut checked_squash = false;
11704 for head in heads {
11705 if git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, &base])
11706 .await?
11707 .ok()
11708 {
11709 continue;
11710 }
11711 if !checked_squash {
11712 squash_winner = proven_squash_winner(state, &base).await?;
11713 checked_squash = true;
11714 }
11715 let incorporated = if let Some(winner) = &squash_winner {
11716 git::git_raw(&state.repo, &["merge-base", "--is-ancestor", &head, winner])
11717 .await?
11718 .ok()
11719 } else {
11720 false
11721 };
11722 anyhow::ensure!(
11723 incorporated,
11724 "recovery source {head} is not proven incorporated in {reference}; retaining the entire run"
11725 );
11726 }
11727 }
11728 Ok(cargo_guards)
11729}
11730
11731async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
11737 let Some(winner) = state.winner() else {
11738 return Ok(None);
11739 };
11740 let head = git::rev_parse(&state.repo, &winner.branch).await?;
11741 if !git::git_raw(
11742 &state.repo,
11743 &["merge-base", "--is-ancestor", &state.base_commit, &head],
11744 )
11745 .await?
11746 .ok()
11747 {
11748 return Ok(None);
11749 }
11750 let patch = fold_source_patch(&state.repo, &state.base_commit, &head).await?;
11751 if patch.is_empty() {
11752 return Ok(None);
11753 }
11754 let range = format!("{}..{base}", state.base_commit);
11755 let commits = git::git(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
11756 for commit in commits.lines() {
11757 let parent = format!("{commit}^");
11758 if fold_source_patch(&state.repo, &parent, commit).await? == patch {
11759 return Ok(Some(head));
11760 }
11761 }
11762 Ok(None)
11763}
11764
11765async fn fold_source_patch(repo: &Path, from: &str, to: &str) -> Result<String> {
11766 git::git(
11767 repo,
11768 &[
11769 "-c",
11770 "core.quotePath=true",
11771 "diff",
11772 "--binary",
11773 "--full-index",
11774 "--no-renames",
11775 "--no-ext-diff",
11776 "--no-textconv",
11777 "--no-color",
11778 "--ignore-submodules=none",
11779 "--submodule=short",
11780 from,
11781 to,
11782 "--",
11783 ],
11784 )
11785 .await
11786}
11787
11788fn remove_if_empty(dir: &Path) {
11799 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11800 std::fs::remove_dir(dir).ok();
11801 }
11802}
11803
11804pub fn worst_open(state: &RunState) -> Option<Severity> {
11806 state
11807 .reviews
11808 .last()?
11809 .reviews
11810 .iter()
11811 .flat_map(|r| r.findings.iter())
11812 .map(|f| f.severity)
11813 .max()
11814}
11815
11816async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11820 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11821 git::acquire_worktree_config(repo).await?;
11822 state.enabled_worktree_config = true;
11823 }
11824 Ok(())
11825}
11826
11827#[cfg(test)]
11828mod tests {
11829 #[tokio::test]
11830 async fn launcher_base_fetch_requires_remote_success() {
11831 let tmp = tempfile::tempdir().unwrap();
11832 let repo = tmp.path().join("repo");
11833 let remote = tmp.path().join("remote");
11834 std::fs::create_dir(&repo).unwrap();
11835 std::fs::create_dir(&remote).unwrap();
11836 git::git(&remote, &["init", "--bare", "-b", "main"])
11837 .await
11838 .unwrap();
11839 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11840 git::git(
11841 &repo,
11842 &[
11843 "-c",
11844 "user.name=Fixture",
11845 "-c",
11846 "user.email=fixture@example.invalid",
11847 "commit",
11848 "--allow-empty",
11849 "-m",
11850 "fixture",
11851 ],
11852 )
11853 .await
11854 .unwrap();
11855 git::git(
11856 &repo,
11857 &["remote", "add", "origin", remote.to_str().unwrap()],
11858 )
11859 .await
11860 .unwrap();
11861 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11862 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11863 assert_eq!(
11864 resolve_base(&repo, "main", "origin", None)
11865 .await
11866 .unwrap()
11867 .commit,
11868 head
11869 );
11870 #[cfg(unix)]
11871 {
11872 let policy: crate::config::Coordination = toml::from_str(
11873 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11874 ).unwrap();
11875 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11876 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11877 let held = crate::repo_lock::SeatLease::operation_with(
11878 &repo,
11879 Some(&policy),
11880 &crate::repo_lock::WaitPolicy::default(),
11881 )
11882 .await
11883 .unwrap();
11884 assert!(held.is_some());
11885 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11886 tokio::pin!(fetch);
11887 assert!(
11888 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11889 .await
11890 .is_err()
11891 );
11892 assert!(
11893 !repo.join(".git/FETCH_HEAD").exists(),
11894 "no fetch before admission"
11895 );
11896 drop(held);
11897 let fetched = fetch.await.unwrap();
11898 assert_eq!(fetched.commit, head);
11899 assert_eq!(fetched.remote, "origin");
11900 assert_eq!(fetched.branch, "main");
11901 }
11902 git::git(
11904 &repo,
11905 &[
11906 "remote",
11907 "set-url",
11908 "origin",
11909 tmp.path().join("missing").to_str().unwrap(),
11910 ],
11911 )
11912 .await
11913 .unwrap();
11914 let error = resolve_base(&repo, "main", "origin", None)
11915 .await
11916 .unwrap_err()
11917 .to_string();
11918 assert!(error.contains("cannot read origin/main"));
11919 assert!(error.contains("refusing to branch off"));
11920 }
11921
11922 #[test]
11923 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11924 let leaked = vec!["chore(deps): update a crate".to_owned()];
11925 let own = "fix(daemon): apply a chosen action";
11926 assert!(should_retitle("", own, &leaked));
11927 assert!(should_retitle(
11928 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11929 own,
11930 &leaked
11931 ));
11932 assert!(should_retitle(
11933 "chore: land candidate A of run 1",
11934 own,
11935 &leaked
11936 ));
11937 assert!(should_retitle(
11938 "magi: candidate A (uncommitted work)",
11939 own,
11940 &leaked
11941 ));
11942 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11943 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11944 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11945 }
11946
11947 #[test]
11948 fn pr_merge_plan_creates_adopts_or_stops() {
11949 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11950 assert_eq!(
11951 pr_merge_plan(Ok(land::OpenPr::One {
11952 url: "u".into(),
11953 title: "t".into()
11954 })),
11955 PrPlan::Adopt {
11956 url: "u".into(),
11957 title: "t".into()
11958 }
11959 );
11960 let PrPlan::Stop(many) =
11961 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11962 else {
11963 panic!("many must stop");
11964 };
11965 assert!(many.contains('a') && many.contains('b'));
11966 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11967 panic!("a failed lookup must stop");
11968 };
11969 assert!(err.contains("bad token"));
11970 }
11971
11972 use super::*;
11973 use crate::run::GateStatus;
11974 use std::collections::BTreeMap;
11975 use std::time::Duration;
11976
11977 fn conductor() -> AgentSpec {
11978 AgentSpec {
11979 id: "conductor".to_owned(),
11980 kind: crate::config::AgentKind::Command,
11981 model: None,
11982 command: vec!["true".to_owned()],
11983 extra_args: Vec::new(),
11984 env: BTreeMap::new(),
11985 prompt_delivery: None,
11986 }
11987 }
11988
11989 fn spec(id: &str) -> AgentSpec {
11990 AgentSpec {
11991 id: id.to_owned(),
11992 kind: crate::config::AgentKind::Command,
11993 model: None,
11994 command: vec!["true".to_owned()],
11995 extra_args: Vec::new(),
11996 env: BTreeMap::new(),
11997 prompt_delivery: None,
11998 }
11999 }
12000
12001 fn ids(xs: &[&str]) -> BTreeSet<String> {
12002 xs.iter().map(|s| (*s).to_owned()).collect()
12003 }
12004
12005 #[test]
12006 fn pick_successor_skips_an_agent_another_seat_holds() {
12007 let roster = [spec("a"), spec("b"), spec("c")];
12009 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12010 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12011 }
12012
12013 #[test]
12014 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
12015 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
12017 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12018 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12019 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
12020 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
12021 }
12022
12023 #[test]
12024 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
12025 let roster = [spec("a"), spec("b")];
12026 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
12027 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12028 let carried = ids(&["b"]);
12029 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
12030 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12031 }
12032
12033 #[test]
12034 fn pick_successor_returns_none_without_any_untried_successor() {
12035 let roster = [spec("a"), spec("b")];
12036 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
12037 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
12038 let carried = ids(&["a"]);
12039 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
12040 }
12041
12042 #[test]
12043 fn pick_successor_rescue_avoids_another_seats_occupant() {
12044 let roster = [spec("a"), spec("b"), spec("c")];
12046 let carried = ids(&["b", "c"]);
12047 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
12048 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12049 }
12050
12051 #[test]
12052 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12053 let roster = [spec("a"), spec("b"), spec("c")];
12054 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12055 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12056 }
12057
12058 #[test]
12059 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12060 let roster = [spec("a"), spec("b"), spec("c")];
12061 let failed = ids(&["a", "b", "c"]);
12062 let mut tried = ids(&["b"]);
12064 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12065 assert_eq!(first.id, "a");
12066 tried.insert(first.id.clone());
12067 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12068 assert_eq!(second.id, "c");
12069 tried.insert(second.id.clone());
12070 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12071 }
12072
12073 #[test]
12074 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12075 let roster = [spec("a"), spec("b")];
12076 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12077 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12078 }
12079
12080 #[test]
12081 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12082 let roster = [spec("a"), spec("b"), spec("c")];
12083 let h = SeatHistory {
12084 failed: ids(&["a"]),
12085 last_ok: Some("b".to_owned()),
12086 last_fail: None,
12087 };
12088 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12089 let gone = SeatHistory {
12091 last_ok: Some("zzz".to_owned()),
12092 ..h.clone()
12093 };
12094 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12095 let failed = SeatHistory {
12096 failed: ids(&["a", "b"]),
12097 last_ok: Some("b".to_owned()),
12098 last_fail: None,
12099 };
12100 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12101 }
12102
12103 #[test]
12104 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12105 let first = SeatState::new("review-1", "alpha", 7);
12106 let next = handover_seat("review-1", "gamma", 7);
12107 assert_ne!(first.claude_session, next.claude_session);
12108 }
12109
12110 #[test]
12111 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12112 let mut state = state_with_summary("x", "y");
12113 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12114 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12115 assert_ne!(a.claude_session, b.claude_session);
12116 }
12117
12118 #[test]
12119 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12120 let roster = [spec("a"), spec("b")];
12121 let h = SeatHistory {
12122 failed: ids(&["a", "b"]),
12123 ..SeatHistory::default()
12124 };
12125 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12126 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12127 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12128 }
12129
12130 #[test]
12137 fn failure_signature_ignores_numbers_and_paths() {
12138 assert_eq!(
12139 failure_signature("exited with Some(2) and no usable output"),
12140 failure_signature("exited with Some(137) and no usable output")
12141 );
12142 assert_eq!(
12143 failure_signature("process exited with no usable output (exit code 2)"),
12144 failure_signature("process exited with no usable output (exit code 137)")
12145 );
12146 for (old, new) in [
12147 (
12148 "exited with Some(2) and no usable output",
12149 "process exited with no usable output (exit code 2)",
12150 ),
12151 (
12152 "exited with None and no usable output",
12153 "process exited with no usable output (exit code unavailable)",
12154 ),
12155 ] {
12156 let previous = FailClass::Other(failure_signature(old));
12157 let current = FailClass::Other(failure_signature(new));
12158 assert_eq!(previous, current, "resumption retains the failure class");
12159 assert!(!should_hand_over(Some(&previous), ¤t));
12160 }
12161 assert_eq!(
12162 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12163 failure_signature("cannot open /var/x.txt: denied")
12164 );
12165 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12166 }
12167
12168 #[test]
12169 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12170 let other = FailClass::Other("x".into());
12171 assert!(should_hand_over(None, &FailClass::Quota));
12172 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12173 assert!(should_hand_over(
12174 Some(&FailClass::Timeout),
12175 &FailClass::Timeout
12176 ));
12177 assert!(should_hand_over(None, &other));
12178 assert!(!should_hand_over(Some(&other), &other));
12179 assert!(should_hand_over(
12180 Some(&other),
12181 &FailClass::Other("y".into())
12182 ));
12183 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12185 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12186 }
12187
12188 #[test]
12189 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12190 let a = SeatState::new("judge-1", "alpha", 7);
12191 let b = handover_seat("judge-1", "beta", 7);
12192 assert_ne!(a.claude_session, b.claude_session);
12193 assert_eq!(b.turns, 0);
12194 }
12195
12196 #[test]
12197 fn a_timeout_is_classified_apart_from_other_failures() {
12198 assert_eq!(
12199 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12200 Some(FailClass::Timeout)
12201 );
12202 assert!(matches!(
12203 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12204 Some(FailClass::Other(_))
12205 ));
12206 }
12207
12208 #[test]
12209 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12210 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12211 let tried = BTreeSet::from(["beta".to_owned()]);
12212 let next = next_untried_in_roster(&roster, 1, &tried);
12215 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12216 }
12217
12218 #[test]
12219 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12220 let roster = vec![spec("alpha"), spec("beta")];
12221 let tried = BTreeSet::from(["beta".to_owned()]);
12222 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12226 }
12227
12228 #[test]
12229 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12230 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12231 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12232 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12236 }
12237
12238 #[test]
12239 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12240 let roster = vec![spec("a"), spec("a"), spec("b")];
12241 let tried = BTreeSet::from(["a".to_owned()]);
12242 let next = next_untried_in_roster(&roster, 0, &tried);
12243 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12244 }
12245
12246 #[test]
12247 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12248 let roster = vec![spec("a"), spec("b")];
12249 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12250 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12251 }
12252
12253 #[test]
12254 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12255 let dir = tempfile::tempdir().unwrap();
12256 let bay = dir.path().join("ffff");
12257
12258 remove_if_empty(&bay);
12260 assert!(!bay.exists());
12261
12262 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12265 remove_if_empty(&bay);
12266 assert!(bay.exists(), "non-empty directory must survive");
12267
12268 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12270 remove_if_empty(&bay);
12271 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12272 }
12273
12274 #[test]
12283 fn a_full_panel_that_found_nothing_is_clean() {
12284 assert!(round_is_clean(
12285 0,
12286 true,
12287 2,
12288 2,
12289 0,
12290 IncompleteReviewPolicy::Block
12291 ));
12292 }
12293
12294 #[test]
12295 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12296 assert!(!round_is_clean(
12297 0,
12298 true,
12299 1,
12300 2,
12301 0,
12302 IncompleteReviewPolicy::Block
12303 ));
12304 }
12305
12306 #[test]
12307 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12308 assert!(!round_is_clean(
12309 1,
12310 true,
12311 1,
12312 2,
12313 0,
12314 IncompleteReviewPolicy::Warn
12315 ));
12316 }
12317
12318 #[test]
12319 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12320 assert!(round_is_clean(
12321 0,
12322 true,
12323 1,
12324 2,
12325 0,
12326 IncompleteReviewPolicy::Warn
12327 ));
12328 }
12329
12330 #[test]
12331 fn a_full_panel_with_an_open_finding_is_not_clean() {
12332 assert!(!round_is_clean(
12333 1,
12334 true,
12335 2,
12336 2,
12337 0,
12338 IncompleteReviewPolicy::Block
12339 ));
12340 }
12341
12342 #[test]
12343 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12344 assert!(!round_is_clean(
12345 0,
12346 false,
12347 2,
12348 2,
12349 0,
12350 IncompleteReviewPolicy::Block
12351 ));
12352 }
12353
12354 #[test]
12361 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12362 assert!(round_is_clean(
12365 0,
12366 true,
12367 1,
12368 2,
12369 1,
12370 IncompleteReviewPolicy::Block
12371 ));
12372 }
12373
12374 #[test]
12375 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12376 assert!(!round_is_clean(
12379 0,
12380 true,
12381 1,
12382 2,
12383 0,
12384 IncompleteReviewPolicy::Block
12385 ));
12386 }
12387
12388 #[test]
12389 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12390 assert!(!round_is_clean(
12391 1,
12392 true,
12393 1,
12394 2,
12395 1,
12396 IncompleteReviewPolicy::Block
12397 ));
12398 assert!(!round_is_clean(
12399 0,
12400 false,
12401 1,
12402 2,
12403 1,
12404 IncompleteReviewPolicy::Block
12405 ));
12406 }
12407
12408 #[test]
12409 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12410 assert!(!round_is_clean(
12414 0,
12415 true,
12416 0,
12417 2,
12418 2,
12419 IncompleteReviewPolicy::Block
12420 ));
12421 }
12422
12423 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12424 CommandOutcome {
12425 command: "test".to_owned(),
12426 code,
12427 output_tail: String::new(),
12428 duration_ms: 0,
12429 resource_blocked,
12430 }
12431 }
12432
12433 #[test]
12434 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12435 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12436 assert!(
12437 !verify_inconclusive(&[outcome(Some(1), false)]),
12438 "an ordinary failure is still evidence about the patch"
12439 );
12440 assert!(verify_inconclusive(&[outcome(None, true)]));
12441 assert!(
12442 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12443 "one inconclusive outcome taints the whole batch"
12444 );
12445 assert!(!verify_inconclusive(&[]));
12446 }
12447
12448 #[tokio::test]
12449 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12450 let calls = std::sync::atomic::AtomicUsize::new(0);
12454 let started = Instant::now();
12455 wait_for_pids_with(
12456 &[123],
12457 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12458 Duration::from_millis(5),
12459 Duration::from_secs(5),
12460 )
12461 .await;
12462 assert!(
12463 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12464 "must keep checking rather than deciding on the first answer"
12465 );
12466 assert!(
12467 started.elapsed() < Duration::from_secs(1),
12468 "must return the moment it is confirmed dead, not wait out the ceiling"
12469 );
12470 }
12471
12472 #[tokio::test]
12473 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12474 let started = Instant::now();
12475 wait_for_pids_with(
12476 &[123],
12477 |_| true, Duration::from_millis(5),
12479 Duration::from_millis(30),
12480 )
12481 .await;
12482 let elapsed = started.elapsed();
12483 assert!(
12484 elapsed >= Duration::from_millis(30),
12485 "must not give up before its own ceiling: {elapsed:?}"
12486 );
12487 assert!(
12488 elapsed < Duration::from_secs(1),
12489 "must not wait past its own ceiling either: {elapsed:?}"
12490 );
12491 }
12492
12493 #[tokio::test]
12494 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12495 let started = Instant::now();
12496 wait_for_pids_with(
12497 &[],
12498 |_| true,
12499 Duration::from_secs(5),
12500 Duration::from_secs(5),
12501 )
12502 .await;
12503 assert!(
12504 started.elapsed() < Duration::from_millis(200),
12505 "an empty pid list has nothing to confirm"
12506 );
12507 }
12508
12509 fn review_round(
12515 clean: bool,
12516 blocking: usize,
12517 answered: usize,
12518 expected: usize,
12519 progressed: bool,
12520 e2e_ok: bool,
12521 ) -> ReviewRound {
12522 ReviewRound {
12523 round: 1,
12524 head: "h".to_owned(),
12525 verified_head: None,
12526 verified_at: None,
12527 reviews: Vec::new(),
12528 e2e: vec![CommandOutcome {
12529 command: "test".to_owned(),
12530 code: Some(if e2e_ok { 0 } else { 1 }),
12531 output_tail: String::new(),
12532 duration_ms: 0,
12533 resource_blocked: false,
12534 }],
12535 verify_retried: false,
12536 e2e_deferred: false,
12537 e2e_defer_reason: None,
12538 fix: None,
12539 blocking,
12540 answered,
12541 expected,
12542 clean,
12543 progressed,
12544 vote_split: false,
12545 reconsideration: Vec::new(),
12546 verdict: None,
12547 }
12548 }
12549
12550 #[test]
12551 fn review_conclusion_is_none_when_nothing_has_run() {
12552 assert_eq!(review_conclusion(&[], 3), None);
12553 }
12554
12555 #[test]
12556 fn review_conclusion_is_none_while_rounds_remain() {
12557 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12558 assert_eq!(review_conclusion(&rounds, 3), None);
12559 }
12560
12561 #[test]
12562 fn review_conclusion_is_gating_once_a_round_is_clean() {
12563 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12564 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12565 }
12566
12567 #[test]
12568 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12569 let rounds = vec![
12570 review_round(false, 1, 2, 2, true, true),
12571 review_round(false, 1, 2, 2, true, true),
12572 ];
12573 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12574 }
12575
12576 #[test]
12577 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12578 let rounds = vec![
12579 review_round(false, 1, 2, 2, true, true),
12580 review_round(false, 1, 2, 2, true, false),
12581 ];
12582 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12583 }
12584
12585 #[test]
12586 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12587 let mut blocked = review_round(false, 1, 2, 2, true, false);
12594 blocked.e2e[0].resource_blocked = true;
12595 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12596 assert_eq!(review_conclusion(&rounds, 2), None);
12597 }
12598
12599 #[test]
12600 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12601 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12603 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12604 }
12605
12606 #[test]
12607 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12608 let rounds = vec![
12609 review_round(false, 1, 2, 2, false, true),
12610 review_round(false, 1, 2, 2, false, true),
12611 ];
12612 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12613 }
12614
12615 fn secs(n: u64) -> Duration {
12616 Duration::from_secs(n)
12617 }
12618
12619 #[tokio::test]
12622 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12623 ask_test_home();
12624 let tmp = tempfile::tempdir().unwrap();
12625 let repo = tmp.path().join("repo");
12626 std::fs::create_dir_all(&repo).unwrap();
12627 init_repo(&repo);
12628 let g = |args: &[&str]| {
12629 let out = std::process::Command::new("git")
12630 .args(args)
12631 .current_dir(&repo)
12632 .quiet()
12633 .output()
12634 .unwrap();
12635 assert!(
12636 out.status.success(),
12637 "{}",
12638 String::from_utf8_lossy(&out.stderr)
12639 );
12640 };
12641 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12642 g(&["checkout", "-b", "historical"]);
12643 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12644 g(&["add", "."]);
12645 g(&["commit", "-m", "old alternative"]);
12646 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12647 g(&["checkout", "-b", "magi/task-selected", &base]);
12648 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12649 g(&["add", "."]);
12650 g(&["commit", "-m", "measurement infrastructure"]);
12651 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12652 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12653 g(&["add", "."]);
12654 g(&["commit", "-m", "dependent implementation"]);
12655 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12656 g(&["checkout", "main"]);
12657 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12658 g(&["add", "."]);
12659 g(&["commit", "-m", "unrelated base change"]);
12660 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12661 let origin = tmp.path().join("origin.git");
12662 g(&[
12663 "clone",
12664 "--bare",
12665 repo.to_str().unwrap(),
12666 origin.to_str().unwrap(),
12667 ]);
12668 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12669 g(&["fetch", "origin"]);
12670 let selection = crate::pr_recovery::Selection {
12671 url: "https://example.test/project/pull/1".into(),
12672 branch: "magi/task-selected".into(),
12673 head: head.clone(),
12674 source_head: None,
12675 source_run: None,
12676 source_proof: None,
12677 disposition: Some(crate::pr_recovery::Disposition::Review),
12678 instruction: "preserve measured behavior".into(),
12679 approved_at: jiff::Timestamp::now(),
12680 };
12681 let mut template = runner_at(RunStatus::Reviewing);
12682 template.state.repo = repo.clone();
12683 template.state.base_commit = current.clone();
12684 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12685 template.state.origin = Some(crate::run::Origin::queue("selected"));
12686 template.state.pr_head = Some(selection.branch.clone());
12687 template.state.pr_recovery = Some(selection.clone());
12688 let mut runner =
12689 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12690 .await
12691 .unwrap();
12692 let wt = runner.state.winner().unwrap().worktree.clone();
12693 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12694 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12695 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12696 assert!(runner.state.seeds.is_empty());
12697 assert_eq!(runner.state.instruction, "preserve measured behavior");
12698 runner.sync_to_base().await.unwrap();
12699 assert_eq!(
12700 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12701 "prerequisite"
12702 );
12703 assert_eq!(
12704 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12705 "measured implementation"
12706 );
12707 assert_eq!(
12708 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12709 "current main change"
12710 );
12711 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12712 assert_eq!(
12713 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12714 head
12715 );
12716
12717 let mut partial = runner_at(RunStatus::Reviewing);
12719 partial.state.repo = repo.clone();
12720 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12721 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12722 run: Some("exit 1".into()),
12723 ..Default::default()
12724 }];
12725 let selection = runner.state.pr_recovery.clone().unwrap();
12726 partial.state.pr_recovery = Some(selection.clone());
12727 partial.state.pr_head = Some(selection.branch.clone());
12728 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12729 let id = partial.state.id.clone();
12730 let error = match Runner::open_selected(
12731 &repo,
12732 partial.state,
12733 partial.roles,
12734 1,
12735 runner.state.base_commit.clone(),
12736 selection,
12737 )
12738 .await
12739 {
12740 Ok(_) => panic!("setup must fail"),
12741 Err(e) => e,
12742 };
12743 assert_eq!(
12744 error
12745 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12746 .unwrap()
12747 .run,
12748 id
12749 );
12750 let failed = RunState::load(&id).unwrap();
12751 assert_eq!(failed.status, RunStatus::Blocked);
12752 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12753 assert!(Runner::resume(&id).is_err());
12754 runner.state.status = RunStatus::Ready;
12755 runner.state.driver_exited = true;
12756 runner.state.save().unwrap();
12757 let mut retry = runner_at(RunStatus::Reviewing);
12758 retry.state.repo = repo.clone();
12759 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12760 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12761 reselected.approved_at = jiff::Timestamp::now();
12762 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12763 retry.state.pr_head = Some(reselected.branch.clone());
12764 retry.state.pr_recovery = Some(reselected.clone());
12765 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12766 .await
12767 .unwrap();
12768 let restarted = Runner::open_selected(
12769 &repo,
12770 retry.state,
12771 retry.roles,
12772 1,
12773 runner.state.base_commit.clone(),
12774 reselected,
12775 )
12776 .await
12777 .unwrap();
12778 assert_eq!(
12779 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12780 .await
12781 .unwrap(),
12782 head
12783 );
12784 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12785 assert_eq!(
12786 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12787 head
12788 );
12789 }
12790
12791 #[tokio::test]
12792 #[cfg(unix)]
12793 async fn fallback_and_interrupted_restoration_require_coordination() {
12794 use std::os::fd::AsRawFd as _;
12795 ask_test_home();
12796 let tmp = tempfile::tempdir().unwrap();
12797 let mut runner = registered_runner(tmp.path()).await;
12798 runner.state.config.graph.timeout_resource_wait = 1;
12799 let winner = runner.state.candidates[0].clone();
12800 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12801 let leftover = winner.worktree.join("leftover");
12802 std::fs::write(&leftover, "untrusted edits").unwrap();
12803 let next = tmp.path().join("next-fixer");
12804 let mut failed = spec("first");
12805 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12806 let mut successor = spec("second");
12807 successor.command = vec![
12808 "sh".into(),
12809 "-c".into(),
12810 format!("touch '{}'; echo done", next.display()),
12811 ];
12812 let attempts = vec![
12813 (failed.clone(), "first".into()),
12814 (successor.clone(), "second".into()),
12815 ];
12816 let lock_path = runner.state.repo.join(".git/updater.lock");
12817 let external = std::fs::File::create(&lock_path).unwrap();
12818 assert_eq!(
12819 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12820 0
12821 );
12822 let artifacts = tmp.path().join("artifacts");
12823 let build = |spec, seat| SeatJob {
12824 spec,
12825 seat,
12826 cwd: winner.worktree.clone(),
12827 prompt: "fixture".into(),
12828 timeout: Duration::from_secs(10),
12829 allow_write: true,
12830 sessions: false,
12831 artifacts: artifacts.clone(),
12832 stem: "fix".into(),
12833 handover: None,
12834 };
12835 let (_, _, out) = runner
12836 .ask_fixer_among(
12837 attempts.clone(),
12838 "graft",
12839 None,
12840 Some((&winner.worktree, &head)),
12841 build,
12842 )
12843 .await;
12844 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12845 assert!(leftover.exists(), "contention must prevent clean");
12846 assert!(!next.exists());
12847 assert!(runner.state.handovers.is_empty());
12848 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12850 from_run: "earlier".into(),
12851 generation: 1,
12852 mode: crate::recovery::Mode::VerifyRepair,
12853 reasons: Vec::new(),
12854 reviewed_head: None,
12855 rounds: Vec::new(),
12856 open_findings: Vec::new(),
12857 evidence: None,
12858 criteria_inherited: true,
12859 repair: crate::recovery::Repair::Started {
12860 before: head.clone(),
12861 },
12862 });
12863 assert!(
12864 runner
12865 .recovery_repair(&winner)
12866 .await
12867 .unwrap_err()
12868 .is::<crate::repo_lock::ResourceBlocked>()
12869 );
12870 assert!(matches!(
12871 runner.state.recovery.as_ref().unwrap().repair,
12872 crate::recovery::Repair::Started { .. }
12873 ));
12874 assert!(leftover.exists());
12875 drop(external);
12876 runner.recovery_repair(&winner).await.unwrap();
12877 assert!(!leftover.exists());
12878 assert!(matches!(
12879 runner.state.recovery.as_ref().unwrap().repair,
12880 crate::recovery::Repair::Done { .. }
12881 ));
12882 std::fs::remove_file(&lock_path).unwrap();
12884 std::fs::create_dir(&lock_path).unwrap();
12885 std::fs::write(&leftover, "untrusted edits").unwrap();
12886 let (_, _, out) = runner
12887 .ask_fixer_among(
12888 attempts,
12889 "graft",
12890 None,
12891 Some((&winner.worktree, &head)),
12892 build,
12893 )
12894 .await;
12895 assert!(matches!(out, AgentOutcome::Failed(_)));
12896 assert!(leftover.exists());
12897 assert!(!next.exists());
12898 assert_eq!(
12899 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12900 head
12901 );
12902 }
12903
12904 #[cfg(unix)]
12905 async fn registered_runner(root: &Path) -> Runner {
12906 ask_test_home();
12907 let repo = root.join("repo");
12908 std::fs::create_dir_all(&repo).unwrap();
12909 init_repo(&repo);
12910 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12911 let mut runner = runner_at(RunStatus::Prep);
12912 runner.state.repo = repo.clone();
12913 runner.state.config.coordination = crate::config::test_coordination();
12914 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12915 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12916 runner.state.config.graph.advise = false;
12917 runner.roles.implementers = vec![spec("implement")];
12918 runner.roles.judges = vec![spec("judge")];
12919 runner.prep().await.unwrap();
12920 runner
12921 }
12922
12923 #[tokio::test]
12924 #[cfg(unix)]
12925 async fn parked_implementer_persists_completed_work() {
12926 for registered in [false, true] {
12927 let tmp = tempfile::tempdir().unwrap();
12928 let mut runner = registered_runner(tmp.path()).await;
12929 if !registered {
12930 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12931 }
12932 let marker = tmp.path().join("started");
12933 let release = tmp.path().join("release");
12934 let mut implementer = spec("implement");
12935 implementer.command = vec![
12936 "sh".into(),
12937 "-c".into(),
12938 format!(
12939 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12940 marker.display(),
12941 release.display()
12942 ),
12943 ];
12944 runner.roles.implementers = vec![implementer.clone()];
12945 runner.state.config.agents = vec![implementer];
12946 let pause = runner.pause.clone();
12947 let (result, ()) = tokio::join!(runner.implement(), async {
12948 tokio::time::timeout(Duration::from_secs(10), async {
12949 while !marker.exists() {
12950 tokio::time::sleep(Duration::from_millis(10)).await;
12951 }
12952 })
12953 .await
12954 .unwrap();
12955 pause.park();
12956 std::fs::write(&release, "go").unwrap();
12957 });
12958 result.unwrap();
12959 let candidate = &runner.state.candidates[0];
12960 assert!(candidate.commits > 0, "{candidate:?}");
12961 assert!(candidate.summary.contains("saved-result"));
12962 assert_eq!(candidate.files, 1);
12963 let before = candidate.commits;
12964 runner.implement().await.unwrap();
12965 assert_eq!(runner.state.candidates[0].commits, before);
12966 }
12967 }
12968
12969 #[tokio::test]
12970 #[cfg(unix)]
12971 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12972 let tmp = tempfile::tempdir().unwrap();
12973 let mut runner = registered_runner(tmp.path()).await;
12974 let winner = runner.state.candidates[0].clone();
12975 let cache = tmp.path().join("cache");
12976 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12977 runner.state.config.graph.timeout_resource_wait = 0;
12978 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12979 let crate::cache::AcquireOutcome::Acquired(held) =
12980 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12981 else {
12982 panic!("cache must be free")
12983 };
12984 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12985 from_run: "earlier".into(),
12986 generation: 1,
12987 mode: crate::recovery::Mode::VerifyRepair,
12988 reasons: Vec::new(),
12989 reviewed_head: None,
12990 rounds: Vec::new(),
12991 open_findings: Vec::new(),
12992 evidence: None,
12993 criteria_inherited: true,
12994 repair: crate::recovery::Repair::NotStarted,
12995 });
12996 runner.recovery_repair(&winner).await.unwrap();
12997 assert!(matches!(
12998 runner.state.recovery.as_ref().unwrap().repair,
12999 crate::recovery::Repair::WaitingForRestore { .. }
13000 ));
13001 drop(held);
13002 runner.state.resource_block = None;
13003 runner.state.config.graph.timeout_resource_wait = 1;
13004 runner.recovery_repair(&winner).await.unwrap();
13005 assert!(matches!(
13006 runner.state.recovery.as_ref().unwrap().repair,
13007 crate::recovery::Repair::Done { .. }
13008 ));
13009 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
13010 }
13011
13012 #[tokio::test]
13013 #[cfg(unix)]
13014 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
13015 use std::os::fd::AsRawFd as _;
13016 for graft in [false, true] {
13017 let tmp = tempfile::tempdir().unwrap();
13018 let mut runner = registered_runner(tmp.path()).await;
13019 runner.state.config.graph.timeout_resource_wait = 1;
13020 let winner = runner.state.candidates[0].clone();
13021 let marker = tmp.path().join("fixer-started");
13022 let mut fixer = spec("fixer");
13023 fixer.command = vec![
13024 "sh".into(),
13025 "-c".into(),
13026 format!("echo started >> '{}'; echo done", marker.display()),
13027 ];
13028 runner.roles.fixer = Some(vec![fixer]);
13029 if graft {
13030 let mut loser = winner.clone();
13031 loser.label = 'B';
13032 loser.index = 1;
13033 runner.state.candidates.push(loser);
13034 runner.state.config.graph.graft_runner_up = true;
13035 runner.state.tally = Some(Tally {
13036 first_choice: BTreeMap::new(),
13037 borda: BTreeMap::new(),
13038 winner: winner.label,
13039 rankings: 1,
13040 unanimous_initial: true,
13041 deliberated: false,
13042 changed_votes: 0,
13043 unanimous_final: true,
13044 tie_break: None,
13045 judges: 1,
13046 present: 1,
13047 quorum: 1,
13048 met_quorum: true,
13049 uncontested: None,
13050 });
13051 runner.state.judgements.push(crate::run::Judgement {
13052 judge: 1,
13053 seat: "judge-1".into(),
13054 agent: "judge".into(),
13055 ranking: vec![winner.label, 'B'],
13056 reasons: BTreeMap::new(),
13057 confidence: None,
13058 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13059 order: vec![0, 1],
13060 failed: None,
13061 duration_ms: 0,
13062 });
13063 } else {
13064 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13065 from_run: "earlier".into(),
13066 generation: 1,
13067 mode: crate::recovery::Mode::VerifyRepair,
13068 reasons: Vec::new(),
13069 reviewed_head: None,
13070 rounds: Vec::new(),
13071 open_findings: Vec::new(),
13072 evidence: None,
13073 criteria_inherited: true,
13074 repair: crate::recovery::Repair::NotStarted,
13075 });
13076 }
13077 let reader = crate::repo_lock::SeatLease::for_checkout(
13078 &winner.worktree,
13079 false,
13080 crate::config::test_coordination().as_ref(),
13081 )
13082 .await
13083 .unwrap();
13084 let pause = runner.pause.clone();
13085 let (result, ()) = tokio::join!(
13086 async {
13087 if graft {
13088 runner.graft().await
13089 } else {
13090 runner.recovery_repair(&winner).await
13091 }
13092 },
13093 async {
13094 tokio::time::sleep(Duration::from_millis(100)).await;
13095 pause.park();
13096 }
13097 );
13098 result.unwrap();
13099 assert!(!marker.exists(), "a blocked seat must not spawn");
13100 runner.state = RunState::load(&runner.state.id).unwrap();
13102 if graft {
13103 assert_eq!(
13104 runner.state.graft.as_ref().unwrap().outcome,
13105 GraftOutcome::WaitingForRestore
13106 );
13107 } else {
13108 assert!(matches!(
13109 runner.state.recovery.as_ref().unwrap().repair,
13110 crate::recovery::Repair::WaitingForRestore { .. }
13111 ));
13112 }
13113 drop(reader);
13114 runner.pause = Pause::new();
13115 runner.state.resource_block = None;
13116 let external =
13118 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13119 assert_eq!(
13120 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13121 0
13122 );
13123 let leftover = winner.worktree.join("leftover");
13124 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13125 let result = if graft {
13126 runner.graft().await
13127 } else {
13128 runner.recovery_repair(&winner).await
13129 };
13130 assert!(
13131 result
13132 .unwrap_err()
13133 .is::<crate::repo_lock::ResourceBlocked>()
13134 );
13135 assert!(leftover.exists());
13136 assert!(!marker.exists());
13137 drop(external);
13138 runner.state.resource_block = None;
13139 let result = if graft {
13140 runner.graft().await
13141 } else {
13142 runner.recovery_repair(&winner).await
13143 };
13144 result.unwrap();
13145 assert!(!leftover.exists(), "restore must precede retry");
13146 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13147 if graft {
13148 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13149 } else {
13150 assert!(matches!(
13151 runner.state.recovery.as_ref().unwrap().repair,
13152 crate::recovery::Repair::Done { .. }
13153 ));
13154 }
13155 }
13156 }
13157
13158 #[tokio::test]
13159 #[cfg(unix)]
13160 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13161 use std::os::fd::AsRawFd as _;
13162 let tmp = tempfile::tempdir().unwrap();
13163 let mut runner = registered_runner(tmp.path()).await;
13164 runner.state.config.graph.timeout_resource_wait = 1;
13165 let winner = runner.state.candidates[0].clone();
13166 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13167 runner.state.tally = Some(Tally {
13168 first_choice: BTreeMap::new(),
13169 borda: BTreeMap::new(),
13170 winner: winner.label,
13171 rankings: 0,
13172 unanimous_initial: false,
13173 deliberated: false,
13174 changed_votes: 0,
13175 unanimous_final: false,
13176 tie_break: None,
13177 judges: 0,
13178 present: 0,
13179 quorum: 0,
13180 met_quorum: true,
13181 uncontested: Some("fixture".into()),
13182 });
13183 runner.state.graft = Some(GraftRecord {
13184 pre: pre.clone(),
13185 post: None,
13186 lines: None,
13187 limit: None,
13188 agent: None,
13189 outcome: GraftOutcome::Running,
13190 requested: Vec::new(),
13191 adoption: None,
13192 adoption_error: None,
13193 });
13194 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13195 git::commit_all(&winner.worktree, "unfinished graft")
13196 .await
13197 .unwrap();
13198 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13199 let leftover = winner.worktree.join("leftover");
13200 std::fs::write(&leftover, "untrusted edits").unwrap();
13201 let path = runner.state.repo.join(".git/updater.lock");
13202 let external = std::fs::File::create(&path).unwrap();
13203 assert_eq!(
13204 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13205 0
13206 );
13207 assert!(
13208 runner
13209 .graft()
13210 .await
13211 .unwrap_err()
13212 .is::<crate::repo_lock::ResourceBlocked>()
13213 );
13214 assert!(matches!(
13215 runner.state.graft.as_ref().unwrap().outcome,
13216 GraftOutcome::Running
13217 ));
13218 assert_eq!(
13219 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13220 unfinished
13221 );
13222 assert!(leftover.exists());
13223 drop(external);
13224 std::fs::remove_file(&path).unwrap();
13225 std::fs::create_dir(&path).unwrap();
13226 assert!(runner.graft().await.is_err());
13227 assert!(matches!(
13228 runner.state.graft.as_ref().unwrap().outcome,
13229 GraftOutcome::Running
13230 ));
13231 assert_eq!(
13232 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13233 unfinished
13234 );
13235 assert!(leftover.exists());
13236 std::fs::remove_dir(&path).unwrap();
13237 runner.graft().await.unwrap();
13238 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13239 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13240 assert!(!leftover.exists());
13241 }
13242
13243 #[tokio::test]
13244 #[cfg(unix)]
13245 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13246 use std::os::fd::AsRawFd as _;
13247 let tmp = tempfile::tempdir().unwrap();
13248 let mut runner = registered_runner(tmp.path()).await;
13249 runner.state.prep_complete = Some(false);
13250 runner.state.config.graph.timeout_resource_wait = 1;
13251 runner.state.save().unwrap();
13252 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13253 assert_eq!(
13254 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13255 0
13256 );
13257 runner.execute().await.unwrap();
13258 let disk = RunState::load(&runner.state.id).unwrap();
13259 assert!(disk.driver_exited);
13260 assert_eq!(disk.driver_pid, None);
13261 assert_eq!(runner.state.status, RunStatus::Blocked);
13262 assert_eq!(
13263 runner.state.resource_block.as_ref().unwrap().kind,
13264 "repository coordination"
13265 );
13266 assert!(runner.state.prep_failure.is_none());
13267 assert!(runner.state.seats.is_empty());
13268 runner.pause.park();
13269 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13272 .await
13273 .unwrap()
13274 .unwrap();
13275 assert!(runner.state.parked);
13276 assert!(runner.state.prep_failure.is_none());
13277 let disk = RunState::load(&runner.state.id).unwrap();
13278 assert!(disk.driver_exited);
13279 assert_eq!(disk.driver_pid, None);
13280 }
13281
13282 #[tokio::test]
13283 #[cfg(unix)]
13284 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13285 use std::os::fd::AsRawFd as _;
13286 let tmp = tempfile::tempdir().unwrap();
13287 let mut runner = registered_runner(tmp.path()).await;
13288 runner.state.prep_complete = Some(false);
13289 runner.state.config.graph.timeout_resource_wait = 30;
13290 runner.state.save().unwrap();
13291 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13292 assert_eq!(
13293 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13294 0
13295 );
13296 assert!(
13297 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13298 .await
13299 .is_err()
13300 );
13301 let disk = RunState::load(&runner.state.id).unwrap();
13302 assert!(disk.driver_exited);
13303 assert_eq!(disk.driver_pid, None);
13304 assert!(disk.status.resumable());
13305 assert!(runner.state.driver_exited);
13306 assert_eq!(runner.state.driver_pid, None);
13307 runner.state.save().unwrap();
13308 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13309 }
13310
13311 #[tokio::test]
13312 #[cfg(unix)]
13313 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13314 let tmp = tempfile::tempdir().unwrap();
13315 let mut runner = registered_runner(tmp.path()).await;
13316 let winner = runner.state.candidates[0].clone();
13317 let cache = tmp.path().join("cache");
13318 let started = tmp.path().join("pre-gate-started");
13319 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13320 runner.state.config.verify.gate =
13321 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13322 let home = crate::run::home();
13323 let owner =
13324 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13325 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13326 crate::cache::AcquireOutcome::Acquired(g) => g,
13327 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13328 };
13329 let mut gate = Box::pin(runner.run_gate(&winner));
13330 assert!(
13331 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13332 .await
13333 .is_err()
13334 );
13335 assert!(!started.exists(), "pre-gate must wait for the cache too");
13336 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13340 let editor = crate::repo_lock::SeatLease::checkout_with(
13341 &winner.worktree,
13342 true,
13343 crate::config::test_coordination().as_ref(),
13344 &wait,
13345 )
13346 .await
13347 .unwrap();
13348 drop(editor);
13349 guard.release();
13350 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13351 .await
13352 .unwrap()
13353 .unwrap();
13354 drop(gate);
13355 assert!(outcomes.iter().all(CommandOutcome::ok));
13356 assert!(started.exists());
13357 }
13358
13359 #[tokio::test]
13360 #[cfg(unix)]
13361 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13362 use std::os::fd::AsRawFd as _;
13363
13364 ask_test_home();
13365 let tmp = tempfile::tempdir().unwrap();
13366 let repo = tmp.path().join("repo");
13367 std::fs::create_dir_all(&repo).unwrap();
13368 init_repo(&repo);
13369 let common = repo.join(".git");
13370 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13371 let mut runner = runner_at(RunStatus::Prep);
13372 runner.state.repo = repo.clone();
13373 runner.state.config.coordination = crate::config::test_coordination();
13374 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13375 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13376 runner.state.config.graph.advise = false;
13377 runner.roles.implementers = vec![spec("implement")];
13378 runner.roles.judges = vec![spec("judge")];
13379 runner.prep().await.unwrap();
13380 let winner = runner.state.candidates[0].clone();
13381 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13384 let completed = tmp.path().join("seat-completed");
13385 runner.roles.implementers[0].command = vec![
13386 "sh".into(),
13387 "-c".into(),
13388 format!("touch '{}'; echo completed", completed.display()),
13389 ];
13390 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13391 assert_eq!(
13392 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13393 0
13394 );
13395 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13396 let mut implement = Box::pin(runner.implement());
13397 tokio::select! {
13398 result = &mut implement => panic!("rescue must wait: {result:?}"),
13399 result = tokio::time::timeout(Duration::from_secs(10), async {
13400 while !completed.exists() {
13401 tokio::time::sleep(Duration::from_millis(20)).await;
13402 }
13403 }) => result.unwrap(),
13404 }
13405 assert!(
13406 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13407 .await
13408 .is_err()
13409 );
13410 assert_eq!(
13411 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13412 before
13413 );
13414 assert_eq!(
13415 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13416 0
13417 );
13418 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13419 .await
13420 .unwrap()
13421 .unwrap();
13422 drop(implement);
13423 assert_ne!(
13424 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13425 before
13426 );
13427 assert!(git::is_clean(&winner.worktree).await.unwrap());
13428
13429 let started = tmp.path().join("pre-gate-started");
13430 let proceed = tmp.path().join("pre-gate-proceed");
13431 let checked = tmp.path().join("gate-ran");
13432 runner.state.config.verify.pre_gate = vec![format!(
13433 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13434 started.display(),
13435 proceed.display()
13436 )];
13437 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13438 assert_eq!(
13439 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13440 0
13441 );
13442 let mut gate = Box::pin(runner.run_gate(&winner));
13443 assert!(
13444 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13445 .await
13446 .is_err()
13447 );
13448 assert!(!started.exists());
13449 assert!(!checked.exists());
13450 assert!(git::is_clean(&winner.worktree).await.unwrap());
13451 assert_eq!(
13452 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13453 0
13454 );
13455 tokio::select! {
13456 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13457 result = tokio::time::timeout(Duration::from_secs(10), async {
13458 while !started.exists() {
13459 tokio::time::sleep(Duration::from_millis(20)).await;
13460 }
13461 }) => result.unwrap(),
13462 }
13463 assert_eq!(
13464 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13465 -1,
13466 "mechanical edits must hold the real lock"
13467 );
13468 std::fs::write(proceed, "ready").unwrap();
13469 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13470 .await
13471 .unwrap()
13472 .unwrap();
13473 drop(gate);
13474 assert!(outcomes.iter().all(CommandOutcome::ok));
13475 assert!(checked.exists());
13476 assert!(runner.state.pre_gate_commit.is_some());
13477 assert!(git::is_clean(&winner.worktree).await.unwrap());
13478 assert_eq!(
13479 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13480 0,
13481 "gate completion must release coordination"
13482 );
13483 drop(outsider);
13484 std::fs::remove_file(common.join("updater.lock")).unwrap();
13485 std::fs::create_dir(common.join("updater.lock")).unwrap();
13486 std::fs::remove_file(&started).unwrap();
13487 std::fs::remove_file(&checked).unwrap();
13488 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13489 let error = runner.run_gate(&winner).await.unwrap_err();
13490 assert!(
13491 error
13492 .to_string()
13493 .contains("acquire mandatory repository lock")
13494 );
13495 assert!(
13496 !started.exists(),
13497 "failed acquisition must prevent pre-gate"
13498 );
13499 assert!(!checked.exists(), "failed acquisition must prevent gate");
13500 assert!(git::is_clean(&winner.worktree).await.unwrap());
13501 assert_eq!(
13502 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13503 head
13504 );
13505 }
13506
13507 #[tokio::test]
13508 #[cfg(unix)]
13509 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13510 use std::os::fd::AsRawFd as _;
13511 use std::time::Duration;
13512
13513 ask_test_home();
13514 let tmp = tempfile::tempdir().unwrap();
13515 let repo = tmp.path().join("repo");
13516 std::fs::create_dir_all(&repo).unwrap();
13517 init_repo(&repo);
13518 let common = repo.join(".git");
13519 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13520 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13521 assert_eq!(
13522 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13523 0
13524 );
13525 let mut runner = runner_at(RunStatus::Prep);
13526 runner.state.repo = repo.clone();
13527 runner.state.config.coordination = crate::config::test_coordination();
13528 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13529 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13530 runner.state.config.graph.advise = true;
13531 runner.state.config.graph.advisors = 1;
13532 runner.roles.implementers = vec![spec("implement")];
13533 runner.roles.judges = vec![spec("judge")];
13534 let root = runner.state.worktree_root();
13535 let started = tmp.path().join("setup-started");
13536 let proceed = tmp.path().join("setup-proceed");
13537 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13538 run: Some(format!(
13539 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13540 started.display(),
13541 proceed.display()
13542 )),
13543 ..Default::default()
13544 }];
13545 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13546 let config = git::git(&repo, &["config", "--local", "--list"])
13547 .await
13548 .unwrap();
13549 let mut prep = Box::pin(runner.prep());
13550 assert!(
13551 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13552 .await
13553 .is_err()
13554 );
13555 assert!(
13556 !root.exists(),
13557 "contention must prevent even checkout creation"
13558 );
13559 assert!(!started.exists(), "contention must prevent setup");
13560 assert_eq!(
13561 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13562 branches
13563 );
13564 assert_eq!(
13565 git::git(&repo, &["config", "--local", "--list"])
13566 .await
13567 .unwrap(),
13568 config
13569 );
13570 drop(lock);
13571 tokio::select! {
13572 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13573 result = tokio::time::timeout(Duration::from_secs(10), async {
13574 while !started.exists() {
13575 tokio::time::sleep(Duration::from_millis(20)).await;
13576 }
13577 }) => result.unwrap(),
13578 }
13579 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13580 assert_eq!(
13581 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13582 -1
13583 );
13584 assert!(
13585 tokio::time::timeout(
13586 Duration::from_millis(100),
13587 crate::repo_lock::SeatLease::for_checkout(
13588 &repo,
13589 true,
13590 crate::config::test_coordination().as_ref()
13591 )
13592 )
13593 .await
13594 .is_err(),
13595 "preparation must also exclude local editors"
13596 );
13597 std::fs::write(proceed, "ready").unwrap();
13598 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13599 .await
13600 .unwrap()
13601 .unwrap();
13602 drop(prep);
13603 assert_eq!(runner.state.prep_complete, Some(true));
13604 assert!(root.join("judge-1").exists());
13605 assert!(root.join("advisor-1").exists());
13606 assert_eq!(
13607 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13608 0
13609 );
13610 }
13611
13612 #[tokio::test]
13613 #[cfg(unix)]
13614 async fn preparation_lock_failure_prevents_repository_changes() {
13615 ask_test_home();
13616 let tmp = tempfile::tempdir().unwrap();
13617 let repo = tmp.path().join("repo");
13618 std::fs::create_dir_all(&repo).unwrap();
13619 init_repo(&repo);
13620 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13621 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13624 let mut runner = runner_at(RunStatus::Prep);
13625 runner.state.repo = repo.clone();
13626 runner.state.config.coordination = crate::config::test_coordination();
13627 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13628 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13629 runner.roles.implementers = vec![spec("implement")];
13630 let error = runner.prep().await.unwrap_err();
13631 assert!(
13632 error
13633 .to_string()
13634 .contains("acquire mandatory repository lock")
13635 );
13636 assert!(!runner.state.worktree_root().exists());
13637 assert!(runner.state.candidates.is_empty());
13638 assert!(!repo.join(".git/config.worktree").exists());
13639 assert_eq!(
13640 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13641 "* main"
13642 );
13643 }
13644
13645 #[tokio::test]
13646 async fn preparation_records_creation_intent_before_git_can_fail() {
13647 ask_test_home();
13648 let tmp = tempfile::tempdir().unwrap();
13649 let repo = tmp.path().join("repo");
13650 std::fs::create_dir_all(&repo).unwrap();
13651 init_repo(&repo);
13652 let mut runner = runner_at(RunStatus::Prep);
13653 runner.state.repo = repo.clone();
13654 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13655 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13656 runner.state.config.blind.commit_msg_hook = false;
13657 runner.roles.implementers = vec![spec("one")];
13658 let label = blind::assign_labels(1, runner.state.seed)[0];
13659 let branch = runner.state.branch_for(label);
13660 git::git(&repo, &["branch", &branch]).await.unwrap();
13663 assert!(runner.prep().await.is_err());
13664 let saved = RunState::load(&runner.state.id).unwrap();
13665 assert!(saved.candidates.is_empty());
13666 let event = saved
13667 .events
13668 .iter()
13669 .find(|e| e.node == "prep-intent")
13670 .unwrap();
13671 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13672 assert_eq!(intent["branch"], branch);
13673 assert_eq!(intent["start"], runner.state.base_commit);
13674 assert_eq!(
13675 intent["worktree"],
13676 runner
13677 .state
13678 .worktree_root()
13679 .join(format!("cand-{label}"))
13680 .to_string_lossy()
13681 .as_ref()
13682 );
13683 assert_eq!(intent["picks"], serde_json::json!([]));
13684 }
13685
13686 #[tokio::test]
13687 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13688 ask_test_home();
13689 let tmp = tempfile::tempdir().unwrap();
13690 let repo = tmp.path().join("repo");
13691 std::fs::create_dir_all(&repo).unwrap();
13692 init_repo(&repo);
13693 let mut runner = runner_at(RunStatus::Prep);
13694 runner.state.repo = repo.clone();
13695 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13696 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13697 runner.state.sources.pick_commits.push(refs::Input {
13698 token: "missing".into(),
13699 sha: "0".repeat(40),
13700 provenance: refs::Provenance::ExplicitCli,
13701 });
13702 assert!(runner.prep().await.is_err());
13703 assert!(runner.state.candidates.is_empty());
13704 assert!(!runner.state.worktree_root().exists());
13705 }
13706
13707 #[tokio::test]
13708 async fn prose_and_legacy_inference_never_import_into_candidates() {
13709 ask_test_home();
13710 let tmp = tempfile::tempdir().unwrap();
13711 let repo = tmp.path().join("repo");
13712 std::fs::create_dir_all(&repo).unwrap();
13713 init_repo(&repo);
13714 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13715 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13716 .await
13717 .unwrap();
13718 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13719 git::git(&repo, &["add", "."]).await.unwrap();
13720 git::git(&repo, &["commit", "-m", "historical implementation"])
13721 .await
13722 .unwrap();
13723 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13724 git::git(&repo, &["checkout", "main"]).await.unwrap();
13725 for instruction in [
13726 format!("Example: magi/history/A at {source}"),
13727 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13728 format!("Never import magi/history/A or {source}"),
13729 format!("Conversation answer: I mentioned {source} only as background"),
13730 ] {
13731 let mut runner = runner_at(RunStatus::Prep);
13732 runner.state.repo = repo.clone();
13733 runner.state.base_commit = base.clone();
13734 runner.state.instruction = instruction.clone();
13735 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13736 runner.state.config.blind.commit_msg_hook = false;
13737 runner.state.seeds = vec![refs::Seed {
13738 token: "magi/history/A".into(),
13739 sha: source.clone(),
13740 kind: refs::SeedKind::Unmerged,
13741 branch: true,
13742 detail: String::new(),
13743 }];
13744 runner.roles.implementers = vec![spec("one")];
13745 assert_eq!(seeded_instruction(&runner.state), instruction);
13746 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13748 legacy.as_object_mut().unwrap().remove("sources");
13749 runner.state = serde_json::from_value(legacy).unwrap();
13750 assert_eq!(runner.state.sources, refs::Sources::default());
13751 runner.state.save().unwrap();
13752 runner.state = RunState::load(&runner.state.id).unwrap();
13753 runner.prep().await.unwrap();
13754 for candidate in &runner.state.candidates {
13755 assert_eq!(
13756 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13757 base
13758 );
13759 assert!(!candidate.worktree.join("unwanted").exists());
13760 }
13761 }
13762 }
13763
13764 #[tokio::test]
13765 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13766 ask_test_home();
13767 let tmp = tempfile::tempdir().unwrap();
13768 let repo = tmp.path().join("repo");
13769 std::fs::create_dir_all(&repo).unwrap();
13770 init_repo(&repo);
13771 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13772 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13773 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13774 git::git(&repo, &["add", "."]).await.unwrap();
13775 git::git(&repo, &["commit", "-m", "required seed"])
13776 .await
13777 .unwrap();
13778 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13779 git::git(&repo, &["checkout", "main"]).await.unwrap();
13780 let mut runner = runner_at(RunStatus::Prep);
13781 runner.state.repo = repo.clone();
13782 runner.state.base_commit = base.clone();
13783 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13784 runner.state.config.graph.advise = false;
13785 runner.state.config.blind.commit_msg_hook = false;
13786 runner.state.sources =
13787 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13788 .await
13789 .unwrap();
13790 runner.roles.implementers = vec![spec("one"), spec("two")];
13791 runner.roles.judges = vec![spec("judge")];
13792 let label = blind::assign_labels(2, runner.state.seed)[0];
13793 let branch = runner.state.branch_for(label);
13794 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13795 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13796 .await
13797 .unwrap();
13798 runner.state.candidates.push(Candidate {
13799 index: 0,
13800 label,
13801 agent: "one".into(),
13802 branch,
13803 worktree,
13804 summary: String::new(),
13805 stat: String::new(),
13806 files: 0,
13807 commits: 0,
13808 empty: false,
13809 failed: None,
13810 verified_noop: None,
13811 duration_ms: 0,
13812 folded: false,
13813 });
13814 runner.state.save().unwrap();
13815 runner.state = RunState::load(&runner.state.id).unwrap();
13817 assert_eq!(runner.state.prep_complete, Some(false));
13818 runner.prep().await.unwrap();
13819 assert_eq!(runner.state.prep_complete, Some(true));
13820 assert_eq!(runner.state.candidates.len(), 2);
13821 for candidate in &runner.state.candidates {
13822 assert_eq!(
13823 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13824 git::tree_of(&repo, &seed).await.unwrap()
13825 );
13826 assert_eq!(
13827 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13828 "required commit"
13829 );
13830 }
13831 assert!(runner.state.worktree_root().join("judge-1").exists());
13832 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13833 runner.prep().await.unwrap();
13834 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13835
13836 runner.state.prep_complete = Some(false);
13837 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13838 run: Some(format!(
13839 "test -f '{}'",
13840 tmp.path().join("dependency-ready").display()
13841 )),
13842 ..Default::default()
13843 }];
13844 assert!(runner.prep().await.is_err());
13845 assert!(runner.state.prep_failure.is_some());
13846 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13847 runner.state = RunState::load(&runner.state.id).unwrap();
13848 assert!(
13849 runner.prep().await.is_err(),
13850 "correcting the cause alone must not auto-retry"
13851 );
13852 let failed_at = runner.state.events.last().unwrap().at;
13853 assert!(
13854 runner.retry_preparation(failed_at).is_err(),
13855 "old approvals cannot unlock the hold"
13856 );
13857 let mut task = crate::queue::Task::new(
13858 "retry".into(),
13859 "retry setup".into(),
13860 repo.clone(),
13861 crate::queue::Source::Human,
13862 );
13863 task.start(runner.state.id.clone());
13864 task.hold_machine(Some("setup failed".into()));
13865 task.release_operator("cli", None);
13866 runner
13867 .retry_preparation(task.resumption.as_ref().unwrap().at)
13868 .unwrap();
13869 runner.prep().await.unwrap();
13870 assert!(runner.state.prep_failure.is_none());
13871 assert_eq!(runner.state.prep_complete, Some(true));
13872 assert_eq!(runner.state.candidates.len(), 2);
13873 assert!(runner.state.worktree_root().join("judge-1").exists());
13874 for candidate in &runner.state.candidates {
13875 assert_eq!(
13876 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13877 git::tree_of(&repo, &seed).await.unwrap()
13878 );
13879 }
13880 }
13881
13882 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13883 let mut heads = Vec::new();
13884 for candidate in candidates {
13885 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13886 }
13887 heads
13888 }
13889
13890 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13891 ask_test_home();
13892 let tmp = tempfile::tempdir().unwrap();
13893 let repo = tmp.path().join("repo");
13894 std::fs::create_dir(&repo).unwrap();
13895 init_repo(&repo);
13896 let mut runner = runner_at(RunStatus::Reviewing);
13897 runner.state.repo = repo.clone();
13898 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13899 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13900 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13901 let branch = format!("candidate-{label}");
13902 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13903 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13904 .await
13905 .unwrap();
13906 if label == 'B' {
13907 std::fs::write(worktree.join("change"), "winner").unwrap();
13908 git::rescue_commit(&worktree, "candidate change")
13909 .await
13910 .unwrap();
13911 }
13912 runner.state.candidates.push(Candidate {
13913 index,
13914 label,
13915 agent: "alpha".into(),
13916 branch,
13917 worktree,
13918 summary: "completed reply".into(),
13919 stat: String::new(),
13920 files: usize::from(label == 'B'),
13921 commits: usize::from(label == 'B'),
13922 empty: label == 'A',
13923 failed: None,
13924 verified_noop: None,
13925 duration_ms: 1,
13926 folded: false,
13927 });
13928 }
13929 runner.state.judge_skipped = true;
13930 runner.state.tally = Some(Tally {
13931 first_choice: BTreeMap::new(),
13932 borda: BTreeMap::new(),
13933 winner: 'B',
13934 rankings: 0,
13935 unanimous_initial: false,
13936 deliberated: false,
13937 changed_votes: 0,
13938 unanimous_final: false,
13939 tie_break: None,
13940 judges: 0,
13941 present: 0,
13942 quorum: 0,
13943 met_quorum: true,
13944 uncontested: Some("only B produced a change".into()),
13945 });
13946 runner.state.save().unwrap();
13947 (tmp, runner)
13948 }
13949
13950 #[tokio::test]
13951 #[cfg(unix)]
13952 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13953 for (solo, failed_checkpoint) in
13954 [(true, false), (false, false), (true, true), (false, true)]
13955 {
13956 let (_tmp, mut runner) = preservation_resume_fixture().await;
13957 if solo {
13958 runner.state.candidates.retain(|c| c.label == 'A');
13959 }
13960 runner.state.candidates[0].empty = false;
13961 runner.state.candidates[0].summary.clear();
13962 runner.state.tally = None;
13963 runner.state.judge_skipped = false;
13964 runner.state.status = RunStatus::Implementing;
13965 runner.state.config.disk.min_free_bytes = u64::MAX;
13966 let mut implementer = spec("alpha");
13967 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13968 runner.roles.implementers = vec![implementer];
13969 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13970 if failed_checkpoint {
13971 std::fs::create_dir_all(&patch).unwrap();
13972 }
13973 runner.state.save().unwrap();
13974 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13975 runner.state = RunState::load(&runner.state.id).unwrap();
13976 assert_eq!(runner.state.status, RunStatus::Blocked);
13977 assert!(runner.state.interrupted_implementations.contains(&'A'));
13978 assert_eq!(runner.state.candidates[0].commits, 0);
13979 assert!(!runner.state.candidates[0].empty);
13980 assert!(runner.state.candidates[0].failed.is_none());
13981 if failed_checkpoint {
13982 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13983 std::fs::remove_dir(patch).unwrap();
13984 }
13985 runner.state.resource_block = None;
13986 runner.state.config.disk.min_free_bytes = 0;
13987 runner.roles.implementers[0].command = vec![
13988 "sh".into(),
13989 "-c".into(),
13990 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13991 ];
13992 runner.implement().await.unwrap();
13993 assert!(runner.state.interrupted_implementations.is_empty());
13994 assert!(runner.state.candidate_preservation.is_empty());
13995 assert_eq!(runner.state.candidates[0].commits, 1);
13996 assert!(!runner.state.candidates[0].empty);
13997 assert!(
13998 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13999 .unwrap()
14000 .contains("retried empty interrupted seat")
14001 );
14002 if !solo {
14003 assert_eq!(runner.state.candidates[1].commits, 1);
14004 }
14005 }
14006 }
14007
14008 #[tokio::test]
14009 #[cfg(unix)]
14010 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
14011 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
14012 let (_tmp, mut runner) = preservation_resume_fixture().await;
14013 runner.state.candidates.retain(|c| c.label == 'A');
14014 runner.state.candidates[0].empty = false;
14015 runner.state.candidates[0].summary.clear();
14016 runner.state.tally = None;
14017 runner.state.judge_skipped = false;
14018 runner.state.status = RunStatus::Implementing;
14019 runner.state.config.disk.min_free_bytes = u64::MAX;
14022 let wt = runner.state.candidates[0].worktree.clone();
14023 std::fs::write(
14024 wt.join("interrupted-source"),
14025 "retain implementation work\n",
14026 )
14027 .unwrap();
14028 if committed {
14029 git::rescue_commit(&wt, "implementation checkpoint")
14030 .await
14031 .unwrap();
14032 }
14033 let lock = git::git(
14034 &wt,
14035 &[
14036 "rev-parse",
14037 "--path-format=absolute",
14038 "--git-path",
14039 "index.lock",
14040 ],
14041 )
14042 .await
14043 .unwrap();
14044 if block_rescue {
14045 std::fs::write(&lock, "fixture-owned lock").unwrap();
14046 }
14047 let mut implementer = spec("alpha");
14048 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
14049 runner.roles.implementers = vec![implementer];
14050 runner.state.save().unwrap();
14051 let result = runner.implement().await;
14052 assert_eq!(result.is_err(), block_rescue);
14053 assert_eq!(
14054 runner.state.resource_block.as_ref().unwrap().kind,
14055 "disk pressure"
14056 );
14057 assert!(
14058 runner
14059 .state
14060 .events
14061 .iter()
14062 .any(|e| e.message.contains("agent execution interrupted"))
14063 );
14064 assert!(
14065 runner
14066 .state
14067 .dir()
14068 .join("artifacts/candidate-A-recovery/staged.patch")
14069 .exists()
14070 );
14071 if block_rescue {
14072 let saved = RunState::load(&runner.state.id).unwrap();
14073 assert!(saved.candidate_preservation.contains_key(&'A'));
14074 assert!(!saved.candidates[0].empty);
14075 assert_eq!(
14076 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14077 "retain implementation work\n"
14078 );
14079 std::fs::remove_file(lock).unwrap();
14080 runner.state = saved;
14081 } else {
14082 assert_eq!(runner.state.status, RunStatus::Blocked);
14083 assert_eq!(runner.state.candidates[0].commits, 1);
14084 }
14085 runner.roles.implementers.clear();
14088 runner.state.config.disk.min_free_bytes = 0;
14089 runner.state.resource_block = None;
14090 runner.implement().await.unwrap();
14091 assert!(runner.state.candidate_preservation.is_empty());
14092 assert_eq!(runner.state.candidates[0].commits, 1);
14093 assert!(!runner.state.candidates[0].empty);
14094 assert!(git::is_clean(&wt).await.unwrap());
14095 assert!(
14096 crate::run::read_artifact(&runner.state, "cand-A.patch")
14097 .unwrap()
14098 .contains("retain implementation work")
14099 );
14100 }
14101 }
14102
14103 #[tokio::test]
14104 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14105 let (_tmp, mut runner) = preservation_resume_fixture().await;
14106 runner.implement().await.unwrap();
14108 assert!(runner.state.candidate_preservation.is_empty());
14109 assert!(
14110 !runner
14111 .state
14112 .dir()
14113 .join("artifacts/candidate-A-recovery")
14114 .exists()
14115 );
14116 runner.fold_losers().await.unwrap();
14117 assert!(!runner.state.candidates[0].worktree.exists());
14118 for schema in [14, crate::run::SCHEMA] {
14119 runner.state.schema = schema;
14120 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14121 runner.state.save().unwrap();
14122 runner.state = RunState::load(&runner.state.id).unwrap();
14123 runner.implement().await.unwrap();
14124 assert!(runner.state.candidate_preservation.is_empty());
14125 assert!(runner.state.candidates[0].folded);
14126 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14127 }
14128 }
14129
14130 #[tokio::test]
14131 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14132 let (_tmp, mut runner) = preservation_resume_fixture().await;
14133 let wt = runner.state.candidates[0].worktree.clone();
14134 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14135 runner.state.schema = 14;
14136 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14137 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14138 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14139 for _ in 0..2 {
14140 assert!(runner.implement().await.is_err());
14141 assert_eq!(runner.state.status, RunStatus::Blocked);
14142 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14143 assert!(
14144 runner.state.candidates[0].empty,
14145 "old verdict must remain unusable"
14146 );
14147 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14148 assert!(runner.state.judge_skipped);
14149 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14150 assert!(
14151 crate::run::read_artifact(&runner.state, "cand-A.patch")
14152 .unwrap()
14153 .contains("owned edits")
14154 );
14155 assert!(wt.exists());
14156 assert_eq!(
14157 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14158 .await
14159 .unwrap(),
14160 1
14161 );
14162 runner.state = RunState::load(&runner.state.id).unwrap();
14163 }
14164 }
14165
14166 #[tokio::test]
14167 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14168 let (_tmp, mut runner) = preservation_resume_fixture().await;
14169 let wt = runner.state.candidates[0].worktree.clone();
14170 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14171 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14172 let index_lock = PathBuf::from(
14173 git::git(
14174 &wt,
14175 &[
14176 "rev-parse",
14177 "--path-format=absolute",
14178 "--git-path",
14179 "index.lock",
14180 ],
14181 )
14182 .await
14183 .unwrap(),
14184 );
14185 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14186 runner.state.schema = 14;
14187 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14188 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14189 assert!(runner.implement().await.is_err());
14190 assert!(runner.after_implement().is_err());
14191 assert!(runner.fold_losers().await.is_err());
14192 assert_eq!(
14193 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14194 "failed seat edits"
14195 );
14196 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14197 runner.state = RunState::load(&runner.state.id).unwrap();
14198 std::fs::remove_file(index_lock).unwrap();
14199 assert!(runner.implement().await.is_err());
14201 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14202 assert!(runner.state.candidates[0].empty);
14203 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14204 assert!(
14205 crate::run::read_artifact(&runner.state, "cand-A.patch")
14206 .unwrap()
14207 .contains("failed seat edits")
14208 );
14209 assert!(wt.exists());
14210 }
14211
14212 #[tokio::test]
14213 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14214 let (_tmp, mut runner) = preservation_resume_fixture().await;
14215 let repo = runner.state.repo.clone();
14216 let common = git::git(
14217 &repo,
14218 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14219 )
14220 .await
14221 .unwrap();
14222 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14223 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14224 assert!(runner.fold_losers().await.is_err());
14225 assert!(!runner.state.candidates[0].worktree.exists());
14226 assert!(!runner.state.candidates[0].folded);
14227 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14228 runner.state = RunState::load(&runner.state.id).unwrap();
14229 std::fs::remove_file(branch_lock).unwrap();
14230 let judge = runner.state.worktree_root().join("judge-1");
14231 git::worktree_add_detached(&repo, &judge, "main")
14232 .await
14233 .unwrap();
14234 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14235 runner.roles.judges = vec![spec("judge")];
14236 assert!(runner.fold_losers().await.is_err());
14237 runner.state = RunState::load(&runner.state.id).unwrap();
14238 assert!(
14239 runner.state.candidates[0].folded,
14240 "candidate progress saved before judge failure"
14241 );
14242 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14243 assert!(judge.join("uncommitted").exists());
14244 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14245 runner.fold_losers().await.unwrap();
14246 assert!(!judge.exists());
14247 runner.fold_losers().await.unwrap();
14248 }
14249
14250 #[tokio::test]
14251 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14252 let (_tmp, mut runner) = preservation_resume_fixture().await;
14253 runner.state.candidates.retain(|c| c.label == 'B');
14254 runner.state.tally = None;
14255 runner.state.judge_skipped = false;
14256 runner.state.prep_complete = Some(true);
14257 runner.state.config.graph.advise = false;
14258 runner.state.config.graph.selected_implementer = Some("alpha".into());
14259 let wt = runner.state.candidates[0].worktree.clone();
14260 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14261 runner
14262 .state
14263 .candidate_preservation
14264 .insert('B', "awaiting persistence".into());
14265 let index_lock = git::git(
14266 &wt,
14267 &[
14268 "rev-parse",
14269 "--path-format=absolute",
14270 "--git-path",
14271 "index.lock",
14272 ],
14273 )
14274 .await
14275 .unwrap();
14276 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14277 runner.state.save().unwrap();
14278 assert!(runner.execute_graph().await.is_err());
14279 assert_eq!(runner.state.status, RunStatus::Blocked);
14280 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14281 std::fs::remove_file(index_lock).unwrap();
14282 for _ in 0..2 {
14285 runner.state = RunState::load(&runner.state.id).unwrap();
14286 runner.pause.park();
14287 runner.execute_graph().await.unwrap();
14288 assert_eq!(runner.state.status, RunStatus::Implementing);
14289 assert!(runner.state.candidate_preservation.is_empty());
14290 assert_eq!(runner.state.candidates[0].commits, 2);
14291 assert!(runner.state.tally.is_none());
14292 }
14293 runner.implement().await.unwrap();
14294 assert_eq!(runner.state.status, RunStatus::Judging);
14295 runner.state.status = RunStatus::Blocked;
14297 runner.state.candidates[0].failed = Some("implementation failed".into());
14298 runner.state.save().unwrap();
14299 let error = runner.execute_graph().await.unwrap_err();
14300 assert!(
14301 error
14302 .to_string()
14303 .contains("selected implementation is held")
14304 );
14305 }
14306
14307 #[tokio::test]
14308 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14309 let (_tmp, mut runner) = preservation_resume_fixture().await;
14310 let repo = runner.state.repo.clone();
14311 let candidate = runner.state.candidates[0].clone();
14312 let registered = git::worktree_holding(&repo, &candidate.branch)
14315 .await
14316 .unwrap()
14317 .unwrap();
14318 assert_eq!(
14319 registered.canonicalize().unwrap(),
14320 candidate.worktree.canonicalize().unwrap()
14321 );
14322 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14323 assert_eq!(
14324 git::worktree_holding(&repo, &candidate.branch)
14325 .await
14326 .unwrap(),
14327 Some(registered)
14328 );
14329 assert!(
14330 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14331 "the stale registration initially keeps the branch checked out"
14332 );
14333 runner.fold_losers().await.unwrap();
14334 assert!(runner.state.candidates[0].folded);
14335 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14336 assert_eq!(
14337 git::worktree_holding(&repo, &candidate.branch)
14338 .await
14339 .unwrap(),
14340 None
14341 );
14342 runner.state = RunState::load(&runner.state.id).unwrap();
14343 runner.fold_losers().await.unwrap();
14344 }
14345
14346 #[tokio::test]
14347 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14348 let (_tmp, mut runner) = preservation_resume_fixture().await;
14349 let repo = runner.state.repo.clone();
14350 let candidate = runner.state.candidates[0].clone();
14351 let winner = runner.state.candidates[1].clone();
14352 let registered = git::worktree_holding(&repo, &candidate.branch)
14353 .await
14354 .unwrap()
14355 .unwrap();
14356 assert_eq!(
14357 registered.canonicalize().unwrap(),
14358 candidate.worktree.canonicalize().unwrap()
14359 );
14360 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14361 assert_eq!(
14362 git::worktree_holding(&repo, &candidate.branch)
14363 .await
14364 .unwrap(),
14365 Some(registered)
14366 );
14367 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14368
14369 for _ in 0..2 {
14370 fold_run(&mut runner.state, false, &crate::run::home())
14371 .await
14372 .unwrap();
14373 runner.state = RunState::load(&runner.state.id).unwrap();
14374 assert!(runner.state.candidates[0].folded);
14375 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14376 assert_eq!(
14377 git::worktree_holding(&repo, &candidate.branch)
14378 .await
14379 .unwrap(),
14380 None
14381 );
14382 assert!(!runner.state.candidates[1].folded);
14383 assert!(winner.worktree.exists());
14384 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14385 }
14386 }
14387
14388 #[tokio::test]
14389 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14390 ask_test_home();
14391 let tmp = tempfile::tempdir().unwrap();
14392 let repo = tmp.path().join("repo");
14393 std::fs::create_dir(&repo).unwrap();
14394 init_repo(&repo);
14395 let mut runner = runner_at(RunStatus::Implementing);
14396 runner.state.repo = repo.clone();
14397 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14398 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14399 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14400 let branch = format!("candidate-{label}");
14401 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14402 git::worktree_add_branch(&repo, &wt, &branch, "main")
14403 .await
14404 .unwrap();
14405 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14406 git::git(&wt, &["add", "README.md"]).await.unwrap();
14407 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14408 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14409 runner.state.candidates.push(Candidate {
14410 index,
14411 label,
14412 agent: "alpha".to_owned(),
14413 branch,
14414 worktree: wt,
14415 summary: "completed implementation".to_owned(),
14416 stat: String::new(),
14417 files: 0,
14418 commits: 0,
14419 empty: false,
14420 failed: None,
14421 verified_noop: None,
14422 duration_ms: 42,
14423 folded: false,
14424 });
14425 runner
14426 .state
14427 .candidate_preservation
14428 .insert(label, "awaiting persistence".to_owned());
14429 }
14430 let wt = runner.state.candidates[0].worktree.clone();
14431 let index_lock = git::git(
14432 &wt,
14433 &[
14434 "rev-parse",
14435 "--path-format=absolute",
14436 "--git-path",
14437 "index.lock",
14438 ],
14439 )
14440 .await
14441 .unwrap();
14442 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14443 runner.state.save().unwrap();
14444 assert!(runner.persist_candidates().await.is_err());
14445 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14446 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14447 assert_eq!(
14448 runner.state.candidates[1].commits, 1,
14449 "other results are still collected"
14450 );
14451 assert!(
14452 !runner.state.candidates[0].empty,
14453 "failed rescue is not no change"
14454 );
14455 assert_eq!(
14456 runner.state.candidates[0].summary,
14457 "completed implementation"
14458 );
14459 assert!(runner.after_implement().is_err());
14460 let home = tmp.path().join("fold-home");
14461 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14462 assert!(wt.join("new-file").exists());
14463 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14464 assert!(recovery.join("staged.patch").exists());
14465 assert!(recovery.join("unstaged.patch").exists());
14466 assert_eq!(
14467 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14468 "untracked edits\n"
14469 );
14470 runner.state.begin_pass();
14471 runner.state.end_pass_parked(ParkCause::Upgrade {
14472 from: "1.0".into(),
14473 to: Some("1.1".into()),
14474 });
14475 runner.state.save().unwrap();
14476 runner.state = RunState::load(&runner.state.id).unwrap();
14477 assert!(matches!(
14478 runner.state.passes.last().unwrap().end,
14479 Some(crate::run::PassEnd::Parked {
14480 cause: ParkCause::Upgrade { .. }
14481 })
14482 ));
14483 std::fs::remove_file(index_lock).unwrap();
14484 runner.implement().await.unwrap();
14486 assert!(runner.state.candidate_preservation.is_empty());
14487 assert!(
14488 runner
14489 .state
14490 .candidates
14491 .iter()
14492 .all(|c| c.commits == 1 && !c.empty)
14493 );
14494 assert!(git::is_clean(&wt).await.unwrap());
14495 assert!(
14496 crate::run::read_artifact(&runner.state, "cand-A.patch")
14497 .unwrap()
14498 .contains("later tracked edits")
14499 );
14500 }
14501
14502 #[tokio::test]
14503 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14504 ask_test_home();
14505 let tmp = tempfile::tempdir().unwrap();
14506 let repo = tmp.path().join("repo");
14507 std::fs::create_dir(&repo).unwrap();
14508 init_repo(&repo);
14509 let wt = tmp.path().join("candidate");
14510 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14511 .await
14512 .unwrap();
14513 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14514 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14515 assert!(wt.join("uncommitted").exists());
14516 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14517 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14518 let mut runner = runner_at(RunStatus::Implementing);
14519 runner.state.repo = repo.clone();
14520 runner.state.base_commit = "missing-base".to_owned();
14521 runner.state.candidates.push(Candidate {
14522 index: 0,
14523 label: 'A',
14524 agent: "alpha".into(),
14525 branch: "candidate".into(),
14526 worktree: wt.clone(),
14527 summary: "completed".into(),
14528 stat: String::new(),
14529 files: 0,
14530 commits: 0,
14531 empty: false,
14532 failed: None,
14533 verified_noop: None,
14534 duration_ms: 1,
14535 folded: false,
14536 });
14537 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14538 assert!(
14539 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14540 .await
14541 .is_err()
14542 );
14543 assert!(!runner.state.candidates[0].folded);
14544 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14545 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14546 runner
14547 .state
14548 .candidate_preservation
14549 .insert('A', "pending".into());
14550 assert!(runner.persist_candidates().await.is_err());
14551 assert!(!runner.state.candidates[0].empty);
14552 assert!(!runner.state.candidates[0].folded);
14553 assert!(wt.exists());
14554 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14555 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14556 std::fs::create_dir_all(&patch_path).unwrap();
14557 assert!(
14558 runner.persist_candidates().await.is_err(),
14559 "artifact failure stays pending"
14560 );
14561 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14562 assert!(!runner.state.candidates[0].empty);
14563 std::fs::remove_dir(patch_path).unwrap();
14564 runner.persist_candidates().await.unwrap();
14565 assert!(
14566 runner.state.candidates[0].empty,
14567 "only successful inspection confirms no change"
14568 );
14569 }
14570
14571 fn init_repo(dir: &Path) {
14572 let run = |args: &[&str]| {
14573 let out = std::process::Command::new("git")
14574 .args(args)
14575 .current_dir(dir)
14576 .quiet()
14577 .output()
14578 .expect("spawn git");
14579 assert!(
14580 out.status.success(),
14581 "git {args:?} failed: {}",
14582 String::from_utf8_lossy(&out.stderr)
14583 );
14584 };
14585 run(&["init", "-b", "main"]);
14586 run(&["config", "user.name", "magi test"]);
14587 run(&["config", "user.email", "magi@example.com"]);
14588 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14589 run(&["add", "-A"]);
14590 run(&["commit", "-m", "init"]);
14591 }
14592
14593 fn ask_test_home() {
14601 crate::run::pin_test_home();
14602 }
14603
14604 fn runner_at(status: RunStatus) -> Runner {
14607 let mut state = RunState::new(
14608 PathBuf::from("/nonexistent/repo"),
14609 "main".to_owned(),
14610 "deadbeef".to_owned(),
14611 "task".to_owned(),
14612 Config::default(),
14613 );
14614 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14617 state.id = format!(
14618 "20000101-000000-{:04x}",
14619 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14620 );
14621 state.status = status;
14622 Runner {
14623 state,
14624 roles: ResolvedRoles {
14625 implementers: Vec::new(),
14626 judges: Vec::new(),
14627 reviewers: Vec::new(),
14628 fixer: None,
14629 conductor: conductor(),
14630 implementer_roster: Vec::new(),
14631 judge_roster: Vec::new(),
14632 reviewer_roster: Vec::new(),
14633 },
14634 sem: Arc::new(Semaphore::new(1)),
14635 pause: Pause::new(),
14636 interrupt: Pause::new(),
14637 }
14638 }
14639
14640 #[test]
14641 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14642 crate::run::pin_test_home();
14643 let mut runner = runner_at(RunStatus::Gating);
14644 runner.state.save().unwrap();
14645 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14648 assert!(runner.claim_driver().is_err());
14649 assert!(runner.state.driver_exited);
14650 assert_eq!(runner.state.driver_pid, None);
14651 let disk = RunState::load(&runner.state.id).unwrap();
14652 assert!(disk.driver_exited);
14653 assert_eq!(disk.driver_pid, None);
14654 assert_eq!(disk.status, RunStatus::Gating);
14655 }
14656
14657 #[tokio::test]
14658 async fn an_early_graph_error_releases_its_driver() {
14659 crate::run::pin_test_home();
14660 let mut runner = runner_at(RunStatus::Blocked);
14661 runner.state.config.graph.selected_implementer = Some("selected".into());
14662 runner.state.save().unwrap();
14663 assert!(runner.execute().await.is_err());
14664 let disk = RunState::load(&runner.state.id).unwrap();
14665 assert!(disk.driver_exited);
14666 assert_eq!(disk.driver_pid, None);
14667 assert_eq!(disk.status, RunStatus::Blocked);
14668 }
14669
14670 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14671 crate::run::MergeRepair {
14672 question: "repair-question".into(),
14673 head: "old-head".into(),
14674 pr: "https://github.com/o/r/pull/42".into(),
14675 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14676 gate: vec![outcome(Some(0), false)],
14677 gate_head: Some("old-head".into()),
14678 gate_fixes: Vec::new(),
14679 pre_gate: Vec::new(),
14680 phase,
14681 failure: None,
14682 }
14683 }
14684
14685 #[tokio::test]
14686 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14687 crate::run::pin_test_home();
14688 let mut runner = runner_at(RunStatus::Landing);
14689 runner.state.merge_repairs.push(merge_repair_request(
14690 crate::run::MergeRepairPhase::Requested,
14691 ));
14692 runner.state.save().unwrap();
14693 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14694 assert!(
14695 runner
14696 .execute()
14697 .await
14698 .unwrap_err()
14699 .to_string()
14700 .contains("already running")
14701 );
14702 assert_eq!(
14703 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14704 crate::run::MergeRepairPhase::Requested
14705 );
14706 drop(claim);
14707 let mut disk = runner.state.clone();
14708 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14709 disk.save().unwrap();
14710 assert!(
14711 runner
14712 .execute()
14713 .await
14714 .unwrap_err()
14715 .to_string()
14716 .contains("decision changed")
14717 );
14718 assert_eq!(
14719 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14720 crate::run::MergeRepairPhase::Complete
14721 );
14722 }
14723
14724 #[test]
14725 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14726 crate::run::pin_test_home();
14727 let mut runner = runner_at(RunStatus::Landing);
14728 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14729 runner.state.reviews = request.reviews.clone();
14730 runner.state.gate = request.gate.clone();
14731 runner.state.gate_ran = true;
14732 runner.state.land_approval = Some(crate::run::LandApproval {
14733 question: "old".into(),
14734 head: "old-head".into(),
14735 });
14736 runner.state.merge_repairs.push(request);
14737 runner.begin_merge_repair_review(0).unwrap();
14738 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14739 assert!(!runner.state.gate_status().ok());
14740 assert!(runner.state.land_approval.is_none());
14741 let disk = RunState::load(&runner.state.id).unwrap();
14742 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14743 assert!(disk.merge_repairs[0].gate[0].ok());
14744 assert_eq!(
14745 disk.merge_repairs[0].phase,
14746 crate::run::MergeRepairPhase::Reviewing
14747 );
14748 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14749 assert!(disk.reviews.is_empty());
14750 }
14751
14752 #[tokio::test]
14753 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14754 crate::run::pin_test_home();
14755 for phase in [
14756 crate::run::MergeRepairPhase::Fixing,
14757 crate::run::MergeRepairPhase::Requested,
14758 ] {
14759 let mut runner = runner_at(RunStatus::Landing);
14760 runner.state.config.graph.land_rounds = 0;
14761 runner.state.merge_repairs.push(merge_repair_request(phase));
14762 runner.state.save().unwrap();
14763 assert!(!runner.repair_merge_review().await.unwrap());
14764 assert_eq!(runner.state.status, RunStatus::Blocked);
14765 assert_eq!(
14766 runner.state.merge_repairs[0].phase,
14767 crate::run::MergeRepairPhase::Failed
14768 );
14769 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14770 assert!(
14771 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14772 "interrupted"
14773 } else {
14774 "budget exhausted"
14775 })
14776 );
14777 assert!(runner.state.merge.is_none());
14778 }
14779 }
14780
14781 #[test]
14785 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14786 crate::run::pin_test_home();
14787 let mut runner = runner_at(RunStatus::Implementing);
14788 let interrupt = Pause::new();
14789 runner.watch_interrupt(interrupt.clone());
14790
14791 interrupt.park_because("task a1b2 asked to run first");
14792
14793 assert!(runner.park_here().expect("park_here"));
14794 assert!(runner.state.parked);
14795 let last = runner.state.events.last().expect("a park event");
14796 assert_eq!(last.node, "park");
14797 assert!(
14798 last.message.contains("task a1b2 asked to run first"),
14799 "expected the interrupt reason in {:?}",
14800 last.message
14801 );
14802 }
14803
14804 #[test]
14807 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14808 crate::run::pin_test_home();
14809 let mut runner = runner_at(RunStatus::Implementing);
14810 runner.state.begin_pass();
14811 let shutdown = Pause::new();
14812 runner.on_pause(shutdown.clone());
14813 let upgrade = ParkCause::Upgrade {
14814 from: "0.121.0".to_owned(),
14815 to: Some("0.121.1".to_owned()),
14816 };
14817 shutdown.park_for(upgrade.clone());
14818 shutdown.park_for(ParkCause::Operator);
14819
14820 assert!(runner.park_here().expect("park_here"));
14821 assert_eq!(
14822 runner.state.passes[0].end,
14823 Some(crate::run::PassEnd::Parked { cause: upgrade })
14824 );
14825 let last = runner.state.events.last().expect("a park event");
14826 assert!(last.message.contains("0.121.0"), "{}", last.message);
14827 }
14828
14829 #[test]
14837 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14838 crate::run::pin_test_home();
14839 let mut runner = runner_at(RunStatus::Implementing);
14840 let shutdown = Pause::new();
14841 runner.on_pause(shutdown.clone());
14842 let interrupt = Pause::new();
14843 runner.watch_interrupt(interrupt.clone());
14844
14845 assert!(!runner.park_here().expect("park_here"));
14847 assert!(!runner.state.parked);
14848
14849 interrupt.park_because("test");
14851 assert!(!shutdown.parked());
14852 assert!(runner.park_here().expect("park_here"));
14853 }
14854
14855 #[tokio::test]
14869 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14870 crate::run::pin_test_home();
14871 let mut runner = runner_at(RunStatus::Implementing);
14872 let interrupt = Pause::new();
14873 runner.watch_interrupt(interrupt.clone());
14874
14875 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14876 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14877
14878 let node = async move {
14882 started_tx.send(()).expect("send started");
14883 finish_rx.await.expect("recv finish");
14884 "node finished"
14885 };
14886
14887 let interrupter = async move {
14888 started_rx.await.expect("recv started");
14889 interrupt.park_because("higher-priority task waiting");
14891 tokio::task::yield_now().await;
14895 finish_tx.send(()).expect("send finish");
14896 };
14897
14898 let (node_result, ()) = tokio::join!(node, interrupter);
14899 assert_eq!(
14900 node_result, "node finished",
14901 "the in-flight call ran to completion"
14902 );
14903
14904 assert!(runner.park_here().expect("park_here"));
14907 assert!(runner.state.parked);
14908 }
14909
14910 #[test]
14916 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14917 crate::run::pin_test_home();
14918 let mut runner = runner_at(RunStatus::Judging);
14919 runner.state.config.agents = vec![conductor()];
14923 runner.state.candidates = vec![Candidate {
14924 index: 0,
14925 label: 'A',
14926 agent: "alpha".to_owned(),
14927 branch: "magi/x/A".to_owned(),
14928 worktree: PathBuf::from("/nonexistent/worktree"),
14929 summary: "did the thing".to_owned(),
14930 stat: "1 file changed".to_owned(),
14931 files: 1,
14932 commits: 1,
14933 empty: false,
14934 failed: None,
14935 verified_noop: None,
14936 duration_ms: 1234,
14937 folded: false,
14938 }];
14939 let run_id = runner.state.id.clone();
14940
14941 let interrupt = Pause::new();
14942 runner.watch_interrupt(interrupt.clone());
14943 interrupt.park_because("task c3d4 asked to run first");
14944 assert!(runner.park_here().expect("park_here"));
14945
14946 let resumed = Runner::resume(&run_id).expect("resume");
14947 assert_eq!(resumed.state.candidates.len(), 1);
14948 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14949 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14950 assert_eq!(resumed.state.status, runner.state.status);
14951 assert!(
14952 resumed.state.parked,
14953 "still parked until `execute` actually walks the graph again"
14954 );
14955 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14956 }
14957
14958 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14960 let mut q = ask::Question::new(
14961 run.to_owned(),
14962 "implement".to_owned(),
14963 "impl-A".to_owned(),
14964 "Which storage backend should the cache use?".to_owned(),
14965 String::new(),
14966 vec!["SQLite".to_owned(), "Redis".to_owned()],
14967 );
14968 store.put(&mut q).unwrap();
14969 q
14970 }
14971
14972 #[test]
14973 fn a_failed_runs_open_question_is_abandoned() {
14974 ask_test_home();
14975 let store = ask::Questions::open();
14976 let mut runner = runner_at(RunStatus::Failed);
14977 let run = runner.state.id.clone();
14978 let q = ask_open_question(&store, &run);
14979
14980 runner.settle_questions();
14981
14982 let back = store.get(&q.id).unwrap();
14983 assert!(
14984 !back.status.open(),
14985 "the seat that asked died with the run; nobody is left to read an answer"
14986 );
14987 assert!(
14988 back.detail.contains(&run) && back.detail.contains("failed"),
14989 "the reason names what the run became, not just that it is gone: {}",
14990 back.detail
14991 );
14992 }
14993
14994 #[test]
14995 fn a_merged_runs_open_question_is_abandoned_too() {
14996 ask_test_home();
14997 let store = ask::Questions::open();
14998 for status in [RunStatus::Merged, RunStatus::Ready] {
15001 let mut runner = runner_at(status);
15002 let run = runner.state.id.clone();
15003 let q = ask_open_question(&store, &run);
15004
15005 runner.settle_questions();
15006
15007 let back = store.get(&q.id).unwrap();
15008 assert!(
15009 !back.status.open(),
15010 "{status:?} run's question must not outlive the run"
15011 );
15012 }
15013 }
15014
15015 #[test]
15016 fn a_still_resumable_runs_open_question_is_left_alone() {
15017 ask_test_home();
15018 let store = ask::Questions::open();
15019 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15025 let mut runner = runner_at(status);
15026 let run = runner.state.id.clone();
15027 let q = ask_open_question(&store, &run);
15028
15029 runner.settle_questions();
15030
15031 let back = store.get(&q.id).unwrap();
15032 assert!(
15033 back.status.open(),
15034 "{status:?} is still alive; the question must still be waiting"
15035 );
15036 }
15037 }
15038
15039 #[test]
15040 fn settle_questions_never_touches_an_already_answered_question() {
15041 ask_test_home();
15042 let store = ask::Questions::open();
15043 let mut runner = runner_at(RunStatus::Failed);
15044 let run = runner.state.id.clone();
15045 let mut q = ask_open_question(&store, &run);
15046 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15047 .unwrap();
15048 store.put(&mut q).unwrap();
15049
15050 runner.settle_questions();
15055 runner.settle_questions();
15056
15057 let back = store.get(&q.id).unwrap();
15058 assert_eq!(
15059 back.status,
15060 ask::QuestionStatus::Answered,
15061 "a real answer is a decision on record, never overwritten by a sweep"
15062 );
15063 }
15064
15065 #[tokio::test]
15076 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15077 crate::run::pin_test_home();
15078 let tmp = tempfile::tempdir().expect("tempdir");
15079 let repo = tmp.path().join("repo");
15080 std::fs::create_dir_all(&repo).unwrap();
15081 init_repo(&repo);
15082
15083 let mut config = Config::default();
15084 config.graph.worktree_root = Some(tmp.path().join("wt"));
15085
15086 let mut state = RunState::new(
15087 repo.clone(),
15088 "main".to_owned(),
15089 "deadbeef".to_owned(),
15090 "task".to_owned(),
15091 config,
15092 );
15093 let root = state.worktree_root();
15094 let wt_a = root.join("cand-A");
15095 let wt_b = root.join("cand-B");
15096 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15097 .await
15098 .expect("worktree A");
15099 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15100 .await
15101 .expect("worktree B");
15102
15103 state.candidates = vec![
15104 Candidate {
15105 index: 0,
15106 label: 'A',
15107 agent: "alpha".to_owned(),
15108 branch: "magi/x/A".to_owned(),
15109 worktree: wt_a.clone(),
15110 summary: String::new(),
15111 stat: String::new(),
15112 files: 0,
15113 commits: 0,
15114 empty: false,
15115 failed: None,
15116 verified_noop: None,
15117 duration_ms: 0,
15118 folded: false,
15119 },
15120 Candidate {
15121 index: 1,
15122 label: 'B',
15123 agent: "beta".to_owned(),
15124 branch: "magi/x/B".to_owned(),
15125 worktree: wt_b.clone(),
15126 summary: String::new(),
15127 stat: String::new(),
15128 files: 0,
15129 commits: 0,
15130 empty: false,
15131 failed: None,
15132 verified_noop: None,
15133 duration_ms: 0,
15134 folded: false,
15135 },
15136 ];
15137 state.tally = Some(Tally {
15138 first_choice: BTreeMap::from([('A', 1)]),
15139 borda: BTreeMap::new(),
15140 winner: 'A',
15141 rankings: 1,
15142 unanimous_initial: true,
15143 deliberated: false,
15144 changed_votes: 0,
15145 unanimous_final: true,
15146 tie_break: None,
15147 judges: 1,
15148 present: 1,
15149 quorum: 1,
15150 met_quorum: true,
15151 uncontested: None,
15152 });
15153 state.status = RunStatus::Ready;
15154
15155 fold_run(&mut state, false, &crate::run::home())
15156 .await
15157 .expect("fold_run");
15158
15159 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15160 assert!(
15161 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15162 "the unmerged winner's branch survives"
15163 );
15164 assert!(
15165 !state.candidates[0].folded,
15166 "the winner is not marked folded"
15167 );
15168
15169 assert!(!wt_b.exists(), "the loser's worktree is removed");
15170 assert!(
15171 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15172 "the loser's branch is removed"
15173 );
15174 assert!(state.candidates[1].folded, "the loser is marked folded");
15175 }
15176
15177 #[tokio::test]
15178 #[cfg(unix)]
15179 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15180 use std::os::fd::AsRawFd as _;
15181 use std::time::Duration;
15182
15183 let tmp = tempfile::tempdir().unwrap();
15184 let repo = tmp.path().join("repo");
15185 std::fs::create_dir_all(&repo).unwrap();
15186 init_repo(&repo);
15187 let common = repo.join(".git");
15188 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15189 let home = tmp.path().join("home");
15190 let mut config = Config::default();
15191 config.coordination = crate::config::test_coordination();
15192 config.graph.worktree_root = Some(tmp.path().join("wt"));
15193 config.graph.timeout_resource_wait = 1;
15194 let mut state = RunState::new(
15195 repo.clone(),
15196 "main".into(),
15197 "deadbeef".into(),
15198 "task".into(),
15199 config,
15200 );
15201 let root = state.worktree_root();
15202 let candidate = root.join("cand-A");
15203 let reviewer = root.join("review-1");
15204 let branch = state.branch_for('A');
15205 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15206 .await
15207 .unwrap();
15208 git::worktree_add_detached(&repo, &reviewer, "main")
15209 .await
15210 .unwrap();
15211 git::acquire_worktree_config(&repo).await.unwrap();
15212 state.enabled_worktree_config = true;
15213 state.status = RunStatus::Ready;
15214 state.candidates.push(Candidate {
15215 index: 0,
15216 label: 'A',
15217 agent: "implement".into(),
15218 branch: branch.clone(),
15219 worktree: candidate.clone(),
15220 summary: String::new(),
15221 stat: String::new(),
15222 files: 0,
15223 commits: 0,
15224 empty: false,
15225 failed: None,
15226 verified_noop: None,
15227 duration_ms: 0,
15228 folded: false,
15229 });
15230 state.save_under(&home).unwrap();
15231 let config_before = std::fs::read(common.join("config")).unwrap();
15232 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15233 .await
15234 .unwrap();
15235 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15238 assert_eq!(
15239 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15240 0
15241 );
15242 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15243 assert!(
15244 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15245 .await
15246 .is_err()
15247 );
15248 assert!(candidate.exists());
15249 assert!(reviewer.exists());
15250 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15251 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15252 assert_eq!(
15253 git::git(&repo, &["worktree", "list", "--porcelain"])
15254 .await
15255 .unwrap(),
15256 worktrees_before
15257 );
15258 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15259 .await
15260 .unwrap()
15261 .unwrap_err();
15262 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15263 drop(fold);
15264 assert!(candidate.exists() && reviewer.exists());
15265 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15266 assert_eq!(
15267 state.resource_block.as_ref().unwrap().kind,
15268 "repository coordination"
15269 );
15270 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15271 drop(external);
15272 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15273 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15274 .await
15275 .unwrap()
15276 .unwrap();
15277 drop(fold);
15278 assert!(removed.contains(&branch));
15279 assert!(!root.exists());
15280 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15281 assert!(state.candidates[0].folded);
15282 assert!(!state.enabled_worktree_config);
15283 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15284 let enabled = git::git_raw(
15285 &repo,
15286 &["config", "--local", "--get", "extensions.worktreeConfig"],
15287 )
15288 .await
15289 .unwrap();
15290 assert!(!enabled.ok(), "the last config reference was released");
15291 }
15292
15293 #[tokio::test]
15296 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15297 let tmp = tempfile::tempdir().expect("tempdir");
15298 let repo = tmp.path().join("repo");
15299 std::fs::create_dir_all(&repo).unwrap();
15300 init_repo(&repo);
15301 let home = tmp.path().join("home");
15302
15303 let mut config = Config::default();
15304 config.graph.worktree_root = Some(tmp.path().join("wt"));
15305 let mut state = RunState::new(
15306 repo.clone(),
15307 "main".to_owned(),
15308 "deadbeef".to_owned(),
15309 "task".to_owned(),
15310 config,
15311 );
15312 git::git(&repo, &["branch", "magi/x/A", "main"])
15314 .await
15315 .expect("branch");
15316 state.candidates = vec![Candidate {
15317 index: 0,
15318 label: 'A',
15319 agent: "alpha".to_owned(),
15320 branch: "magi/x/A".to_owned(),
15321 worktree: state.worktree_root().join("cand-A"),
15322 summary: String::new(),
15323 stat: String::new(),
15324 files: 0,
15325 commits: 0,
15326 empty: false,
15327 failed: None,
15328 verified_noop: None,
15329 duration_ms: 0,
15330 folded: true,
15331 }];
15332 state.released_to = Some("20260901-000000-new1".to_owned());
15333 state.released_branches = vec!["magi/x/A".to_owned()];
15334
15335 fold_run(&mut state, true, &home).await.expect("fold_run");
15336
15337 assert!(
15338 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15339 "the handed-over branch survives a fold"
15340 );
15341 }
15342
15343 #[tokio::test]
15347 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15348 let tmp = tempfile::tempdir().expect("tempdir");
15349 let repo = tmp.path().join("repo");
15350 std::fs::create_dir_all(&repo).unwrap();
15351 init_repo(&repo);
15352 let run = |args: &[&str]| {
15353 let out = std::process::Command::new("git")
15354 .quiet()
15355 .args(args)
15356 .current_dir(&repo)
15357 .output()
15358 .expect("spawn git");
15359 assert!(out.status.success(), "git {args:?}");
15360 };
15361 run(&["branch", "magi/x/A"]);
15362 run(&["checkout", "-q", "-b", "magi/x/B"]);
15363 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15364 run(&["add", "-A"]);
15365 run(&["commit", "-q", "-m", "work"]);
15366 run(&["checkout", "-q", "main"]);
15367
15368 let probe = RunState::new(
15372 repo.clone(),
15373 "main".to_owned(),
15374 "deadbeef".to_owned(),
15375 "task".to_owned(),
15376 Config::default(),
15377 );
15378 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15379 let bare = tmp.path().join("origin.git");
15380 let out = std::process::Command::new("git")
15381 .quiet()
15382 .args(["init", "-q", "--bare"])
15383 .arg(&bare)
15384 .output()
15385 .expect("spawn git");
15386 assert!(out.status.success(), "git init --bare");
15387 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15388 run(&["push", "-q", "origin", "main"]);
15389
15390 let mut state = RunState::new(
15391 repo.clone(),
15392 "main".to_owned(),
15393 "deadbeef".to_owned(),
15394 "task".to_owned(),
15395 Config::default(),
15396 );
15397 state.seeds = vec![refs::Seed {
15398 token: "magi/27b2/A".to_owned(),
15399 kind: refs::SeedKind::Unresolved,
15400 sha: String::new(),
15401 branch: true,
15402 detail: "no branch or commit named magi/27b2/A".to_owned(),
15403 }];
15404
15405 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15406 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15407 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15408 let detail = empty_candidate_detail(&state, "main");
15409 assert!(detail.starts_with("empty candidate"), "{detail}");
15410 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15411 assert!(detail.contains("--start-from"), "{detail}");
15412 }
15413
15414 #[tokio::test]
15423 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15424 let tmp = tempfile::tempdir().expect("tempdir");
15425 let repo = tmp.path().join("repo");
15426 std::fs::create_dir_all(&repo).unwrap();
15427 init_repo(&repo);
15428
15429 let mut config = Config::default();
15430 config.merge.mode = MergeMode::Local;
15431
15432 let mut state = RunState::new(
15433 repo.clone(),
15434 "main".to_owned(),
15435 "deadbeef".to_owned(),
15436 "task".to_owned(),
15437 config,
15438 );
15439 state.candidates = vec![Candidate {
15440 index: 0,
15441 label: 'A',
15442 agent: "alpha".to_owned(),
15443 branch: "does-not-exist".to_owned(),
15444 worktree: repo.clone(),
15445 summary: String::new(),
15446 stat: String::new(),
15447 files: 0,
15448 commits: 0,
15449 empty: false,
15450 failed: None,
15451 verified_noop: None,
15452 duration_ms: 0,
15453 folded: false,
15454 }];
15455 state.tally = Some(Tally {
15456 first_choice: BTreeMap::from([('A', 1)]),
15457 borda: BTreeMap::new(),
15458 winner: 'A',
15459 rankings: 1,
15460 unanimous_initial: true,
15461 deliberated: false,
15462 changed_votes: 0,
15463 unanimous_final: true,
15464 tie_break: None,
15465 judges: 0,
15466 present: 0,
15467 quorum: 0,
15468 met_quorum: true,
15469 uncontested: Some("only candidate A produced a change".to_owned()),
15470 });
15471 state.reviews = vec![ReviewRound {
15472 round: 1,
15473 head: "deadbeef".to_owned(),
15474 verified_head: None,
15475 verified_at: None,
15476 reviews: Vec::new(),
15477 e2e: Vec::new(),
15478 fix: None,
15479 blocking: 0,
15480 answered: 0,
15481 expected: 0,
15482 clean: true,
15483 verify_retried: false,
15484 e2e_deferred: false,
15485 e2e_defer_reason: None,
15486 progressed: false,
15487 vote_split: false,
15488 reconsideration: Vec::new(),
15489 verdict: None,
15490 }];
15491 state.gate = vec![CommandOutcome {
15492 command: "test".to_owned(),
15493 code: Some(0),
15494 output_tail: String::new(),
15495 duration_ms: 0,
15496 resource_blocked: false,
15497 }];
15498 state.gate_ran = true;
15499 state.status = RunStatus::Ready;
15504 state.merge = Some(MergeOutcome {
15505 mode: MergeMode::Local,
15506 ok: false,
15507 detail: "already concluded".to_owned(),
15508 empty: false,
15509 });
15510
15511 let mut runner = Runner {
15512 state,
15513 roles: ResolvedRoles {
15514 implementers: Vec::new(),
15515 judges: Vec::new(),
15516 reviewers: Vec::new(),
15517 fixer: None,
15518 conductor: conductor(),
15519 implementer_roster: Vec::new(),
15520 judge_roster: Vec::new(),
15521 reviewer_roster: Vec::new(),
15522 },
15523 sem: Arc::new(Semaphore::new(1)),
15524 pause: Pause::new(),
15525 interrupt: Pause::new(),
15526 };
15527
15528 runner.merge().await.expect("merge");
15529
15530 assert_eq!(
15531 runner.state.status,
15532 RunStatus::Ready,
15533 "a concluded run's status must not change on reentry"
15534 );
15535 assert_eq!(
15536 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15537 Some("already concluded"),
15538 "merge must not run again once the node already recorded an outcome"
15539 );
15540 }
15541
15542 #[tokio::test]
15551 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15552 let tmp = tempfile::tempdir().expect("tempdir");
15553 let repo = tmp.path().join("repo");
15554 std::fs::create_dir_all(&repo).unwrap();
15555 init_repo(&repo);
15556
15557 let mut config = Config::default();
15558 config.merge.mode = MergeMode::Local;
15559
15560 let mut state = RunState::new(
15561 repo.clone(),
15562 "main".to_owned(),
15563 "deadbeef".to_owned(),
15564 "task".to_owned(),
15565 config,
15566 );
15567 state.candidates = vec![Candidate {
15568 index: 0,
15569 label: 'A',
15570 agent: "alpha".to_owned(),
15571 branch: "does-not-exist".to_owned(),
15572 worktree: repo.clone(),
15573 summary: String::new(),
15574 stat: String::new(),
15575 files: 0,
15576 commits: 0,
15577 empty: false,
15578 failed: None,
15579 verified_noop: None,
15580 duration_ms: 0,
15581 folded: false,
15582 }];
15583 state.tally = Some(Tally {
15584 first_choice: BTreeMap::from([('A', 1)]),
15585 borda: BTreeMap::new(),
15586 winner: 'A',
15587 rankings: 1,
15588 unanimous_initial: true,
15589 deliberated: false,
15590 changed_votes: 0,
15591 unanimous_final: true,
15592 tie_break: None,
15593 judges: 0,
15594 present: 0,
15595 quorum: 0,
15596 met_quorum: true,
15597 uncontested: Some("only candidate A produced a change".to_owned()),
15598 });
15599 state.reviews = vec![ReviewRound {
15600 round: 1,
15601 head: "deadbeef".to_owned(),
15602 verified_head: None,
15603 verified_at: None,
15604 reviews: Vec::new(),
15605 e2e: Vec::new(),
15606 fix: None,
15607 blocking: 0,
15608 answered: 0,
15609 expected: 0,
15610 clean: true,
15611 verify_retried: false,
15612 e2e_deferred: false,
15613 e2e_defer_reason: None,
15614 progressed: false,
15615 vote_split: false,
15616 reconsideration: Vec::new(),
15617 verdict: None,
15618 }];
15619 state.gate = Vec::new();
15621 state.gate_ran = false;
15622 state.status = RunStatus::Gating;
15623
15624 let mut runner = Runner {
15625 state,
15626 roles: ResolvedRoles {
15627 implementers: Vec::new(),
15628 judges: Vec::new(),
15629 reviewers: Vec::new(),
15630 fixer: None,
15631 conductor: conductor(),
15632 implementer_roster: Vec::new(),
15633 judge_roster: Vec::new(),
15634 reviewer_roster: Vec::new(),
15635 },
15636 sem: Arc::new(Semaphore::new(1)),
15637 pause: Pause::new(),
15638 interrupt: Pause::new(),
15639 };
15640
15641 runner.merge().await.expect("merge");
15642
15643 assert!(
15644 runner.state.merge.is_none(),
15645 "an empty gate must never be read as a passing one: {:?}",
15646 runner.state.merge
15647 );
15648 }
15649
15650 #[tokio::test]
15657 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15658 let tmp = tempfile::tempdir().expect("tempdir");
15659 let repo = tmp.path().join("repo");
15660 std::fs::create_dir_all(&repo).unwrap();
15661 init_repo(&repo);
15662
15663 let config = Config::default();
15665
15666 let mut state = RunState::new(
15667 repo.clone(),
15668 "main".to_owned(),
15669 "deadbeef".to_owned(),
15670 "task".to_owned(),
15671 config,
15672 );
15673 state.candidates = vec![Candidate {
15674 index: 0,
15675 label: 'A',
15676 agent: "alpha".to_owned(),
15677 branch: "does-not-exist".to_owned(),
15678 worktree: repo.clone(),
15679 summary: String::new(),
15680 stat: String::new(),
15681 files: 0,
15682 commits: 0,
15683 empty: false,
15684 failed: None,
15685 verified_noop: None,
15686 duration_ms: 0,
15687 folded: false,
15688 }];
15689 state.tally = Some(Tally {
15690 first_choice: BTreeMap::from([('A', 1)]),
15691 borda: BTreeMap::new(),
15692 winner: 'A',
15693 rankings: 1,
15694 unanimous_initial: true,
15695 deliberated: false,
15696 changed_votes: 0,
15697 unanimous_final: true,
15698 tie_break: None,
15699 judges: 0,
15700 present: 0,
15701 quorum: 0,
15702 met_quorum: true,
15703 uncontested: Some("only candidate A produced a change".to_owned()),
15704 });
15705 state.reviews = vec![ReviewRound {
15706 round: 1,
15707 head: "deadbeef".to_owned(),
15708 verified_head: None,
15709 verified_at: None,
15710 reviews: Vec::new(),
15711 e2e: Vec::new(),
15712 fix: None,
15713 blocking: 0,
15714 answered: 0,
15715 expected: 0,
15716 clean: true,
15717 verify_retried: false,
15718 e2e_deferred: false,
15719 e2e_defer_reason: None,
15720 progressed: false,
15721 vote_split: false,
15722 reconsideration: Vec::new(),
15723 verdict: None,
15724 }];
15725
15726 let mut runner = Runner {
15727 state,
15728 roles: ResolvedRoles {
15729 implementers: Vec::new(),
15730 judges: Vec::new(),
15731 reviewers: Vec::new(),
15732 fixer: None,
15733 conductor: conductor(),
15734 implementer_roster: Vec::new(),
15735 judge_roster: Vec::new(),
15736 reviewer_roster: Vec::new(),
15737 },
15738 sem: Arc::new(Semaphore::new(1)),
15739 pause: Pause::new(),
15740 interrupt: Pause::new(),
15741 };
15742
15743 runner.gate().await.expect("gate");
15744 assert!(
15745 runner.state.gate_ran,
15746 "zero configured commands is still a real attempt, not an unrun gate"
15747 );
15748 assert!(runner.state.gate.is_empty());
15749 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15750 assert_ne!(
15751 runner.state.status,
15752 RunStatus::Blocked,
15753 "a gate with nothing to check must not read as failed"
15754 );
15755
15756 runner.merge().await.expect("merge");
15757 assert_eq!(
15758 runner.state.status,
15759 RunStatus::Ready,
15760 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15761 );
15762 }
15763
15764 #[tokio::test]
15775 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15776 crate::run::pin_test_home();
15777 let home = crate::run::home();
15778
15779 let tmp = tempfile::tempdir().expect("tempdir");
15780 let repo = tmp.path().join("repo");
15781 std::fs::create_dir_all(&repo).unwrap();
15782 init_repo(&repo);
15783 let cache_dir = tmp.path().join("target");
15786
15787 let mut config = Config::default();
15788 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15789 config.graph.timeout_verify = Some(2);
15792 config.graph.timeout_resource_wait = 2;
15793
15794 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15795 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15796 .expect("no io error acquiring directly")
15797 {
15798 crate::cache::AcquireOutcome::Acquired(g) => g,
15799 crate::cache::AcquireOutcome::Busy(b) => {
15800 panic!("expected the direct acquire to win the lease first: {b:?}")
15801 }
15802 };
15803
15804 let mut state = RunState::new(
15805 repo.clone(),
15806 "main".to_owned(),
15807 "deadbeef".to_owned(),
15808 "task".to_owned(),
15809 config,
15810 );
15811 state.candidates = vec![Candidate {
15812 index: 0,
15813 label: 'A',
15814 agent: "alpha".to_owned(),
15815 branch: "does-not-exist".to_owned(),
15816 worktree: repo.clone(),
15817 summary: String::new(),
15818 stat: String::new(),
15819 files: 0,
15820 commits: 0,
15821 empty: false,
15822 failed: None,
15823 verified_noop: None,
15824 duration_ms: 0,
15825 folded: false,
15826 }];
15827 state.tally = Some(Tally {
15828 first_choice: BTreeMap::from([('A', 1)]),
15829 borda: BTreeMap::new(),
15830 winner: 'A',
15831 rankings: 1,
15832 unanimous_initial: true,
15833 deliberated: false,
15834 changed_votes: 0,
15835 unanimous_final: true,
15836 tie_break: None,
15837 judges: 0,
15838 present: 0,
15839 quorum: 0,
15840 met_quorum: true,
15841 uncontested: Some("only candidate A produced a change".to_owned()),
15842 });
15843 state.reviews = vec![ReviewRound {
15844 round: 1,
15845 head: "deadbeef".to_owned(),
15846 verified_head: None,
15847 verified_at: None,
15848 reviews: Vec::new(),
15849 e2e: Vec::new(),
15850 fix: None,
15851 blocking: 0,
15852 answered: 0,
15853 expected: 0,
15854 clean: true,
15855 verify_retried: false,
15856 e2e_deferred: false,
15857 e2e_defer_reason: None,
15858 progressed: false,
15859 vote_split: false,
15860 reconsideration: Vec::new(),
15861 verdict: None,
15862 }];
15863
15864 let mut runner = Runner {
15865 state,
15866 roles: ResolvedRoles {
15867 implementers: Vec::new(),
15868 judges: Vec::new(),
15869 reviewers: Vec::new(),
15870 fixer: None,
15871 conductor: conductor(),
15872 implementer_roster: Vec::new(),
15873 judge_roster: Vec::new(),
15874 reviewer_roster: Vec::new(),
15875 },
15876 sem: Arc::new(Semaphore::new(1)),
15877 pause: Pause::new(),
15878 interrupt: Pause::new(),
15879 };
15880
15881 let started = std::time::Instant::now();
15882 runner.gate().await.expect("gate");
15883 assert!(
15884 started.elapsed() < Duration::from_secs(1),
15885 "a gate with nothing to run must never wait on a lease it never needed"
15886 );
15887 assert!(
15888 runner.state.gate_ran,
15889 "zero commands is still a real, immediate attempt"
15890 );
15891 assert!(runner.state.gate.is_empty());
15892 assert_ne!(
15893 runner.state.status,
15894 RunStatus::Blocked,
15895 "must not read as resource-blocked on a lease it never asked for"
15896 );
15897 }
15898
15899 #[cfg(unix)]
15900 #[tokio::test]
15901 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15902 crate::run::pin_test_home();
15903 let tmp = tempfile::tempdir().unwrap();
15904 let mut config = Config::default();
15905 config.disk.min_free_bytes = 0;
15906 let mut state = RunState::new(
15907 tmp.path().to_owned(),
15908 "main".into(),
15909 "base".into(),
15910 "verification evidence".into(),
15911 config,
15912 );
15913 let (outcomes, pids) = run_commands(
15914 &mut state,
15915 "gate",
15916 "gate",
15917 1,
15918 &["sh".into(), "-c".into()],
15919 &[
15920 "printf before-timeout; exec sleep 30".into(),
15921 "kill -TERM $$".into(),
15922 "exit 7".into(),
15923 "printf success".into(),
15924 ],
15925 tmp.path(),
15926 Duration::from_secs(1),
15927 )
15928 .await;
15929 wait_for_timed_out_children_to_die(&pids).await;
15930 assert_eq!(
15931 pids.len(),
15932 1,
15933 "unleased timeout retains its death-check PID"
15934 );
15935 assert_eq!(outcomes.len(), 4);
15936 assert_eq!(outcomes[0].code, None);
15937 assert!(outcomes[0].output_tail.contains("before-timeout"));
15938 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15939 assert_eq!(outcomes[1].code, None);
15940 assert!(outcomes[1].output_tail.contains("command terminated:"));
15941 assert!(!outcomes[1].output_tail.contains("timed out"));
15942 assert_eq!(outcomes[2].code, Some(7));
15943 assert!(!outcomes[2].output_tail.contains("timed out"));
15944 assert_eq!(outcomes[3].code, Some(0));
15945 assert_eq!(outcomes[3].output_tail, "success");
15946 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15947 }
15948
15949 #[tokio::test]
15959 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15960 crate::run::pin_test_home();
15961
15962 let tmp = tempfile::tempdir().expect("tempdir");
15963 let repo = tmp.path().join("repo");
15964 std::fs::create_dir_all(&repo).unwrap();
15965 init_repo(&repo);
15966
15967 let mut config = Config::default();
15968 config.verify.gate = vec![
15969 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15970 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15971 .to_owned(),
15972 ];
15973
15974 let mut state = RunState::new(
15975 repo.clone(),
15976 "main".to_owned(),
15977 "deadbeef".to_owned(),
15978 "task".to_owned(),
15979 config,
15980 );
15981 let run_id = state.id.clone();
15982 state.candidates = vec![Candidate {
15983 index: 0,
15984 label: 'A',
15985 agent: "alpha".to_owned(),
15986 branch: "does-not-exist".to_owned(),
15987 worktree: repo.clone(),
15988 summary: String::new(),
15989 stat: String::new(),
15990 files: 0,
15991 commits: 0,
15992 empty: false,
15993 failed: None,
15994 verified_noop: None,
15995 duration_ms: 0,
15996 folded: false,
15997 }];
15998 state.tally = Some(Tally {
15999 first_choice: BTreeMap::from([('A', 1)]),
16000 borda: BTreeMap::new(),
16001 winner: 'A',
16002 rankings: 1,
16003 unanimous_initial: true,
16004 deliberated: false,
16005 changed_votes: 0,
16006 unanimous_final: true,
16007 tie_break: None,
16008 judges: 0,
16009 present: 0,
16010 quorum: 0,
16011 met_quorum: true,
16012 uncontested: Some("only candidate A produced a change".to_owned()),
16013 });
16014 state.reviews = vec![ReviewRound {
16015 round: 1,
16016 head: "deadbeef".to_owned(),
16017 verified_head: None,
16018 verified_at: None,
16019 reviews: Vec::new(),
16020 e2e: Vec::new(),
16021 fix: None,
16022 blocking: 0,
16023 answered: 0,
16024 expected: 0,
16025 clean: true,
16026 verify_retried: false,
16027 e2e_deferred: false,
16028 e2e_defer_reason: None,
16029 progressed: false,
16030 vote_split: false,
16031 reconsideration: Vec::new(),
16032 verdict: None,
16033 }];
16034
16035 let mut runner = Runner {
16036 state,
16037 roles: ResolvedRoles {
16038 implementers: Vec::new(),
16039 judges: Vec::new(),
16040 reviewers: Vec::new(),
16041 fixer: None,
16042 conductor: conductor(),
16043 implementer_roster: Vec::new(),
16044 judge_roster: Vec::new(),
16045 reviewer_roster: Vec::new(),
16046 },
16047 sem: Arc::new(Semaphore::new(1)),
16048 pause: Pause::new(),
16049 interrupt: Pause::new(),
16050 };
16051
16052 let started_marker = repo.join("started.marker");
16053 let release_marker = repo.join("release.marker");
16054 let poller = tokio::spawn(async move {
16055 for _ in 0..100 {
16060 if started_marker.exists()
16061 && let Ok(s) = crate::run::RunState::load(&run_id)
16062 && let Some(a) = s.active.get("gate")
16063 {
16064 std::fs::write(&release_marker, b"go").expect("release marker");
16065 return Some(a.clone());
16066 }
16067 tokio::time::sleep(Duration::from_millis(50)).await;
16068 }
16069 None
16070 });
16071
16072 runner.gate().await.expect("gate");
16073 let captured = poller.await.expect("poller task");
16074 let captured = captured.expect(
16075 "the poller never saw a `gate` task entry in run.json while the command was \
16076 still blocked on its own release marker",
16077 );
16078
16079 assert_eq!(captured.task.as_deref(), Some("gate"));
16080 assert_eq!(captured.node, "gate");
16081 assert_eq!(captured.index, Some(1));
16082 assert_eq!(captured.total, Some(1));
16083 assert!(
16084 captured
16085 .command
16086 .as_deref()
16087 .is_some_and(|c| c.contains("started.marker")),
16088 "{captured:?}"
16089 );
16090
16091 assert!(
16092 runner.state.active.is_empty(),
16093 "the entry must be cleared once the command actually finished: {:?}",
16094 runner.state.active
16095 );
16096 assert!(runner.state.gate_ran);
16097 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16098 }
16099
16100 #[tokio::test]
16103 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16104 use crate::verdict::{Finding, ReviewVote, Severity};
16105 crate::run::pin_test_home();
16106 let tmp = tempfile::tempdir().expect("tempdir");
16107 let repo = tmp.path().join("repo");
16108 std::fs::create_dir_all(&repo).unwrap();
16109 init_repo(&repo);
16110
16111 for (severity, vote, expect) in [
16112 (Severity::Major, ReviewVote::Reject, true),
16113 (Severity::Minor, ReviewVote::Reject, false),
16114 (Severity::Major, ReviewVote::Approve, false),
16115 ] {
16116 let mut round = review_round(false, 1, 1, 1, false, true);
16117 round.reviews = vec![ReviewRecord {
16118 reviewer: 1,
16119 agent: "alpha".to_owned(),
16120 summary: String::new(),
16121 findings: vec![Finding {
16122 id: "R1-1-1".to_owned(),
16123 severity,
16124 file: None,
16125 line: None,
16126 title: "t".to_owned(),
16127 detail: String::new(),
16128 }],
16129 vote: Some(vote),
16130 failed: None,
16131 duration_ms: 0,
16132 attempts: 0,
16133 }];
16134 let mut state = RunState::new(
16135 repo.clone(),
16136 "main".to_owned(),
16137 "deadbeef".to_owned(),
16138 "task".to_owned(),
16139 Config::default(),
16140 );
16141 state.reviews = vec![round];
16142 let mut runner = Runner {
16143 state,
16144 roles: ResolvedRoles {
16145 implementers: Vec::new(),
16146 judges: Vec::new(),
16147 reviewers: Vec::new(),
16148 fixer: None,
16149 conductor: conductor(),
16150 implementer_roster: Vec::new(),
16151 judge_roster: Vec::new(),
16152 reviewer_roster: Vec::new(),
16153 },
16154 sem: Arc::new(Semaphore::new(1)),
16155 pause: Pause::new(),
16156 interrupt: Pause::new(),
16157 };
16158 let shell = runner.state.config.shell();
16159 runner
16160 .stop_reviewing("round budget spent", &shell, &repo)
16161 .await
16162 .expect("stop_reviewing");
16163 assert_eq!(runner.state.status, RunStatus::Gating);
16164 assert_eq!(
16165 runner.state.contested_handoff.is_some(),
16166 expect,
16167 "{severity:?} + {vote:?}"
16168 );
16169 }
16170 }
16171
16172 #[tokio::test]
16185 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16186 crate::run::pin_test_home();
16187 let home = crate::run::home();
16188
16189 let tmp = tempfile::tempdir().expect("tempdir");
16190 let repo = tmp.path().join("repo");
16191 std::fs::create_dir_all(&repo).unwrap();
16192 init_repo(&repo);
16193 let head = crate::git::rev_parse(&repo, "HEAD")
16194 .await
16195 .expect("rev-parse");
16196 let cache_dir = tmp.path().join("target");
16199
16200 let mut config = Config::default();
16201 config.verify.e2e = vec![format!(
16202 "CARGO_TARGET_DIR='{}' test -f README.md",
16203 cache_dir.display()
16204 )];
16205 config.graph.review_rounds = 1;
16206 config.graph.timeout_verify = Some(2);
16209 config.graph.timeout_resource_wait = 2;
16210
16211 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16212 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16213 .expect("no io error acquiring directly")
16214 {
16215 crate::cache::AcquireOutcome::Acquired(g) => g,
16216 crate::cache::AcquireOutcome::Busy(b) => {
16217 panic!("expected the direct acquire to win the lease first: {b:?}")
16218 }
16219 };
16220
16221 let mut state = RunState::new(
16222 repo.clone(),
16223 "main".to_owned(),
16224 head.clone(),
16225 "task".to_owned(),
16226 config,
16227 );
16228 state.candidates = vec![Candidate {
16229 index: 0,
16230 label: 'A',
16231 agent: "alpha".to_owned(),
16232 branch: "does-not-exist".to_owned(),
16233 worktree: repo.clone(),
16234 summary: String::new(),
16235 stat: String::new(),
16236 files: 0,
16237 commits: 0,
16238 empty: false,
16239 failed: None,
16240 verified_noop: None,
16241 duration_ms: 0,
16242 folded: false,
16243 }];
16244 state.tally = Some(Tally {
16245 first_choice: BTreeMap::from([('A', 1)]),
16246 borda: BTreeMap::new(),
16247 winner: 'A',
16248 rankings: 1,
16249 unanimous_initial: true,
16250 deliberated: false,
16251 changed_votes: 0,
16252 unanimous_final: true,
16253 tie_break: None,
16254 judges: 0,
16255 present: 0,
16256 quorum: 0,
16257 met_quorum: true,
16258 uncontested: Some("only candidate A produced a change".to_owned()),
16259 });
16260 state.reviews = vec![ReviewRound {
16264 round: 1,
16265 head: head.clone(),
16266 verified_head: None,
16267 verified_at: None,
16268 reviews: Vec::new(),
16269 e2e: Vec::new(),
16270 fix: None,
16271 blocking: 1,
16272 answered: 1,
16273 expected: 1,
16274 clean: false,
16275 verify_retried: false,
16276 e2e_deferred: true,
16277 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16278 progressed: false,
16279 vote_split: false,
16280 reconsideration: Vec::new(),
16281 verdict: None,
16282 }];
16283
16284 let mut runner = Runner {
16285 state,
16286 roles: ResolvedRoles {
16287 implementers: Vec::new(),
16288 judges: Vec::new(),
16289 reviewers: Vec::new(),
16290 fixer: None,
16291 conductor: conductor(),
16292 implementer_roster: Vec::new(),
16293 judge_roster: Vec::new(),
16294 reviewer_roster: Vec::new(),
16295 },
16296 sem: Arc::new(Semaphore::new(1)),
16297 pause: Pause::new(),
16298 interrupt: Pause::new(),
16299 };
16300
16301 let shell = runner.state.config.shell();
16302 runner
16303 .stop_reviewing("round budget spent", &shell, &repo)
16304 .await
16305 .expect("stop_reviewing");
16306
16307 let last = runner.state.reviews.last().expect("round record");
16308 assert_eq!(
16309 last.e2e_status(),
16310 E2eStatus::ResourceBlocked,
16311 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16312 failed: {last:?}"
16313 );
16314 assert_eq!(
16315 last.verified_head.as_deref(),
16316 Some(head.as_str()),
16317 "which commit this attempt targeted is known even though nothing finished checking \
16318 it"
16319 );
16320 let first_attempt_at = last
16321 .verified_at
16322 .expect("when this attempt ran is known too");
16323 assert_ne!(
16324 runner.state.status,
16325 RunStatus::Blocked,
16326 "contention is evidence about the machine, not the patch — it must not settle the \
16327 run as blocked: {:?}",
16328 runner.state.status
16329 );
16330 assert!(
16331 !runner
16332 .state
16333 .events
16334 .iter()
16335 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16336 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16337 runner.state.events
16338 );
16339
16340 runner
16345 .stop_reviewing("round budget spent", &shell, &repo)
16346 .await
16347 .expect("stop_reviewing retry");
16348 assert_eq!(
16349 runner.state.reviews.len(),
16350 1,
16351 "no new round was started: {:?}",
16352 runner.state.reviews
16353 );
16354 let last = runner.state.reviews.last().expect("round record");
16355 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16356 assert!(
16357 last.verified_at.expect("still known") > first_attempt_at,
16358 "a second reentry must be a fresh attempt, not a stale copy of the first"
16359 );
16360 assert_ne!(runner.state.status, RunStatus::Blocked);
16361
16362 held.release();
16363 }
16364
16365 #[tokio::test]
16377 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16378 crate::run::pin_test_home();
16379 let home = crate::run::home();
16380
16381 let tmp = tempfile::tempdir().expect("tempdir");
16382 let repo = tmp.path().join("repo");
16383 std::fs::create_dir_all(&repo).unwrap();
16384 init_repo(&repo);
16385 let head = crate::git::rev_parse(&repo, "HEAD")
16386 .await
16387 .expect("rev-parse");
16388 let cache_dir = tmp.path().join("target");
16389
16390 let mut config = Config::default();
16391 config.verify.e2e = vec![format!(
16392 "CARGO_TARGET_DIR='{}' test -f README.md",
16393 cache_dir.display()
16394 )];
16395 config.graph.review_rounds = 1;
16396 config.graph.timeout_verify = Some(2);
16397 config.graph.timeout_resource_wait = 2;
16398
16399 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16400 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16401 .expect("no io error acquiring directly")
16402 {
16403 crate::cache::AcquireOutcome::Acquired(g) => g,
16404 crate::cache::AcquireOutcome::Busy(b) => {
16405 panic!("expected the direct acquire to win the lease first: {b:?}")
16406 }
16407 };
16408
16409 let mut state = RunState::new(
16410 repo.clone(),
16411 "main".to_owned(),
16412 head.clone(),
16413 "task".to_owned(),
16414 config,
16415 );
16416 state.candidates = vec![Candidate {
16417 index: 0,
16418 label: 'A',
16419 agent: "alpha".to_owned(),
16420 branch: "does-not-exist".to_owned(),
16421 worktree: repo.clone(),
16422 summary: String::new(),
16423 stat: String::new(),
16424 files: 0,
16425 commits: 0,
16426 empty: false,
16427 failed: None,
16428 verified_noop: None,
16429 duration_ms: 0,
16430 folded: false,
16431 }];
16432 state.tally = Some(Tally {
16433 first_choice: BTreeMap::from([('A', 1)]),
16434 borda: BTreeMap::new(),
16435 winner: 'A',
16436 rankings: 1,
16437 unanimous_initial: true,
16438 deliberated: false,
16439 changed_votes: 0,
16440 unanimous_final: true,
16441 tie_break: None,
16442 judges: 0,
16443 present: 0,
16444 quorum: 0,
16445 met_quorum: true,
16446 uncontested: Some("only candidate A produced a change".to_owned()),
16447 });
16448 state.reviews = vec![ReviewRound {
16452 round: 1,
16453 head: head.clone(),
16454 verified_head: Some(head.clone()),
16455 verified_at: Some(jiff::Timestamp::now()),
16456 reviews: Vec::new(),
16457 e2e: vec![CommandOutcome {
16458 command: format!(
16459 "CARGO_TARGET_DIR='{}' test -f README.md",
16460 cache_dir.display()
16461 ),
16462 code: None,
16463 output_tail: "waiting for the shared build cache".to_owned(),
16464 duration_ms: 0,
16465 resource_blocked: true,
16466 }],
16467 fix: None,
16468 blocking: 1,
16469 answered: 1,
16470 expected: 1,
16471 clean: false,
16472 verify_retried: false,
16473 e2e_deferred: false,
16474 e2e_defer_reason: None,
16475 progressed: false,
16476 vote_split: false,
16477 reconsideration: Vec::new(),
16478 verdict: None,
16479 }];
16480
16481 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16482 let mut runner = Runner {
16483 state,
16484 roles: ResolvedRoles {
16485 implementers: Vec::new(),
16486 judges: Vec::new(),
16487 reviewers: Vec::new(),
16488 fixer: None,
16489 conductor: conductor(),
16490 implementer_roster: Vec::new(),
16491 judge_roster: Vec::new(),
16492 reviewer_roster: Vec::new(),
16493 },
16494 sem: Arc::new(Semaphore::new(1)),
16495 pause: Pause::new(),
16496 interrupt: Pause::new(),
16497 };
16498
16499 runner.review_loop().await.expect("review_loop");
16504
16505 assert_eq!(
16506 runner.state.reviews.len(),
16507 1,
16508 "no new round was started on top of the unresolved one: {:?}",
16509 runner.state.reviews
16510 );
16511 let last = &runner.state.reviews[0];
16512 assert_eq!(
16513 last.e2e_status(),
16514 E2eStatus::ResourceBlocked,
16515 "still contended: {last:?}"
16516 );
16517 assert!(
16518 last.verified_at.expect("still known") > first_attempt_at,
16519 "review_loop must have actually retried the check, not left it exactly as found"
16520 );
16521 assert_ne!(
16522 runner.state.status,
16523 RunStatus::Blocked,
16524 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16525 runner.state.status
16526 );
16527
16528 held.release();
16529 }
16530
16531 #[tokio::test]
16532 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16533 crate::run::pin_test_home();
16534 for status in [RunStatus::Blocked, RunStatus::Gating] {
16535 let mut runner = runner_at(status);
16536 runner.state.config.graph.land = true;
16537 runner.state.gate_ran = true;
16538 runner.state.merge = Some(MergeOutcome {
16539 mode: MergeMode::Pr,
16540 ok: false,
16541 detail: "the fixer produced no commit".into(),
16542 empty: false,
16543 });
16544 let url = "https://example.invalid/o/r/pull/1";
16545 runner.state.pr = Some(crate::run::PrRecord {
16546 url: url.into(),
16547 number: 1,
16548 state: "open".into(),
16549 checks: "red".into(),
16550 round: 1,
16551 rounds: 1,
16552 red_at_merge: Vec::new(),
16553 });
16554 runner.state.save().unwrap();
16555 runner.execute_graph().await.unwrap();
16559 assert_eq!(runner.state.status, RunStatus::Blocked);
16560 assert!(runner.state.land_sync_pending);
16561 assert!(runner.state.events.iter().any(|event|
16562 event.node == "land" && event.message == format!("watching {url}")));
16563 assert_eq!(
16564 runner.state.merge.as_ref().unwrap().detail,
16565 "the fixer produced no commit"
16566 );
16567 }
16568 }
16569
16570 #[tokio::test]
16571 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16572 crate::run::pin_test_home();
16573 let tmp = tempfile::tempdir().expect("tempdir");
16574 let repo = tmp.path().join("repo");
16575 std::fs::create_dir_all(&repo).unwrap();
16576 init_repo(&repo);
16577
16578 let mut config = Config::default();
16579 config.merge.mode = MergeMode::Pr;
16580 config.graph.land = true;
16581 config.graph.land_approval = false;
16582
16583 let mut state = RunState::new(
16584 repo.clone(),
16585 "main".to_owned(),
16586 "deadbeef".to_owned(),
16587 "task".to_owned(),
16588 config,
16589 );
16590 state.candidates = vec![Candidate {
16591 index: 0,
16592 label: 'A',
16593 agent: "alpha".to_owned(),
16594 branch: "does-not-exist".to_owned(),
16595 worktree: repo.clone(),
16596 summary: String::new(),
16597 stat: String::new(),
16598 files: 0,
16599 commits: 0,
16600 empty: false,
16601 failed: None,
16602 verified_noop: None,
16603 duration_ms: 0,
16604 folded: false,
16605 }];
16606 state.tally = Some(Tally {
16607 first_choice: BTreeMap::from([('A', 1)]),
16608 borda: BTreeMap::new(),
16609 winner: 'A',
16610 rankings: 1,
16611 unanimous_initial: true,
16612 deliberated: false,
16613 changed_votes: 0,
16614 unanimous_final: true,
16615 tie_break: None,
16616 judges: 0,
16617 present: 0,
16618 quorum: 0,
16619 met_quorum: true,
16620 uncontested: Some("only candidate A produced a change".to_owned()),
16621 });
16622 state.reviews = vec![ReviewRound {
16623 round: 1,
16624 head: "deadbeef".to_owned(),
16625 verified_head: None,
16626 verified_at: None,
16627 reviews: Vec::new(),
16628 e2e: Vec::new(),
16629 fix: None,
16630 blocking: 0,
16631 answered: 0,
16632 expected: 0,
16633 clean: true,
16634 verify_retried: false,
16635 e2e_deferred: false,
16636 e2e_defer_reason: None,
16637 progressed: false,
16638 vote_split: false,
16639 reconsideration: Vec::new(),
16640 verdict: None,
16641 }];
16642 state.gate = vec![CommandOutcome {
16643 command: "test".to_owned(),
16644 code: Some(0),
16645 output_tail: String::new(),
16646 duration_ms: 0,
16647 resource_blocked: false,
16648 }];
16649 state.gate_ran = true;
16650 state.status = RunStatus::Landing;
16654 state.merge = Some(MergeOutcome {
16655 mode: MergeMode::Pr,
16656 ok: true,
16657 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16658 empty: false,
16659 });
16660
16661 ask_test_home();
16665 state.save().expect("save landing run");
16668 let store = ask::Questions::open();
16669 let q = ask_open_question(&store, &state.id);
16670
16671 let mut runner = Runner {
16672 state,
16673 roles: ResolvedRoles {
16674 implementers: Vec::new(),
16675 judges: Vec::new(),
16676 reviewers: Vec::new(),
16677 fixer: None,
16678 conductor: conductor(),
16679 implementer_roster: Vec::new(),
16680 judge_roster: Vec::new(),
16681 reviewer_roster: Vec::new(),
16682 },
16683 sem: Arc::new(Semaphore::new(1)),
16684 pause: Pause::new(),
16685 interrupt: Pause::new(),
16686 };
16687
16688 runner.execute().await.expect("execute");
16693
16694 assert_eq!(
16695 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16696 Some("https://example.invalid/x/y/pull/1"),
16697 "reentry must not push again or open a second pull request over the \
16698 one `land` is already watching"
16699 );
16700 assert_ne!(
16701 runner.state.status,
16702 RunStatus::Landing,
16703 "land could not actually reach the fake pull request, so it must \
16704 have given up rather than left the run silently parked forever"
16705 );
16706 assert_eq!(runner.state.status, RunStatus::Blocked);
16710 assert!(
16711 store.get(&q.id).unwrap().status.open(),
16712 "Blocked is still alive; settle_questions must have been a no-op here"
16713 );
16714 }
16715
16716 fn state_with_round(round: ReviewRound) -> RunState {
16717 let mut s = RunState::new(
16718 PathBuf::from("/repo"),
16719 "main".to_owned(),
16720 "abc1234".to_owned(),
16721 "add retries".to_owned(),
16722 Config::default(),
16723 );
16724 s.reviews = vec![round];
16725 s
16726 }
16727
16728 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16729 crate::verdict::Finding {
16730 id: id.to_owned(),
16731 severity,
16732 file: None,
16733 line: None,
16734 title: title.to_owned(),
16735 detail: String::new(),
16736 }
16737 }
16738
16739 #[test]
16740 fn pr_body_names_open_findings_and_declined_ones() {
16741 let round = ReviewRound {
16742 round: 2,
16743 head: "deadbee".to_owned(),
16744 verified_head: None,
16745 verified_at: None,
16746 reviews: vec![ReviewRecord {
16747 attempts: 0,
16748 reviewer: 1,
16749 agent: "alpha".to_owned(),
16750 summary: String::new(),
16751 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16752 vote: None,
16753 failed: None,
16754 duration_ms: 0,
16755 }],
16756 e2e: vec![CommandOutcome {
16757 command: "cargo test".to_owned(),
16758 code: Some(0),
16759 output_tail: String::new(),
16760 duration_ms: 0,
16761 resource_blocked: false,
16762 }],
16763 verify_retried: false,
16764 e2e_deferred: false,
16765 e2e_defer_reason: None,
16766 fix: Some(FixRecord {
16767 agent: "alpha".to_owned(),
16768 addressed: Vec::new(),
16769 rejected: vec![crate::verdict::Rejection {
16770 id: "R1-1-1".to_owned(),
16771 why: "not reachable from any caller".to_owned(),
16772 }],
16773 notes: String::new(),
16774 committed: true,
16775 failed: None,
16776 duration_ms: 0,
16777 continuation: None,
16778 }),
16779 blocking: 0,
16780 answered: 1,
16781 expected: 1,
16782 clean: false,
16783 progressed: true,
16784 vote_split: false,
16785 reconsideration: Vec::new(),
16786 verdict: None,
16787 };
16788 let state = state_with_round(round);
16789 let body = pr_message(&state, 'A').body;
16790
16791 assert!(body.contains("add retries"), "the task must still be there");
16792 assert!(body.contains("R2-1-1"), "{body}");
16793 assert!(!body.contains("unused import"), "{body}");
16794 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16795 assert!(
16796 !body.contains("not reachable from any caller"),
16797 "the reason it was declined: {body}"
16798 );
16799 }
16800
16801 #[test]
16802 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16803 let round = ReviewRound {
16804 round: 1,
16805 head: "deadbee".to_owned(),
16806 verified_head: None,
16807 verified_at: None,
16808 reviews: vec![ReviewRecord {
16809 attempts: 0,
16810 reviewer: 1,
16811 agent: "alpha".to_owned(),
16812 summary: String::new(),
16813 findings: Vec::new(),
16814 vote: None,
16815 failed: None,
16816 duration_ms: 0,
16817 }],
16818 e2e: Vec::new(),
16819 verify_retried: false,
16820 e2e_deferred: false,
16821 e2e_defer_reason: None,
16822 fix: None,
16823 blocking: 0,
16824 answered: 1,
16825 expected: 1,
16826 clean: true,
16827 progressed: false,
16828 vote_split: false,
16829 reconsideration: Vec::new(),
16830 verdict: None,
16831 };
16832 let state = state_with_round(round);
16833 let body = pr_message(&state, 'A').body;
16834 assert!(!body.contains("Open review findings"), "{body}");
16835 assert!(!body.contains("Declined"), "{body}");
16836 }
16837
16838 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16839 let mut state = RunState::new(
16840 PathBuf::from("/repo"),
16841 "main".to_owned(),
16842 "abc1234".to_owned(),
16843 instruction.to_owned(),
16844 Config::default(),
16845 );
16846 state.candidates.push(Candidate {
16847 index: 0,
16848 label: 'A',
16849 agent: "alpha".to_owned(),
16850 branch: "magi/x/A".to_owned(),
16851 worktree: PathBuf::from("/wt"),
16852 summary: summary.to_owned(),
16853 stat: String::new(),
16854 files: 1,
16855 commits: 1,
16856 empty: false,
16857 failed: None,
16858 verified_noop: None,
16859 folded: false,
16860 duration_ms: 0,
16861 });
16862 state
16863 }
16864
16865 fn review_state(subjects: &[&str]) -> RunState {
16866 let mut state = state_with_summary(
16867 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16868 "",
16869 );
16870 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16871 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16872 state
16873 }
16874
16875 #[tokio::test]
16878 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16879 ask_test_home();
16880 let tmp = tempfile::tempdir().unwrap();
16881 let repo = tmp.path().join("repo");
16882 std::fs::create_dir_all(&repo).unwrap();
16883 init_repo(&repo);
16884 let origin = tmp.path().join("origin.git");
16885 let g = |dir: &Path, args: &[&str]| {
16886 let out = std::process::Command::new("git")
16887 .args(args)
16888 .current_dir(dir)
16889 .quiet()
16890 .output()
16891 .expect("spawn git");
16892 assert!(
16893 out.status.success(),
16894 "git {args:?}: {}",
16895 String::from_utf8_lossy(&out.stderr)
16896 );
16897 };
16898 g(
16899 tmp.path(),
16900 &[
16901 "clone",
16902 "--bare",
16903 "-q",
16904 repo.to_str().unwrap(),
16905 origin.to_str().unwrap(),
16906 ],
16907 );
16908 g(
16909 &repo,
16910 &["remote", "add", "origin", origin.to_str().unwrap()],
16911 );
16912 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16913
16914 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16916 g(&repo, &["add", "-A"]);
16917 g(
16918 &repo,
16919 &["commit", "-q", "-m", "chore(deps): update a crate"],
16920 );
16921 g(&repo, &["push", "-q", "origin", "main"]);
16922 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16923 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16924 g(&repo, &["add", "-A"]);
16925 g(
16926 &repo,
16927 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16928 );
16929 g(&repo, &["checkout", "-q", "main"]);
16930
16931 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16932 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16933 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16935 assert_eq!(fallback, c1);
16936
16937 let mut state = review_state(&[
16938 "chore(deps): update a crate",
16939 "fix(daemon): apply a chosen action",
16940 ]);
16941 state.repo = repo.clone();
16942 state.base_branch = "main".to_owned();
16943 state.base_commit = c1;
16944 refresh_reviewed_commits(&mut state, "feat/own").await;
16945 assert_eq!(
16946 state.reviewed_commits,
16947 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16948 );
16949 assert_eq!(
16950 review_title(&state).as_deref(),
16951 Some("fix(daemon): apply a chosen action")
16952 );
16953 assert_eq!(
16954 leaked_subjects(&state, "feat/own").await,
16955 Some(vec!["chore(deps): update a crate".to_owned()])
16956 );
16957 state.config.merge.remote = "nowhere".to_owned();
16960 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16961 }
16962
16963 struct SummaryFixture {
16967 _tmp: tempfile::TempDir,
16968 repo: PathBuf,
16969 reply: PathBuf,
16970 runner: Runner,
16971 }
16972
16973 fn summary_fixture() -> SummaryFixture {
16974 ask_test_home();
16975 let tmp = tempfile::tempdir().unwrap();
16976 let repo = tmp.path().join("repo");
16977 std::fs::create_dir_all(&repo).unwrap();
16978 init_repo(&repo);
16979 let origin = tmp.path().join("origin.git");
16980 let g = |args: &[&str]| {
16981 let out = std::process::Command::new("git")
16982 .args(args)
16983 .current_dir(&repo)
16984 .quiet()
16985 .output()
16986 .expect("spawn git");
16987 assert!(
16988 out.status.success(),
16989 "git {args:?}: {}",
16990 String::from_utf8_lossy(&out.stderr)
16991 );
16992 };
16993 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16994 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16995 g(&["push", "-q", "origin", "main"]);
16996 g(&["checkout", "-q", "-b", "feat/own"]);
16997 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16998 g(&["add", "-A"]);
16999 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
17000 let reply = tmp.path().join("reply.txt");
17001 std::fs::write(
17002 &reply,
17003 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
17004 )
17005 .unwrap();
17006 let mut runner = runner_at(RunStatus::Gating);
17007 let mut state = review_state(&["feat(own): add the own file"]);
17008 state.repo = repo.clone();
17009 state.base_branch = "main".to_owned();
17010 state.candidates[0].branch = "feat/own".to_owned();
17011 state.candidates[0].worktree = repo.clone();
17012 let mut author = spec("alpha");
17013 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17014 author
17015 .env
17016 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17017 state.config.agents = vec![author];
17018 runner.state = state;
17019 SummaryFixture {
17020 _tmp: tmp,
17021 repo,
17022 reply,
17023 runner,
17024 }
17025 }
17026
17027 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17028 let winner = f.runner.state.candidates[0].clone();
17029 f.runner
17030 .guarded_pr_message(&winner, None, true)
17031 .await
17032 .unwrap()
17033 .expect("an English description never parks")
17034 }
17035
17036 fn clean_round_with_minor_findings() -> ReviewRound {
17037 let mut round = passing_round(2, None);
17038 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17039 round.reviews = vec![ReviewRecord {
17040 attempts: 0,
17041 reviewer: 1,
17042 agent: "alpha".into(),
17043 summary: "内部レビュー".into(),
17044 findings: vec![
17045 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17046 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17047 ],
17048 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17049 failed: None,
17050 duration_ms: 0,
17051 }];
17052 round
17053 }
17054
17055 #[tokio::test]
17056 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17057 for review_only in [false, true] {
17058 let mut f = summary_fixture();
17059 if !review_only {
17060 f.runner.state.candidates[0].agent = "alpha".into();
17061 }
17062 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17063 assert!(f.runner.state.reviews[0].clean);
17064 assert!(f.runner.state.reviews[0].fix.is_none());
17065 assert!(f.runner.state.open_findings().is_empty());
17066 assert_eq!(
17067 crate::pr_outcome::required_finding_ids(&f.runner.state),
17068 ["R2-1-1", "R2-1-2"]
17069 );
17070 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17071 let evidence = crate::pr_outcome::summary_evidence(
17072 &f.runner.state,
17073 crate::pr_outcome::FinalHead::At(&head),
17074 );
17075 assert!(
17076 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17077 );
17078 let omitted = posted(&mut f).await;
17080 assert!(omitted.body.contains("No description"));
17081 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17082 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17083 assert!(f.runner.state.change_summary.is_none());
17084 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17085 let described = posted(&mut f).await;
17086 assert!(described.body.contains("improve documentation for callers"));
17087 assert!(described.body.contains("remaining edge case"));
17088 assert!(described.body.contains("## Open review findings"));
17089 assert!(!described.body.contains("未修正"));
17090 }
17091 }
17092
17093 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17094 let mut f = summary_fixture();
17095 if !review_only {
17096 f.runner.state.candidates[0].agent = "alpha".into();
17097 }
17098 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17099 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17100 fixed
17101 .fix
17102 .as_mut()
17103 .unwrap()
17104 .rejected
17105 .push(crate::verdict::Rejection {
17106 id: "R1-1-1".into(),
17107 why: "呼び出されないコードのため辞退".into(),
17108 });
17109 let mut clean = clean_round_with_minor_findings();
17110 clean.verified_head = Some(head.clone());
17111 f.runner.state.reviews = vec![fixed, clean];
17112 ok_gate(&mut f.runner.state, Some(&head));
17113 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17114 f.runner.state.config.agents.push(scripted("beta", reply));
17115 f.runner.state.config.roles.language_judge =
17116 Some(crate::config::AgentChoice::One("beta".into()));
17117 f
17118 }
17119
17120 #[tokio::test]
17121 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17122 for review_only in [false, true] {
17123 let mut f = rewrite_fixture_with_findings(review_only,
17124 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17125 let m = posted(&mut f).await;
17126 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17127 let expected =
17128 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17129 assert!(m.body.contains(&expected));
17130 assert!(m.body.contains("## Review rounds"));
17131 assert!(
17132 m.body
17133 .contains("Gate (`verify.gate`): passed on the final commit")
17134 );
17135 assert!(
17136 m.body
17137 .contains("Tests (`verify.e2e`): passed on the final commit")
17138 );
17139 let c = f.runner.state.change_summary.as_ref().unwrap();
17140 assert!(
17141 m.body
17142 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17143 );
17144 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17145 assert!(crate::pr_outcome::covers_required_findings(
17146 &f.runner.state,
17147 &m.body
17148 ));
17149 assert!(m.body.contains("documentation remains incomplete"));
17150 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17151 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17152 assert!(f.runner.state.github_text.is_none());
17153 }
17154 }
17155
17156 #[tokio::test]
17157 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17158 for review_only in [false, true] {
17159 for reply in [
17160 "TITLE: fix: public description\nA short English description.",
17161 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17162 ] {
17163 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17164 let winner = f.runner.state.candidates[0].clone();
17165 let posted = f
17166 .runner
17167 .guarded_pr_message(&winner, None, true)
17168 .await
17169 .unwrap();
17170 assert!(posted.is_none());
17171 assert!(f.runner.state.parked);
17172 let withheld = f.runner.state.github_text.as_ref().unwrap();
17173 assert!(
17174 withheld.rewrite.is_none(),
17175 "an incomplete rewrite must not be selectable"
17176 );
17177 let q = ask::Questions::open()
17178 .get(withheld.question.as_ref().unwrap())
17179 .unwrap();
17180 assert!(
17181 !q.choices
17182 .contains(&crate::github_text::USE_REWRITE.to_owned())
17183 );
17184 assert!(
17185 q.detail.contains("R1-1-1")
17186 && q.detail.contains("R2-1-1")
17187 && q.detail.contains("R2-1-2")
17188 );
17189 }
17190 }
17191 }
17192
17193 #[tokio::test]
17194 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17195 for review_only in [false, true] {
17196 let mut f = summary_fixture();
17197 if !review_only {
17198 f.runner.state.candidates[0].agent = "alpha".into();
17199 }
17200 f.runner.state.instruction = "内部タスクの説明".into();
17201 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17202 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17203 f.runner.state.reviews = vec![
17204 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17205 round_with_notes(2, Some("二回目の内部修正報告です。")),
17206 passing_round(3, Some(&head)),
17207 ];
17208 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17209 ok_gate(&mut f.runner.state, Some(&head));
17210 let captured = f.repo.join("public-prompt.txt");
17211 let author = &mut f.runner.state.config.agents[0];
17212 author.command = vec![
17213 "sh".into(),
17214 "-c".into(),
17215 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17216 ];
17217 author.env.insert(
17218 "PROMPT_FILE".into(),
17219 captured.to_string_lossy().into_owned(),
17220 );
17221 let m = posted(&mut f).await;
17222 assert!(m.body.contains("Adds `own.txt`"));
17223 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17224 assert!(m.body.contains("R1-1-1"));
17225 assert!(
17226 m.body
17227 .contains("Gate (`verify.gate`): passed on the final commit")
17228 );
17229 assert!(
17230 m.body
17231 .contains("Tests (`verify.e2e`): passed on the final commit")
17232 );
17233 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17234 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17235 let prompt = std::fs::read_to_string(captured).unwrap();
17236 assert!(prompt.contains(&format!(
17237 "git diff {} {head}",
17238 f.runner.state.change_summary.as_ref().unwrap().base
17239 )));
17240 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17241 assert_eq!(
17242 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17243 "二回目の内部修正報告です。"
17244 );
17245 }
17246 }
17247
17248 #[tokio::test]
17249 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17250 for review_only in [false, true] {
17251 let mut f = summary_fixture();
17252 if !review_only {
17253 f.runner.state.candidates[0].agent = "alpha".into();
17254 }
17255 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17256 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17257 std::fs::write(&f.reply, "no usable description").unwrap();
17258 let failed = posted(&mut f).await;
17259 assert!(
17260 failed
17261 .body
17262 .contains("No description of what this change does could be generated")
17263 );
17264 assert!(failed.body.is_ascii());
17265 assert!(f.runner.state.github_text.is_none());
17266 assert!(f.runner.state.change_summary.is_none());
17267 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17268 let evidence = crate::pr_outcome::summary_evidence(
17269 &f.runner.state,
17270 crate::pr_outcome::FinalHead::At(&head),
17271 );
17272 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17273 assert!(
17274 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17275 .exists()
17276 );
17277 f.runner.state =
17278 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17279 std::fs::write(
17280 &f.reply,
17281 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17282 )
17283 .unwrap();
17284 let retried = posted(&mut f).await;
17285 assert!(retried.body.contains("Adds the committed own file."));
17286 assert_eq!(
17287 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17288 "日本語の修正報告"
17289 );
17290 }
17291 }
17292
17293 #[tokio::test]
17294 async fn same_head_verification_and_findings_invalidate_public_prose() {
17295 let mut f = summary_fixture();
17296 posted(&mut f).await;
17297 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17298 let original = f
17299 .runner
17300 .state
17301 .change_summary
17302 .as_ref()
17303 .unwrap()
17304 .evidence
17305 .clone();
17306 ok_gate(&mut f.runner.state, Some(&head));
17307 std::fs::write(
17308 &f.reply,
17309 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17310 )
17311 .unwrap();
17312 let verified = posted(&mut f).await;
17313 assert!(verified.body.contains("now verified"));
17314 assert_ne!(
17315 f.runner.state.change_summary.as_ref().unwrap().evidence,
17316 original
17317 );
17318 let mut round = round_with_notes(1, Some("内部報告"));
17319 round.reviews.push(ReviewRecord {
17320 attempts: 0,
17321 reviewer: 1,
17322 agent: "alpha".into(),
17323 summary: "内部レビュー".into(),
17324 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17325 vote: None,
17326 failed: None,
17327 duration_ms: 0,
17328 });
17329 round
17330 .fix
17331 .as_mut()
17332 .unwrap()
17333 .rejected
17334 .push(crate::verdict::Rejection {
17335 id: "R1-1-1".into(),
17336 why: "呼び出し元から到達できない".into(),
17337 });
17338 f.runner.state.reviews = vec![round];
17339 let omitted = posted(&mut f).await;
17341 assert!(omitted.body.contains("No description"));
17342 assert!(omitted.body.contains("R1-1-1"));
17343 assert!(!omitted.body.contains("到達"));
17344 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17345 let described = posted(&mut f).await;
17346 assert!(described.body.contains("unreachable from callers"));
17347 assert!(described.body.contains("R1-1-1"));
17348 assert!(!described.body.contains("到達"));
17349 let evidence = f
17350 .runner
17351 .state
17352 .change_summary
17353 .as_ref()
17354 .unwrap()
17355 .evidence
17356 .clone();
17357 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17358 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17359 assert!(posted(&mut f).await.body.contains("rationale updated"));
17360 assert_ne!(
17361 f.runner.state.change_summary.as_ref().unwrap().evidence,
17362 evidence
17363 );
17364 f.runner
17366 .state
17367 .change_summary
17368 .as_mut()
17369 .unwrap()
17370 .evidence
17371 .clear();
17372 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17373 assert!(posted(&mut f).await.body.contains("current specification"));
17374 }
17375
17376 #[tokio::test]
17377 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17378 let mut f = summary_fixture();
17379 let m = posted(&mut f).await;
17380 assert_eq!(m.title, "feat(own): add the own file");
17381 assert!(
17382 m.body
17383 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17384 "{}",
17385 m.body
17386 );
17387 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17388 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17389 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17390 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17391 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17392 let kept = f.runner.state.change_summary.as_ref().unwrap();
17393 assert_eq!(
17394 (kept.head.as_str(), kept.base.as_str()),
17395 (head.as_str(), base.as_str())
17396 );
17397 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17398 }
17399
17400 #[tokio::test]
17401 async fn review_only_summary_receives_the_configured_github_language() {
17402 let mut f = summary_fixture();
17403 f.runner.state.config.graph.github_language = "ja".into();
17404 let captured = f.repo.join("summary-prompt.txt");
17405 let author = &mut f.runner.state.config.agents[0];
17406 author.command = vec![
17407 "sh".into(),
17408 "-c".into(),
17409 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17410 ];
17411 author.env.insert(
17412 "PROMPT_FILE".into(),
17413 captured.to_string_lossy().into_owned(),
17414 );
17415 std::fs::write(
17416 &f.reply,
17417 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17418 )
17419 .unwrap();
17420 let m = posted(&mut f).await;
17421 let prompt = std::fs::read_to_string(captured).unwrap();
17422 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17423 assert!(!prompt.contains("in English"));
17424 assert_eq!(m.title, "feat(own): ファイルを追加");
17425 assert!(m.body.contains("own.txt を追加します。"));
17426 }
17427
17428 #[tokio::test]
17429 async fn review_fixes_come_after_the_whole_change_summary() {
17430 let mut f = summary_fixture();
17431 f.runner.state.reviews = vec![
17432 round_with_notes(1, Some("handled the PATH case")),
17433 round_with_notes(2, None),
17434 ];
17435 let m = posted(&mut f).await;
17436 let summary = m.body.find("## Summary").expect("summary");
17437 let fixes = m.body.find("## Review rounds").expect("review rounds");
17438 assert!(summary < fixes, "{}", m.body);
17439 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17440 assert!(m.body.contains("## Final verification"), "{}", m.body);
17441 }
17442
17443 #[tokio::test]
17444 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17445 let mut f = summary_fixture();
17446 let first = posted(&mut f).await;
17447 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17448 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17450 let again = posted(&mut f).await;
17451 assert_eq!((again.title, again.body), (first.title, first.body));
17452
17453 let g = |args: &[&str]| {
17454 let out = std::process::Command::new("git")
17455 .args(args)
17456 .current_dir(&f.repo)
17457 .quiet()
17458 .output()
17459 .expect("spawn git");
17460 assert!(out.status.success(), "git {args:?}");
17461 };
17462 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17463 g(&["add", "-A"]);
17464 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17465 let second = posted(&mut f).await;
17466 let kept = f.runner.state.change_summary.clone().unwrap();
17467 assert_ne!(kept.head, first_head);
17468 assert!(second.body.contains("Not read yet."), "{}", second.body);
17469 assert!(!second.body.contains("so the first summary is read"));
17470 assert_eq!(second.title, "feat(own): other");
17471 }
17472
17473 #[tokio::test]
17474 async fn an_unavailable_summary_is_stated_and_never_invented() {
17475 let mut f = summary_fixture();
17476 std::fs::write(&f.reply, "no title line here").unwrap();
17477 let m = posted(&mut f).await;
17478 assert!(
17479 m.body
17480 .contains("No description of what this change does could be generated"),
17481 "{}",
17482 m.body
17483 );
17484 assert!(m.body.contains("internal reports remain in local records"));
17485 assert!(f.runner.state.change_summary.is_none());
17486 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17487 let winner = f.runner.state.candidates[0].clone();
17489 let preview = f
17490 .runner
17491 .guarded_pr_message(&winner, None, false)
17492 .await
17493 .unwrap()
17494 .unwrap();
17495 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17496 }
17497
17498 #[test]
17499 fn pr_message_review_single_commit_uses_its_subject() {
17500 let state = review_state(&["feat(nats): per-role user"]);
17501 let m = pr_message(&state, 'A');
17502 assert_eq!(m.title, "feat(nats): per-role user");
17503 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17504 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17505 }
17506
17507 #[test]
17508 fn pr_message_review_multi_commit_takes_the_oldest() {
17509 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17510 let m = pr_message(&state, 'A');
17511 assert_eq!(m.title, "feat: the change");
17512 for s in ["feat: the change", "fix: typo", "fix: again"] {
17513 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17514 }
17515 }
17516
17517 #[test]
17518 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17519 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17520 let mut state = review_state(&[first, "fix: later fixup"]);
17521 state.candidates[0].branch = "機能/ログイン".to_owned();
17522 let m = pr_message(&state, 'A');
17523 assert!(
17524 m.title.starts_with("chore: land candidate A of run"),
17525 "{}",
17526 m.title
17527 );
17528 }
17529 }
17530
17531 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17532 BranchFacts {
17533 commits: commits
17534 .iter()
17535 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17536 .collect(),
17537 stat: stat.to_owned(),
17538 }
17539 }
17540
17541 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17542 let mut r = review_round(true, 0, 1, 1, true, true);
17543 r.round = round;
17544 r.fix = notes.map(|n| FixRecord {
17545 agent: "fixer".to_owned(),
17546 addressed: Vec::new(),
17547 rejected: Vec::new(),
17548 notes: n.to_owned(),
17549 committed: true,
17550 failed: None,
17551 duration_ms: 0,
17552 continuation: None,
17553 });
17554 r
17555 }
17556
17557 #[test]
17558 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17559 let state = review_state(&["ignored"]);
17560 let f = facts(
17561 &[
17562 (
17563 "fix(login): resolve PATH on macOS",
17564 "Login shells skip rc files.",
17565 ),
17566 ("fix: address review", ""),
17567 ],
17568 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17569 );
17570 let m = pr_message_with(&state, 'A', Some(&f));
17571 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17572 assert!(
17573 m.body
17574 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17575 "{}",
17576 m.body
17577 );
17578 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17579 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17580 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17581 for banned in [
17582 "Review the work already",
17583 "no task statement",
17584 "Original task",
17585 ] {
17586 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17587 }
17588 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17589 }
17590
17591 #[test]
17592 fn pr_message_review_truncates_a_huge_first_commit_body() {
17593 let state = review_state(&["ignored"]);
17594 let f = facts(
17595 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17596 "s",
17597 );
17598 let m = pr_message_with(&state, 'A', Some(&f));
17599 assert!(m.body.len() < 30_000, "{}", m.body.len());
17600 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17601 assert!(
17602 m.body.contains("- ... 1 more commit(s)"),
17603 "{}",
17604 m.body.len()
17605 );
17606 assert!(m.body.ends_with("magi:candidate-a\n"));
17607 }
17608
17609 #[test]
17610 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17611 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17612 assert_eq!(m.title, "feat: x");
17613 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17614 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17615 assert!(!m.body.contains("no task statement"), "{}", m.body);
17616 }
17617
17618 #[test]
17619 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17620 let mut state = review_state(&["日本語の件名"]);
17621 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17622 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17623 state.candidates[0].branch = "機能/ログイン".to_owned();
17624 assert!(
17625 pr_message(&state, 'A')
17626 .title
17627 .starts_with("chore: land candidate A")
17628 );
17629 }
17630
17631 #[test]
17632 fn pr_message_review_fixes_survive_a_clean_final_round() {
17633 let mut state = review_state(&["feat: x"]);
17634 state.reviews = vec![
17635 round_with_notes(1, Some("handled the PATH case")),
17636 round_with_notes(2, None),
17637 ];
17638 let body = pr_message(&state, 'A').body;
17639 assert!(
17640 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17641 "{body}"
17642 );
17643 assert!(body.contains("### Round 1"), "{body}");
17644
17645 state.reviews = vec![
17646 round_with_notes(1, Some("first fix")),
17647 round_with_notes(2, Some("")),
17648 round_with_notes(3, Some("second fix")),
17649 round_with_notes(4, None),
17650 ];
17651 let body = pr_message(&state, 'A').body;
17652 assert!(
17653 body.contains("### Round 1") && !body.contains("first fix"),
17654 "{body}"
17655 );
17656 assert!(
17657 body.contains("### Round 3") && !body.contains("second fix"),
17658 "{body}"
17659 );
17660 assert!(body.contains("### Round 2"), "{body}");
17661 }
17662
17663 const FINAL: &str = "abcdef1234567890";
17664
17665 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17666 let mut r = round_with_notes(round, None);
17667 r.verified_head = verified.map(str::to_owned);
17668 r
17669 }
17670
17671 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17672 state.gate = vec![CommandOutcome {
17673 command: "gate".to_owned(),
17674 code: Some(0),
17675 output_tail: String::new(),
17676 duration_ms: 0,
17677 resource_blocked: false,
17678 }];
17679 state.gate_ran = true;
17680 state.gate_head = head.map(str::to_owned);
17681 }
17682
17683 fn body_at(state: &RunState) -> String {
17684 pr_message_raw(
17685 state,
17686 'A',
17687 None,
17688 crate::pr_outcome::FinalHead::At(FINAL),
17689 ChangeNote::NotAsked,
17690 )
17691 .body
17692 }
17693
17694 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17695
17696 #[test]
17697 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17698 let mut state = review_state(&["feat: x"]);
17699 state.reviews = vec![
17700 round_with_notes(1, Some(SANDBOXED)),
17701 passing_round(2, Some(FINAL)),
17702 ];
17703 ok_gate(&mut state, Some(FINAL));
17704 let body = body_at(&state);
17705 assert!(!body.contains("Handled the PATH case."), "{body}");
17706 assert!(!body.contains("could not commit"), "{body}");
17707 assert!(!body.contains("could not run"), "{body}");
17708 assert!(
17709 body.contains("The fix is committed on the branch"),
17710 "{body}"
17711 );
17712 assert!(!body.contains("sandbox"), "{body}");
17713 assert!(
17714 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17715 "{body}"
17716 );
17717 assert!(
17718 body.contains("Gate (`verify.gate`): passed on the final commit"),
17719 "{body}"
17720 );
17721 assert!(
17723 state.reviews[0]
17724 .fix
17725 .as_ref()
17726 .unwrap()
17727 .notes
17728 .contains("could not commit")
17729 );
17730 }
17731
17732 #[test]
17733 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17734 let mut state = review_state(&["feat: x"]);
17735 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17736 r1.fix.as_mut().unwrap().committed = false;
17737 let mut r2 = passing_round(2, Some(FINAL));
17738 r2.e2e[0].code = Some(1);
17739 state.reviews = vec![r1, r2];
17740 let body = body_at(&state);
17741 assert!(!body.contains("could not commit"), "{body}");
17742 assert!(!body.contains("could not run"), "{body}");
17743 assert!(body.contains("This round produced no commit"), "{body}");
17744 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17745 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17746 assert!(!body.contains("passed"), "{body}");
17747 }
17748
17749 #[test]
17750 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17751 let mut state = review_state(&["feat: x"]);
17752 state.reviews = vec![
17753 round_with_notes(1, Some(SANDBOXED)),
17754 passing_round(2, Some("0123456789abcdef")),
17755 ];
17756 ok_gate(&mut state, Some("0123456789abcdef"));
17757 let body = body_at(&state);
17758 assert!(!body.contains("HTTP tests could not run"), "{body}");
17759 assert!(!body.contains("could not commit"), "{body}");
17761 assert!(
17762 body.contains("passed on an earlier commit (`0123456`)"),
17763 "{body}"
17764 );
17765 assert!(!body.contains("passed on the final commit"), "{body}");
17766
17767 ok_gate(&mut state, None);
17769 let body = body_at(&state);
17770 assert!(
17771 body.contains("passed on a commit that was not recorded"),
17772 "{body}"
17773 );
17774 }
17775
17776 #[test]
17777 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17778 let state = state_with_summary(
17779 "add retries to the client",
17780 "TITLE: feat: retries\n\nDid it.",
17781 );
17782 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17783 assert_eq!(m.title, "feat: retries");
17784 assert!(
17785 m.body.contains("<summary>Original task</summary>"),
17786 "{}",
17787 m.body
17788 );
17789 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17790 assert!(
17791 m.body
17792 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17793 "{}",
17794 m.body
17795 );
17796 }
17797
17798 #[test]
17799 fn pr_message_review_bounds_a_long_english_subject() {
17800 let long = format!("feat: {}", "word ".repeat(100));
17801 let m = pr_message(&review_state(&[&long]), 'A');
17802 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17803 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17804 }
17805
17806 #[test]
17807 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17808 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17809 let m = pr_message(&state, 'A');
17810 assert_eq!(m.title, "add retries");
17811 assert!(m.body.contains("<summary>Original task</summary>"));
17812 assert!(!m.body.contains("Commits under review"));
17813 assert_eq!(landing_subject_source(&state), state.instruction);
17814 }
17815
17816 #[test]
17817 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17818 let state = review_state(&["feat: the change", "fix: typo"]);
17819 let source = landing_subject_source(&state);
17820 assert_eq!(land::merge_subject("", &source), "feat: the change");
17821 assert_eq!(
17822 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17823 "feat: the change"
17824 );
17825 assert_eq!(
17827 land::merge_subject("feat: renamed by hand", &source),
17828 "feat: renamed by hand"
17829 );
17830 let blank = review_state(&["日本語"]);
17831 assert!(
17832 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17833 );
17834 }
17835
17836 #[test]
17837 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17838 let state = review_state(&["feat: the change"]);
17839 let source = landing_subject_source(&state);
17840 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17841 assert_eq!(
17842 land::merge_subject(landing_title(&state, old), &source),
17843 "feat: the change"
17844 );
17845 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17846 let task = state_with_summary("add retries", "");
17847 assert_eq!(landing_title(&task, old), old);
17848 }
17849
17850 #[test]
17851 fn pr_message_describes_the_change_not_the_task() {
17852 let state = state_with_summary(
17853 "今回やってほしいこと: results projector を直す",
17854 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17855 );
17856 let m = pr_message(&state, 'A');
17857 assert_eq!(m.title, "fix(web): batch the runs list reads");
17858 assert!(
17859 m.body.starts_with("## Summary\n\n- reads run.json once"),
17860 "{}",
17861 m.body
17862 );
17863 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17864 let task_at = m.body.find("今回やってほしいこと").unwrap();
17865 let details_at = m.body.find("<details>").unwrap();
17866 assert!(
17867 details_at < task_at,
17868 "the task lives inside <details>: {}",
17869 m.body
17870 );
17871 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17872 assert!(m.body.contains("magi:candidate-a"));
17873 }
17874
17875 #[test]
17876 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17877 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17878 let m = pr_message(&state, 'A');
17879 assert_eq!(m.title, "add retries");
17880 assert!(
17881 m.body.contains("## Summary\n\n- did some things"),
17882 "{}",
17883 m.body
17884 );
17885
17886 let none = RunState::new(
17887 PathBuf::from("/repo"),
17888 "main".to_owned(),
17889 "abc1234".to_owned(),
17890 "add retries".to_owned(),
17891 Config::default(),
17892 );
17893 let m = pr_message(&none, 'A');
17894 assert_eq!(m.title, "add retries");
17895 assert!(!m.body.contains("## Summary"), "{}", m.body);
17896 }
17897
17898 #[test]
17899 fn pr_message_refuses_the_candidate_commit_subject() {
17900 for bad in [
17901 "TITLE: magi: candidate A (uncommitted work)",
17902 "TITLE: chore: stuff (uncommitted work)",
17903 "TITLE: ",
17904 ] {
17905 let state = state_with_summary("add retries", bad);
17906 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17907 }
17908 }
17909
17910 #[test]
17911 fn pr_message_bounds_a_very_long_task_and_title() {
17912 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17913 let state = state_with_summary(&long, "- nothing");
17914 let m = pr_message(&state, 'A');
17915 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17916 assert!(!m.title.contains('\n'));
17917
17918 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17919 let m = pr_message(&state, 'A');
17920 assert!(m.title.starts_with("feat: "));
17921 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17922 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17923 }
17924
17925 fn long_title_of(instruction: &str) -> String {
17926 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17927 }
17928
17929 #[test]
17930 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17931 let first = "Make the landing path keep a readable title for long tasks";
17932 let line = format!(
17933 "{first}. {}",
17934 "And then keep going with more words ".repeat(20)
17935 );
17936 let t = long_title_of(&line);
17937 assert_eq!(t, first);
17938 assert!(!t.starts_with("chore: land"));
17939 }
17940
17941 #[test]
17942 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17943 let line = "word ".repeat(200);
17944 let t = long_title_of(&line);
17945 assert!(t.ends_with("word..."), "{t}");
17946 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17947 }
17948
17949 #[test]
17950 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17951 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17952 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17953 }
17954 }
17955
17956 #[test]
17957 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17958 let mut state = state_with_summary("再試行を追加する", "- nothing");
17959 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17960 state.config.graph.github_language = "ja".into();
17961 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17962 }
17963
17964 #[test]
17965 fn pr_message_title_limit_is_exact() {
17966 let at = "a".repeat(PR_TITLE_MAX);
17967 assert_eq!(long_title_of(&at), at);
17968 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17969 assert!(over.ends_with("..."), "{over}");
17970 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17971 }
17972
17973 #[test]
17974 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17975 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17976 let t = long_title_of(&line);
17977 assert!(t.ends_with("..."), "{t}");
17978 assert!(t.is_ascii(), "{t}");
17979 }
17980
17981 #[test]
17982 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17983 let line = format!(
17984 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17985 "plus more filler words ".repeat(20)
17986 );
17987 let t = long_title_of(&line);
17988 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17989 }
17990
17991 #[test]
17992 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17993 let line = format!("\n\n# {}", "title words ".repeat(40));
17994 let state = state_with_summary(&line, "- nothing");
17995 let m = pr_message(&state, 'A');
17996 assert!(m.title.starts_with("title words"), "{}", m.title);
17997 assert_eq!(
17998 land::merge_subject(&m.title, &landing_subject_source(&state)),
17999 m.title
18000 );
18001 assert_eq!(
18003 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
18004 "feat: renamed by hand"
18005 );
18006 }
18007
18008 #[tokio::test]
18009 async fn github_text_rewrite_is_bounded_and_falls_back() {
18010 crate::run::pin_test_home();
18011 for (reply, accepted) in [
18012 (
18013 "TITLE: fix: retries\nAdd retries for failed requests.",
18014 true,
18015 ),
18016 (
18017 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18018 false,
18019 ),
18020 (
18021 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18022 true,
18023 ),
18024 (
18025 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18026 true,
18027 ),
18028 (
18029 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18030 true,
18031 ),
18032 (
18033 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18034 false,
18035 ),
18036 ("TITLE: fix: retries\n", false),
18037 ] {
18038 let dir = tempfile::tempdir().unwrap();
18039 let mut runner = runner_at(RunStatus::Gating);
18040 runner.state = state_with_summary(
18041 "add retries",
18042 "TITLE: fix: retries\n日本語の説明を書きます。",
18043 );
18044 runner.state.repo = dir.path().to_owned();
18045 runner.state.candidates[0].worktree = dir.path().to_owned();
18046 let mut author = spec("alpha");
18047 author.command = vec![
18048 "sh".into(),
18049 "-c".into(),
18050 "printf '%s' \"$REWRITE_REPLY\"".into(),
18051 ];
18052 author.env.insert("REWRITE_REPLY".into(), reply.into());
18053 runner.state.config.agents = vec![author];
18054 seed_public_summary(&mut runner);
18055 let winner = runner.state.candidates[0].clone();
18056 let first = runner
18057 .guarded_pr_message(&winner, None, true)
18058 .await
18059 .unwrap();
18060 let message = if accepted {
18061 first.expect("a passing rewrite never asks")
18062 } else {
18063 assert!(first.is_none());
18065 assert!(runner.state.parked);
18066 let store = ask::Questions::open();
18067 let asked: Vec<_> = store
18068 .list()
18069 .into_iter()
18070 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18071 .collect();
18072 assert_eq!(asked.len(), 1);
18073 assert!(!asked[0].detail.contains("secret"));
18074 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18075 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18076 assert_eq!(
18077 asked[0]
18078 .choices
18079 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18080 rewrite.is_some()
18081 );
18082 store
18085 .update(&asked[0].id, |q| {
18086 q.abandon("test");
18087 Ok(())
18088 })
18089 .unwrap();
18090 let again = runner
18091 .guarded_pr_message(&winner, None, true)
18092 .await
18093 .unwrap()
18094 .expect("an abandoned question falls back");
18095 let n = store
18096 .list()
18097 .into_iter()
18098 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18099 .count();
18100 assert_eq!(n, 1, "asked once per rejected text");
18101 again
18102 };
18103 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18104 assert!(!message.body.contains("secret"));
18105 assert!(!message.body.contains("prefix"));
18106 assert!(!message.title.contains("secret"));
18107 assert_eq!(
18108 message.body.contains("Add retries for failed requests."),
18109 accepted
18110 );
18111 assert_eq!(
18112 runner.state.seats["github-text-rewrite"].turns, 1,
18113 "only one rewrite invocation"
18114 );
18115 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18116 if accepted {
18117 assert!(runner.state.github_text.is_none());
18118 assert!(!runner.state.parked);
18119 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18120 assert!(message.body.contains("## Final verification"));
18121 assert!(message.body.contains("Evidence: final diff"));
18122 assert!(
18123 message
18124 .body
18125 .contains(&format!("magi:run/{}", runner.state.id))
18126 );
18127 }
18128 if !accepted {
18129 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18130 assert!(
18131 message
18132 .body
18133 .contains(&format!("magi:run/{}", runner.state.id))
18134 );
18135 }
18136 }
18137 }
18138
18139 fn scripted(id: &str, reply: &str) -> AgentSpec {
18140 let mut a = spec(id);
18141 a.command = vec![
18142 "sh".into(),
18143 "-c".into(),
18144 "printf '%s' \"$REWRITE_REPLY\"".into(),
18145 ];
18146 a.env.insert("REWRITE_REPLY".into(), reply.into());
18147 a
18148 }
18149
18150 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18151 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18152
18153 fn seed_public_summary(runner: &mut Runner) {
18155 let repo = &runner.state.repo;
18156 let g = |args: &[&str]| {
18157 let out = std::process::Command::new("git")
18158 .args(args)
18159 .current_dir(repo)
18160 .quiet()
18161 .output()
18162 .unwrap();
18163 assert!(out.status.success(), "git {args:?}");
18164 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18165 };
18166 g(&["init", "-q", "-b", "main"]);
18167 g(&[
18168 "-c",
18169 "user.name=Test",
18170 "-c",
18171 "user.email=test@example.invalid",
18172 "commit",
18173 "-q",
18174 "--allow-empty",
18175 "-m",
18176 "feat: fixture",
18177 ]);
18178 let head = g(&["rev-parse", "HEAD"]);
18179 g(&["update-ref", "refs/remotes/origin/main", &head]);
18180 let evidence = crate::pr_outcome::summary_evidence(
18181 &runner.state,
18182 crate::pr_outcome::FinalHead::At(&head),
18183 );
18184 let generated = &runner.state.candidates[0].summary;
18185 runner.state.change_summary = Some(crate::run::ChangeSummary {
18186 base: head.clone(),
18187 head,
18188 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18189 title: summary_title(generated).unwrap(),
18190 body: summary_without_title(generated),
18191 });
18192 }
18193
18194 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18195 let mut runner = runner_at(RunStatus::Gating);
18196 runner.state = state_with_summary("add retries", JA_SUMMARY);
18197 if review_only {
18198 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18199 }
18200 runner.state.repo = dir.to_owned();
18201 runner.state.candidates[0].worktree = dir.to_owned();
18202 runner.state.config.agents = agents;
18203 seed_public_summary(&mut runner);
18204 runner
18205 }
18206
18207 fn gate_questions(runner: &Runner) -> usize {
18208 ask::Questions::open()
18209 .list()
18210 .into_iter()
18211 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18212 .count()
18213 }
18214
18215 #[tokio::test]
18216 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18217 crate::run::pin_test_home();
18218 let dir = tempfile::tempdir().unwrap();
18219 let mut runner = rewrite_runner(
18220 dir.path(),
18221 true,
18222 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18223 );
18224 runner.state.config.roles.language_judge =
18225 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18226 let winner = runner.state.candidates[0].clone();
18227 let msg = runner
18228 .guarded_pr_message(&winner, None, true)
18229 .await
18230 .unwrap()
18231 .expect("an English rewrite posts without asking");
18232 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18233 assert!(msg.body.contains("<details>kept</details>"));
18234 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18235 assert_eq!(gate_questions(&runner), 0);
18236 assert!(runner.state.github_text.is_none());
18237 assert!(!runner.state.seats.contains_key("impl-A"));
18238 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18239 }
18240
18241 #[tokio::test]
18242 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18243 crate::run::pin_test_home();
18244 let dir = tempfile::tempdir().unwrap();
18245 let mut runner = rewrite_runner(
18246 dir.path(),
18247 false,
18248 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18249 );
18250 runner.state.config.roles.language_judge =
18251 Some(crate::config::AgentChoice::One("beta".into()));
18252 runner.state.seats.insert(
18253 "impl-A".into(),
18254 SeatState::new("impl-A", "alpha", runner.state.seed),
18255 );
18256 let winner = runner.state.candidates[0].clone();
18257 let msg = runner
18258 .guarded_pr_message(&winner, None, true)
18259 .await
18260 .unwrap()
18261 .expect("rewritten");
18262 assert!(msg.body.contains("<details>kept</details>"));
18263 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18264 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18265 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18266 }
18267
18268 #[tokio::test]
18269 async fn rewrite_reuses_a_resumable_implementer_session() {
18270 crate::run::pin_test_home();
18271 let dir = tempfile::tempdir().unwrap();
18272 let mut runner = rewrite_runner(
18273 dir.path(),
18274 false,
18275 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18276 );
18277 runner.state.config.roles.language_judge =
18278 Some(crate::config::AgentChoice::One("beta".into()));
18279 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18280 seat.turns = 1;
18281 runner.state.seats.insert("impl-A".into(), seat);
18282 let winner = runner.state.candidates[0].clone();
18283 runner
18284 .guarded_pr_message(&winner, None, true)
18285 .await
18286 .unwrap()
18287 .expect("rewritten");
18288 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18289 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18290 }
18291
18292 #[tokio::test]
18293 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18294 crate::run::pin_test_home();
18295 let dir = tempfile::tempdir().unwrap();
18296 let mut broken = spec("bad");
18297 broken.command = vec!["false".into()];
18298 let mut runner = rewrite_runner(
18299 dir.path(),
18300 true,
18301 vec![
18302 broken,
18303 scripted("noformat", "Add retries."),
18304 scripted("good", EN_REWRITE),
18305 ],
18306 );
18307 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18308 "bad".into(),
18309 "noformat".into(),
18310 "good".into(),
18311 ]));
18312 let winner = runner.state.candidates[0].clone();
18313 let msg = runner
18314 .guarded_pr_message(&winner, None, true)
18315 .await
18316 .unwrap()
18317 .expect("the third agent rewrites");
18318 assert!(msg.body.contains("`src/net.rs`"));
18319 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18320 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18321 }
18322
18323 #[tokio::test]
18324 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18325 crate::run::pin_test_home();
18326 for (agents, role, saved) in [
18328 (
18329 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18330 Some("beta"),
18331 true,
18332 ),
18333 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18334 (Vec::new(), None, false),
18335 ] {
18336 let dir = tempfile::tempdir().unwrap();
18337 let mut runner = rewrite_runner(dir.path(), true, agents);
18338 runner.state.config.roles.language_judge =
18339 role.map(|r| crate::config::AgentChoice::One(r.into()));
18340 let winner = runner.state.candidates[0].clone();
18341 let first = runner
18342 .guarded_pr_message(&winner, None, true)
18343 .await
18344 .unwrap();
18345 assert!(first.is_none(), "withheld text parks the run");
18346 assert!(runner.state.parked);
18347 assert_eq!(gate_questions(&runner), 1);
18348 let g = runner.state.github_text.as_ref().unwrap();
18349 assert_eq!(g.rewrite.is_some(), saved);
18350 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18351 }
18352 }
18353
18354 #[tokio::test]
18355 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18356 crate::run::pin_test_home();
18357 let dir = tempfile::tempdir().unwrap();
18358 let mut runner = runner_at(RunStatus::Gating);
18359 runner.state =
18360 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18361 runner.state.repo = dir.path().to_owned();
18362 runner.state.candidates[0].worktree = dir.path().to_owned();
18363 let mut author = spec("alpha");
18364 author.command = vec![
18365 "sh".into(),
18366 "-c".into(),
18367 "printf '%s' \"$REWRITE_REPLY\"".into(),
18368 ];
18369 author.env.insert(
18370 "REWRITE_REPLY".into(),
18371 format!(
18372 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18373 "English sentence. ".repeat(600)
18374 ),
18375 );
18376 runner.state.config.agents = vec![author];
18377 seed_public_summary(&mut runner);
18378 let winner = runner.state.candidates[0].clone();
18379 assert!(
18380 runner
18381 .guarded_pr_message(&winner, None, true)
18382 .await
18383 .unwrap()
18384 .is_none()
18385 );
18386 let record = runner.state.github_text.clone().unwrap();
18387 let rewrite = record.rewrite.clone().unwrap();
18388 assert_eq!(rewrite.failed, ["body-language"]);
18389 assert!(
18390 rewrite
18391 .body
18392 .ends_with(&format!("magi:run/{}", runner.state.id))
18393 );
18394 let store = ask::Questions::open();
18395 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18396 assert!(
18397 q.choices
18398 .contains(&crate::github_text::USE_REWRITE.to_owned())
18399 );
18400 for label in [
18401 "ORIGINAL",
18402 "FALLBACK",
18403 "REWRITE",
18404 "Rules still failed: body-language",
18405 "truncated",
18406 ] {
18407 assert!(q.detail.contains(label), "{label}");
18408 }
18409 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18410 &runner.state,
18411 &crate::github_text::artifact_name(&record.fingerprint),
18412 ))
18413 .unwrap();
18414 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18415 runner.state =
18417 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18418 store
18419 .update(&q.id, |q| {
18420 q.say("まだ日本語")?;
18421 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18422 })
18423 .unwrap();
18424 assert!(
18425 runner
18426 .guarded_pr_message(&winner, None, true)
18427 .await
18428 .unwrap()
18429 .is_none()
18430 );
18431 let retry = runner.state.github_text.as_ref().unwrap();
18432 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18433 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18434 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18435 assert!(
18436 q.posting_comparison
18437 .as_ref()
18438 .unwrap()
18439 .matches_detail(&q.detail)
18440 );
18441 store
18442 .update(&q.id, |q| {
18443 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18444 })
18445 .unwrap();
18446 let message = runner
18447 .guarded_pr_message(&winner, None, true)
18448 .await
18449 .unwrap()
18450 .unwrap();
18451 assert_eq!(message.title, rewrite.title);
18452 assert_eq!(message.body, rewrite.body);
18453 assert_eq!(
18454 runner.state.github_text.as_ref().unwrap().choice,
18455 Some(GithubTextChoice::Rewrite)
18456 );
18457 let again = runner
18458 .guarded_pr_message(&winner, None, true)
18459 .await
18460 .unwrap()
18461 .unwrap();
18462 assert_eq!(again.title, message.title);
18463 assert_eq!(again.body, message.body);
18464 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18465 assert!(
18466 !runner
18467 .state
18468 .events
18469 .iter()
18470 .any(|e| e.message.contains(&rewrite.title)
18471 || e.message.contains("English sentence."))
18472 );
18473 }
18474
18475 #[tokio::test]
18476 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18477 crate::run::pin_test_home();
18478 let dir = tempfile::tempdir().unwrap();
18479 let mut runner = runner_at(RunStatus::Gating);
18480 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18481 runner.state.repo = dir.path().to_owned();
18482 runner.state.candidates[0].worktree = dir.path().to_owned();
18483 seed_public_summary(&mut runner);
18484 let winner = runner.state.candidates[0].clone();
18485 assert!(
18486 runner
18487 .guarded_pr_message(&winner, None, true)
18488 .await
18489 .unwrap()
18490 .is_none()
18491 );
18492 let record = runner.state.github_text.as_ref().unwrap();
18493 assert!(record.rewrite.is_none());
18494 let mut old = serde_json::to_value(record).unwrap();
18496 old.as_object_mut().unwrap().remove("rewrite");
18497 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18498 assert!(old.rewrite.is_none());
18499 let id = record.question.clone().unwrap();
18500 ask::Questions::open()
18501 .update(&id, |q| {
18502 q.choices.push(crate::github_text::USE_REWRITE.into());
18503 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18504 })
18505 .unwrap();
18506 let message = runner
18507 .guarded_pr_message(&winner, None, true)
18508 .await
18509 .unwrap()
18510 .unwrap();
18511 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18512 assert_eq!(
18513 runner.state.github_text.as_ref().unwrap().choice,
18514 Some(GithubTextChoice::Fallback)
18515 );
18516 }
18517
18518 #[tokio::test]
18519 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18520 crate::run::pin_test_home();
18521 let dir = tempfile::tempdir().unwrap();
18522 let mut runner = runner_at(RunStatus::Gating);
18523 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18524 runner.state.repo = dir.path().to_owned();
18525 runner.state.candidates[0].worktree = dir.path().to_owned();
18526 seed_public_summary(&mut runner);
18527 let winner = runner.state.candidates[0].clone();
18528 assert!(
18529 runner
18530 .guarded_pr_message(&winner, None, true)
18531 .await
18532 .unwrap()
18533 .is_none()
18534 );
18535 let store = ask::Questions::open();
18536 let find = |run: &str| {
18537 store
18538 .list()
18539 .into_iter()
18540 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18541 .unwrap()
18542 };
18543 let run = runner.state.id.clone();
18544 let q = find(&run);
18545 let fp = runner
18546 .state
18547 .github_text
18548 .as_ref()
18549 .unwrap()
18550 .fingerprint
18551 .clone();
18552 let name = crate::github_text::artifact_name(&fp);
18553 let path = crate::run::artifact_path(&runner.state, &name);
18554 assert!(q.detail.contains(&path.display().to_string()));
18555 assert!(q.detail.contains("title-language: title"));
18556 let saved = std::fs::read_to_string(&path).expect("artifact written");
18557 assert!(saved.contains("日本語のタイトル"));
18558 assert!(q.cwd.is_none());
18559 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18560 let say = |id: &str, text: &str, choice: &str| {
18562 store
18563 .update(id, |q| {
18564 q.thread.push(ask::Turn {
18565 who: ask::Who::Operator,
18566 body: text.to_owned(),
18567 at: jiff::Timestamp::now(),
18568 note: None,
18569 });
18570 q.answer(ask::Answer::Choice(choice.to_owned()))
18571 })
18572 .unwrap();
18573 };
18574 say(&q.id, "まだ日本語", "use my text");
18575 assert!(
18576 runner
18577 .guarded_pr_message(&winner, None, true)
18578 .await
18579 .unwrap()
18580 .is_none()
18581 );
18582 let second = find(&run);
18583 assert_ne!(second.id, q.id);
18584 assert!(second.detail.contains("did not pass"));
18585 assert!(second.detail.contains("日本語のタイトル"));
18588 assert!(!second.detail.contains("まだ日本語"));
18589 assert!(second.detail.contains(&path.display().to_string()));
18590 say(&second.id, "fix: retry failed requests", "use my text");
18591 let message = runner
18592 .guarded_pr_message(&winner, None, true)
18593 .await
18594 .unwrap()
18595 .expect("a vetted replacement is posted");
18596 assert_eq!(message.title, "fix: retry failed requests");
18597 assert_eq!(
18598 runner
18599 .state
18600 .github_text
18601 .as_ref()
18602 .unwrap()
18603 .chosen_title
18604 .as_deref(),
18605 Some("fix: retry failed requests")
18606 );
18607 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18608 let again = runner
18610 .guarded_pr_message(&winner, None, true)
18611 .await
18612 .unwrap()
18613 .unwrap();
18614 assert_eq!(again.title, message.title);
18615 assert_eq!(
18616 store
18617 .list()
18618 .into_iter()
18619 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18620 .count(),
18621 2
18622 );
18623 }
18624
18625 #[test]
18626 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18627 let mut state = state_with_summary(
18631 "add retries",
18632 "TITLE: fix(web): batch reads\n- reads run.json once",
18633 );
18634 state.config.graph.language = "ja".to_owned();
18635 let m = pr_message(&state, 'A');
18636 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18637 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18638
18639 let task = "今回やってほしいこと: results projector を直す";
18642 let mut state = state_with_summary(task, "- no title line");
18643 state.config.graph.language = "ja".to_owned();
18644 let m = pr_message(&state, 'A');
18645 assert_eq!(
18646 m.title,
18647 format!("chore: land candidate A of run {}", state.id)
18648 );
18649 assert!(
18650 m.body.contains(&format!(
18651 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18652 )),
18653 "{}",
18654 m.body
18655 );
18656 }
18657
18658 #[test]
18659 fn pr_message_scrubs_home_paths_and_addresses() {
18660 let state = state_with_summary(
18661 "fix it in /Users/someone/src/x",
18662 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18663 );
18664 let m = pr_message(&state, 'A');
18665 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18666 assert!(!m.body.contains(leak), "{}", m.body);
18667 }
18668 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18669 }
18670
18671 #[test]
18672 fn pr_message_survives_a_task_that_closes_details() {
18673 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18674 let m = pr_message(&state, 'A');
18675 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18676 }
18677
18678 #[test]
18679 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18680 let cmd = manual_merge_command(
18681 MergeStyle::Squash,
18682 Path::new("/repo"),
18683 "b",
18684 "fix: \"quoted\" $(x) `y`\n\nbody",
18685 );
18686 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18687 }
18688
18689 #[test]
18690 fn manual_merge_command_matches_the_configured_style() {
18691 let repo = Path::new("/repo");
18692 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18693
18694 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18695 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18696
18697 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18698 assert_eq!(
18699 squash,
18700 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18701 \"Merge magi run 0832 (candidate A)\""
18702 );
18703
18704 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18705 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18706 }
18707
18708 #[test]
18709 fn a_nudge_gets_a_quarter_of_the_budget() {
18710 assert_eq!(retry_budget(secs(1200), true), secs(300));
18712 assert_eq!(retry_budget(secs(3600), true), secs(900));
18713 }
18714
18715 #[test]
18716 fn a_resent_prompt_keeps_the_whole_budget() {
18717 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18720 assert_eq!(retry_budget(secs(60), false), secs(60));
18721 }
18722
18723 #[test]
18724 fn the_floor_never_exceeds_the_original_budget() {
18725 assert_eq!(retry_budget(secs(60), true), secs(60));
18729 assert_eq!(retry_budget(secs(480), true), secs(120));
18730 assert_eq!(retry_budget(secs(0), true), secs(0));
18731 }
18732
18733 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18734 agent::CommandEvidence {
18735 id: "item1".to_owned(),
18736 description: "cargo test".to_owned(),
18737 exit_code,
18738 result_summary: String::new(),
18739 source: "codex".to_owned(),
18740 }
18741 }
18742
18743 #[test]
18744 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18745 assert!(!has_unconfirmed_command(&[]));
18749 }
18750
18751 #[test]
18752 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18753 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18757 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18758 assert!(!has_unconfirmed_command(&[
18759 evidence(Some(0)),
18760 evidence(Some(101))
18761 ]));
18762 }
18763
18764 #[test]
18765 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18766 assert!(has_unconfirmed_command(&[
18767 evidence(Some(0)),
18768 evidence(None)
18769 ]));
18770 }
18771
18772 #[test]
18773 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18774 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18775 assert_eq!(
18776 verified_noop_claim(true, &[], text).as_deref(),
18777 Some("already fixed by b32cfc4, on main.")
18778 );
18779 }
18780
18781 #[test]
18782 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18783 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18786 assert!(verified_noop_claim(false, &[], text).is_none());
18787 }
18788
18789 #[test]
18790 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18791 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18792 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18793 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18795 }
18796
18797 #[test]
18798 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18799 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18800 }
18801
18802 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18805 runner.state.candidates = shape
18806 .iter()
18807 .enumerate()
18808 .map(|(i, &(empty, verified))| Candidate {
18809 index: i,
18810 label: (b'A' + i as u8) as char,
18811 agent: "sonnet".to_owned(),
18812 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18813 worktree: PathBuf::from(format!("/wt/{i}")),
18814 summary: String::new(),
18815 stat: String::new(),
18816 files: 0,
18817 commits: 0,
18818 empty,
18819 failed: None,
18820 verified_noop: verified.map(str::to_owned),
18821 duration_ms: 0,
18822 folded: false,
18823 })
18824 .collect();
18825 }
18826
18827 #[test]
18828 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18829 ask_test_home();
18830 let mut runner = runner_at(RunStatus::Implementing);
18831 set_candidates(
18832 &mut runner,
18833 &[
18834 (true, Some("already on main at b32cfc4")),
18835 (true, Some("same fix, see the existing test")),
18836 ],
18837 );
18838
18839 runner
18840 .after_implement()
18841 .expect("a verified no-op is not an error");
18842
18843 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18844 }
18845
18846 #[test]
18847 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18848 ask_test_home();
18849 let mut runner = runner_at(RunStatus::Implementing);
18850 set_candidates(
18854 &mut runner,
18855 &[(true, Some("already on main at b32cfc4")), (true, None)],
18856 );
18857
18858 let err = runner
18859 .after_implement()
18860 .expect_err("an unverified empty candidate must still fail the run");
18861
18862 assert!(
18863 err.to_string().contains("no candidate produced a change"),
18864 "{err}"
18865 );
18866 assert_eq!(runner.state.status, RunStatus::Failed);
18867 }
18868
18869 #[test]
18870 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18871 ask_test_home();
18872 let mut runner = runner_at(RunStatus::Implementing);
18873 set_candidates(&mut runner, &[(true, None), (true, None)]);
18874
18875 let err = runner
18876 .after_implement()
18877 .expect_err("no candidate declared anything; this is an ordinary failure");
18878
18879 assert!(
18880 err.to_string().contains("no candidate produced a change"),
18881 "{err}"
18882 );
18883 assert_eq!(runner.state.status, RunStatus::Failed);
18884 }
18885}
18886
18887#[derive(Debug)]
18890struct GraftRestoreFailed(String);
18891
18892impl std::fmt::Display for GraftRestoreFailed {
18893 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18894 write!(
18895 f,
18896 "could not restore the winner to its pre-graft commit: {}",
18897 self.0
18898 )
18899 }
18900}
18901
18902impl std::error::Error for GraftRestoreFailed {}
18903
18904#[cfg(test)]
18908mod resource_wait_tests {
18909 use super::*;
18910 use std::collections::BTreeMap;
18911
18912 fn sh(id: &str, script: &str) -> AgentSpec {
18913 AgentSpec {
18914 id: id.to_owned(),
18915 kind: crate::config::AgentKind::Command,
18916 model: None,
18917 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18918 extra_args: Vec::new(),
18919 env: BTreeMap::new(),
18920 prompt_delivery: None,
18921 }
18922 }
18923
18924 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18925 SeatJob {
18926 seat: SeatState::new("impl-A", &spec.id, 1),
18927 spec,
18928 cwd: dir.to_path_buf(),
18929 prompt: "do it".to_owned(),
18930 timeout,
18931 allow_write: true,
18932 sessions: false,
18933 artifacts: dir.join("art"),
18934 stem: "impl-A".to_owned(),
18935 handover: None,
18936 }
18937 }
18938
18939 fn state(resource_wait_secs: u64) -> RunState {
18940 let mut config = Config::default();
18941 config.graph.timeout_resource_wait = resource_wait_secs;
18942 RunState::new(
18943 PathBuf::from("/repo"),
18944 "main".to_owned(),
18945 "abc1234".to_owned(),
18946 "task".to_owned(),
18947 config,
18948 )
18949 }
18950
18951 fn hold(cache: &Path) -> crate::cache::Guard {
18952 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18953 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18954 crate::cache::AcquireOutcome::Acquired(g) => g,
18955 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18956 }
18957 }
18958
18959 async fn run(
18960 jobs: Vec<SeatJob>,
18961 sem: Arc<Semaphore>,
18962 cache: Option<&Path>,
18963 st: &mut RunState,
18964 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18965 let prompts = Prompts::default();
18966 let ctx = WaveCtx {
18967 run: "r",
18968 node: "implement",
18969 prompts: &prompts,
18970 cache,
18971 round: None,
18972 carry_seats: false,
18973 pauses: None,
18974 };
18975 wave(jobs, sem, &ctx, st, 0).await
18976 }
18977
18978 #[tokio::test]
18979 async fn verification_cache_wait_preserves_the_command_budget() {
18980 crate::run::pin_test_home();
18981 let tmp = tempfile::tempdir().unwrap();
18982 let cache = tmp.path().join("target");
18983 let guard = hold(&cache);
18984 let release = tokio::spawn(async move {
18985 tokio::time::sleep(Duration::from_millis(100)).await;
18986 guard.release();
18987 });
18988 let mut st = state(10);
18989 let budget = Duration::from_millis(50);
18990 let wait = repository_wait(&st, None);
18991 let (outcomes, _) = with_cache_lease(
18992 &mut st,
18993 Some(&cache),
18994 "gate",
18995 "gate",
18996 tmp.path(),
18997 "abc1234",
18998 budget,
18999 "test gate",
19000 &wait,
19001 |_, available| async move {
19002 assert_eq!(
19003 available, budget,
19004 "contention must not shorten verification"
19005 );
19006 (Vec::new(), false, Vec::new())
19007 },
19008 )
19009 .await
19010 .unwrap();
19011 release.await.unwrap();
19012 assert!(outcomes.is_empty(), "{outcomes:?}");
19013 assert!(
19014 st.events
19015 .iter()
19016 .any(|event| event.message.contains("waiting for"))
19017 );
19018 assert!(st.resource_wait.is_none());
19019 hold(&cache).release();
19021 }
19022
19023 #[tokio::test]
19024 async fn verification_cache_wait_expires_without_starting_commands() {
19025 crate::run::pin_test_home();
19026 let tmp = tempfile::tempdir().unwrap();
19027 let cache = tmp.path().join("target");
19028 let _held = hold(&cache);
19029 let mut st = state(0);
19030 let wait = repository_wait(&st, None);
19031 let (outcomes, _) = with_cache_lease(
19032 &mut st,
19033 Some(&cache),
19034 "gate",
19035 "gate",
19036 tmp.path(),
19037 "abc1234",
19038 Duration::from_secs(60),
19039 "test gate",
19040 &wait,
19041 |_, _| async { panic!("commands must not start without the cache lease") },
19042 )
19043 .await
19044 .unwrap();
19045 assert_eq!(outcomes.len(), 1);
19046 assert!(outcomes[0].resource_blocked);
19047 assert!(st.resource_wait.is_none());
19048 }
19049
19050 #[tokio::test]
19051 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19052 crate::run::pin_test_home();
19053 let tmp = tempfile::tempdir().unwrap();
19054 let cache = tmp.path().join("target-a");
19055 let guard = hold(&cache);
19056 tokio::spawn(async move {
19058 tokio::time::sleep(Duration::from_millis(1500)).await;
19059 guard.release();
19060 });
19061 let mut st = state(60);
19062 let results = run(
19063 vec![job(
19064 sh("alpha", "echo ok"),
19065 tmp.path(),
19066 Duration::from_secs(1),
19067 )],
19068 Arc::new(Semaphore::new(1)),
19069 Some(&cache),
19070 &mut st,
19071 )
19072 .await;
19073 assert!(
19074 matches!(results[0].2, AgentOutcome::Ok(_)),
19075 "the wait must not be charged to the agent"
19076 );
19077 assert!(st.resource_block.is_none());
19078 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19079 assert!(st.resource_wait.is_none() && st.active.is_empty());
19080 }
19081
19082 #[tokio::test]
19083 #[cfg(unix)]
19084 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19085 use std::os::fd::AsRawFd as _;
19086 crate::run::pin_test_home();
19087 let tmp = tempfile::tempdir().unwrap();
19088 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19089 let common = tmp.path().join(".git");
19090 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19091 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19092 assert_eq!(
19093 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19094 0
19095 );
19096 let mut st = state(1);
19097 st.config.coordination = crate::config::test_coordination();
19098 let mut reader = job(
19099 sh("reader", "echo must-not-run"),
19100 tmp.path(),
19101 Duration::from_secs(30),
19102 );
19103 reader.allow_write = false;
19104 let results = run(
19105 vec![reader.clone()],
19106 Arc::new(Semaphore::new(1)),
19107 None,
19108 &mut st,
19109 )
19110 .await;
19111 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19112 assert_eq!(results[0].1.turns, 0);
19113 assert!(FailClass::of(&results[0].2).is_none());
19114 assert_eq!(
19115 st.resource_block.as_ref().unwrap().kind,
19116 "repository coordination"
19117 );
19118 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19119 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19120 drop(external);
19121 let editor = crate::repo_lock::SeatLease::for_checkout(
19122 tmp.path(),
19123 true,
19124 crate::config::test_coordination().as_ref(),
19125 )
19126 .await
19127 .unwrap();
19128 let pause = Pause::new();
19129 let interrupt = Pause::new();
19130 let prompts = Prompts::default();
19131 let ctx = WaveCtx {
19132 run: "r",
19133 node: "review",
19134 prompts: &prompts,
19135 cache: None,
19136 round: None,
19137 carry_seats: false,
19138 pauses: Some((&pause, &interrupt)),
19139 };
19140 let (results, ()) = tokio::join!(
19141 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19142 async {
19143 tokio::time::sleep(Duration::from_millis(50)).await;
19144 pause.park();
19145 }
19146 );
19147 assert!(
19148 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19149 );
19150 assert_eq!(results[0].1.turns, 0);
19151 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19152 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19153 drop(editor);
19154 }
19155
19156 #[tokio::test]
19157 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19158 crate::run::pin_test_home();
19159 let tmp = tempfile::tempdir().unwrap();
19160 let cache = tmp.path().join("target-b");
19161 let _held = hold(&cache);
19162 let mut st = state(1);
19163 let results = run(
19164 vec![job(
19165 sh("alpha", "echo ok"),
19166 tmp.path(),
19167 Duration::from_secs(1),
19168 )],
19169 Arc::new(Semaphore::new(1)),
19170 Some(&cache),
19171 &mut st,
19172 )
19173 .await;
19174 let (_, seat, out) = &results[0];
19175 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19176 assert!(wave_blocked(&results));
19177 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19178 assert_eq!(seat.turns, 0, "nothing was asked");
19179 assert!(
19180 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19181 "no agent was spawned"
19182 );
19183 assert!(FailClass::of(out).is_none(), "not a failure class");
19184 let block = st.resource_block.as_ref().expect("recorded for the resume");
19185 assert_eq!(block.kind, "shared build cache");
19186 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19187 assert!(st.active.is_empty());
19188 }
19189
19190 #[tokio::test]
19191 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19192 crate::run::pin_test_home();
19193 let tmp = tempfile::tempdir().unwrap();
19194 let mut st = state(1);
19195 let results = run(
19196 vec![job(
19197 sh("alpha", "echo ok"),
19198 tmp.path(),
19199 Duration::from_secs(1),
19200 )],
19201 Arc::new(Semaphore::new(0)),
19202 None,
19203 &mut st,
19204 )
19205 .await;
19206 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19207 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19208 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19209 }
19210
19211 #[tokio::test]
19212 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19213 crate::run::pin_test_home();
19214 let tmp = tempfile::tempdir().unwrap();
19215 let mut st = state(60);
19216 let results = run(
19217 vec![job(
19218 sh("alpha", "sleep 30"),
19219 tmp.path(),
19220 Duration::from_secs(1),
19221 )],
19222 Arc::new(Semaphore::new(1)),
19223 None,
19224 &mut st,
19225 )
19226 .await;
19227 let out = &results[0].2;
19228 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19229 let class = FailClass::of(out).expect("a failure");
19230 assert_eq!(class, FailClass::Timeout);
19231 assert!(should_hand_over(None, &class), "existing handover policy");
19232 assert!(st.resource_block.is_none());
19233 }
19234}