1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 for id in &takeover.earlier {
1033 if let Ok(old) = RunState::load_under(id, &takeover.home)
1034 && old.repo == state.repo
1035 && (old.pr_head.as_deref() == Some(branch)
1036 || old.winner().is_some_and(|w| w.branch == branch))
1037 {
1038 for hold in old.branch_owner_holds {
1039 if !state.branch_owner_holds.contains(&hold) {
1040 state.event(
1041 "approval-policy",
1042 format!(
1043 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044 hold.control.key(),
1045 hold.file,
1046 hold.head
1047 ),
1048 );
1049 state.branch_owner_holds.push(hold);
1050 }
1051 }
1052 }
1053 }
1054 let reviewed_head = git::rev_parse(repo, branch).await?;
1055 state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057 let worktree = state.worktree_root().join("under-review");
1060 if let Some(parent) = worktree.parent() {
1061 tokio::fs::create_dir_all(parent).await.ok();
1062 }
1063 let path = worktree.to_string_lossy().to_string();
1064 git::git(repo, &["worktree", "add", &path, branch])
1065 .await
1066 .with_context(|| {
1067 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068 })?;
1069
1070 let setup = match ensure_setup_config(&mut state, repo).await {
1071 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072 Err(e) => Err(e),
1073 };
1074 if let Err(e) = setup {
1075 git::worktree_remove_clean(repo, &worktree).await.ok();
1076 return Err(e);
1077 }
1078 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079 .await
1080 .unwrap_or(0);
1081 if commits == 0 {
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084 }
1085 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086 .await
1087 .map(|f| f.len())
1088 .unwrap_or(0);
1089 if files == 0
1090 && let (Ok(head_tree), Ok(base_tree)) = (
1091 git::tree_of(&worktree, "HEAD").await,
1092 git::tree_of(&worktree, &base_commit).await,
1093 )
1094 && head_tree == base_tree
1095 {
1096 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097 git::worktree_remove_clean(repo, &worktree).await.ok();
1098 bail!(
1099 "`{branch}` at {} has a tree identical to base {}; this usually means \
1100 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101 against `{}/{branch}`) rather than an empty change",
1102 short(&head),
1103 short(&base_commit),
1104 state.config.merge.remote
1105 );
1106 }
1107 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108 .await
1109 .unwrap_or_default();
1110
1111 state.candidates.push(Candidate {
1112 index: 0,
1113 label: 'A',
1114 agent: EXISTING_BRANCH.to_owned(),
1117 branch: branch.to_owned(),
1118 worktree,
1119 summary: String::new(),
1120 stat,
1121 files,
1122 commits,
1123 empty: false,
1124 failed: None,
1125 verified_noop: None,
1126 duration_ms: 0,
1127 folded: false,
1128 });
1129 state.tally = Some(Tally {
1130 first_choice: BTreeMap::from([('A', 0)]),
1131 borda: BTreeMap::new(),
1132 winner: 'A',
1133 rankings: 0,
1134 unanimous_initial: false,
1135 deliberated: false,
1136 changed_votes: 0,
1137 unanimous_final: false,
1138 tie_break: None,
1139 judges: 0,
1143 present: 0,
1144 quorum: 0,
1145 met_quorum: true,
1146 uncontested: Some("review-only run: nothing competed".to_owned()),
1147 });
1148 state.status = RunStatus::Reviewing;
1149 state.prep_complete = Some(true);
1150 state.event(
1151 "start",
1152 format!(
1153 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154 state.id
1155 ),
1156 );
1157 state.save()?;
1158 Ok(Self {
1159 state,
1160 roles,
1161 sem: Arc::new(Semaphore::new(max_parallel)),
1162 pause: Pause::new(),
1163 interrupt: Pause::new(),
1164 })
1165 }
1166
1167 pub fn resume(id: &str) -> Result<Self> {
1169 let state = RunState::load(id)?;
1170 crate::pr_head::ensure_owner(&state)?;
1171 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172 bail!(
1173 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174 );
1175 }
1176 if let Some(to) = &state.released_to {
1177 bail!(
1178 "run {} cannot be resumed: its worktree was released to run {}",
1179 state.short(),
1180 crate::run::short_of(to)
1181 );
1182 }
1183 let roles = state.config.resolve_roles()?;
1184 if let Some(id) = &state.config.graph.selected_implementer
1185 && state.candidates.iter().any(|c| c.agent != *id)
1186 {
1187 bail!(
1188 "replacement implementer pin does not match the saved candidates; task must stay held"
1189 );
1190 }
1191 let max_parallel = state.config.graph.max_parallel.max(1);
1192 Ok(Self {
1193 state,
1194 roles,
1195 sem: Arc::new(Semaphore::new(max_parallel)),
1196 pause: Pause::new(),
1197 interrupt: Pause::new(),
1198 })
1199 }
1200
1201 pub async fn execute(&mut self) -> Result<()> {
1208 if let Some(selection) = &self.state.pr_recovery
1209 && (selection.source_proof.is_some() || selection.source_run.is_some())
1210 {
1211 Box::pin(selection.verify_source(&self.state.repo)).await?;
1212 }
1213 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216 || self.state.land_approval.is_some()
1217 || self.state.merge_decision.is_some()
1218 || !self.state.merge_repairs.is_empty()
1219 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220 {
1221 let claim = FixClaim::acquire(&self.state.dir())?;
1222 let disk = RunState::load(&self.state.id)?;
1223 let checkpoint = |s: &RunState| {
1224 (
1225 s.merge_repairs.len(),
1226 s.merge_repairs
1227 .last()
1228 .map(|r| (r.question.clone(), r.phase)),
1229 s.land_approval
1230 .as_ref()
1231 .map(|a| (a.question.clone(), a.head.clone())),
1232 )
1233 };
1234 if checkpoint(&disk) != checkpoint(&self.state) {
1235 bail!(
1236 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237 );
1238 }
1239 Some(claim)
1240 } else {
1241 None
1242 };
1243 let mut execution = self.claim_driver()?;
1244 let runner = &mut *execution.runner;
1245 let wait = runner.repository_wait();
1246 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249 let result = match result {
1250 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251 record_repository_block(&mut runner.state, &e);
1252 runner.park_here()?;
1253 Ok(())
1254 }
1255 other => other,
1256 };
1257 execution.driver.finish(&mut runner.state)?;
1258 let ended = if result.is_err() {
1259 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260 } else {
1261 crate::notices::run_ended(&runner.state)
1262 };
1263 if let Some(notice) = ended {
1264 crate::notices::raise(notice);
1265 }
1266 result
1267 }
1268
1269 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271 }
1272
1273 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274 let wait = self.repository_wait();
1275 coordinate_operation(&mut self.state, cwd, &wait).await
1276 }
1277
1278 fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280 let owner = match crate::pr_head::owner_lock(&crate::run::home(), &self.state.id) {
1281 Ok(owner) => owner,
1282 Err(error) => {
1283 let detail = format!("worktree maintenance or ownership update: {error:#}");
1284 self.state.resource_block = Some(crate::run::ResourceBlock {
1288 node: "start".into(),
1289 kind: "worktree maintenance".into(),
1290 detail: detail.clone(),
1291 at: Timestamp::now(),
1292 });
1293 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1294 }
1295 };
1296 let disk = RunState::load(&self.state.id)?;
1297 crate::pr_head::ensure_owner(&disk)?;
1298 if disk.liveness(false) == crate::run::Liveness::Live {
1299 bail!("run already has a live driver; wait for its execution to end");
1300 }
1301 if let Some(to) = &disk.released_to {
1302 bail!(
1303 "run {} cannot continue: its worktree was released to run {}",
1304 self.state.short(),
1305 crate::run::short_of(to)
1306 );
1307 }
1308 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1309 let mut execution = DriverExecution {
1310 runner: self,
1311 driver,
1312 };
1313 let _owner = owner;
1316 let state = &mut execution.runner.state;
1317 state.parked = false;
1322 state.resource_wait = None;
1323 state.resource_block = None;
1324 state.clear_active();
1331 let pid = std::process::id();
1346 state.driver_pid = Some(pid);
1347 state.driver_started_at = crate::proc::process_started_at(pid);
1348 state.driver_exited = false;
1349 state.begin_pass();
1350 execution.driver.bind(state);
1351 state.save()?;
1352 execution.driver.record(state)?;
1353 Ok(execution)
1354 }
1355
1356 async fn execute_graph(&mut self) -> Result<()> {
1357 self.state.configure_recovery_resume()?;
1358 self.persist_candidates().await?;
1361 if self
1362 .state
1363 .merge_decision
1364 .as_ref()
1365 .is_some_and(|decision| !decision.closed)
1366 && self.state.status != RunStatus::Landing
1367 {
1368 if crate::merge_decision::resumable_hold(&self.state) {
1369 self.state.merge = Some(MergeOutcome {
1370 mode: MergeMode::Pr,
1371 ok: false,
1372 detail: self
1373 .state
1374 .merge_decision
1375 .as_ref()
1376 .unwrap()
1377 .target
1378 .pr
1379 .clone(),
1380 empty: false,
1381 });
1382 self.run_land().await?;
1383 self.settle_questions();
1384 return Ok(());
1385 }
1386 bail!(
1387 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1388 );
1389 }
1390 if self.state.config.graph.selected_implementer.is_some()
1391 && self.state.status == RunStatus::Blocked
1392 && self.state.tally.is_none()
1393 && self.state.candidate_preservation.is_empty()
1394 && self.state.interrupted_implementations.is_empty()
1395 {
1396 bail!(
1397 "selected implementation is held; explicitly release the owning task to retry it"
1398 );
1399 }
1400
1401 if self.state.merge_repairs.last().is_some_and(|r| {
1402 matches!(
1403 r.phase,
1404 crate::run::MergeRepairPhase::Requested
1405 | crate::run::MergeRepairPhase::Fixing
1406 | crate::run::MergeRepairPhase::Reviewing
1407 )
1408 }) {
1409 if self.repair_merge_review().await? {
1410 self.run_land().await?;
1411 }
1412 self.settle_questions();
1413 return Ok(());
1414 }
1415 if self.state.pr_publication_pending {
1418 self.merge().await?;
1419 if self.resource_halt()? {
1420 return Ok(());
1421 }
1422 self.state.save()?;
1423 self.settle_questions();
1424 return Ok(());
1425 }
1426 if self.state.status == RunStatus::Stalled {
1439 if self.recover_stall().await? {
1440 self.finish_after_tally().await?;
1441 } else {
1442 self.state.save()?;
1444 }
1445 return Ok(());
1446 }
1447 if self.state.status == RunStatus::Landing
1457 || crate::handover::landing_checkpoint(&self.state)
1458 {
1459 self.state.land_sync_pending = true;
1460 self.state.save()?;
1461 self.run_land().await?;
1462 self.settle_questions();
1467 return Ok(());
1468 }
1469 if self.state.github_text.is_some()
1474 && self.state.merge.is_none()
1475 && self.state.status == RunStatus::Gating
1476 {
1477 self.merge().await?;
1478 if self.resource_halt()? {
1479 return Ok(());
1480 }
1481 self.state.save()?;
1482 return Ok(());
1483 }
1484 if let Err(e) = self.prep().await {
1485 if e.is::<crate::repo_lock::ResourceBlocked>() {
1486 return Err(e);
1487 }
1488 self.state.status = RunStatus::Blocked;
1489 self.state
1490 .prep_failure
1491 .get_or_insert_with(|| format!("{e:#}"));
1492 self.state
1493 .event("prep", format!("preparation stopped: {e:#}"));
1494 self.state.save()?;
1495 return Err(e);
1496 }
1497 if self.park_here()? {
1498 return Ok(());
1499 }
1500 self.advise().await?;
1501 if self.park_here()? {
1502 return Ok(());
1503 }
1504 self.implement().await?;
1505 if self.park_here()? {
1506 return Ok(());
1507 }
1508 if self.state.status == RunStatus::VerifiedNoop {
1512 return Ok(());
1513 }
1514 self.judge().await?;
1515 if self.park_here()? {
1516 return Ok(());
1517 }
1518 self.deliberate().await?;
1519 if self.park_here()? {
1520 return Ok(());
1521 }
1522 self.vote().await?;
1523 if self.park_here()? {
1524 return Ok(());
1525 }
1526 self.tally()?;
1527 if self.state.status == RunStatus::Stalled {
1532 self.state.save()?;
1536 return Ok(());
1537 }
1538 self.finish_after_tally().await?;
1539 Ok(())
1540 }
1541
1542 fn stop_for_resource(&mut self) -> Result<()> {
1547 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1549 {
1550 self.state.status = RunStatus::Blocked;
1551 }
1552 self.state.save()
1553 }
1554
1555 fn resource_halt(&mut self) -> Result<bool> {
1557 if self.state.resource_block.is_none() {
1558 return Ok(false);
1559 }
1560 self.stop_for_resource()?;
1561 Ok(true)
1562 }
1563
1564 fn park_here(&mut self) -> Result<bool> {
1571 let parking = self.pause.parked() || self.interrupt.parked();
1572 if !parking && self.state.resource_block.is_some() {
1573 self.stop_for_resource()?;
1574 return Ok(true);
1575 }
1576 if !self.pause.parked() && !self.interrupt.parked() {
1581 return Ok(false);
1582 }
1583 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1584 let why = match cause.as_ref().map(ParkCause::label) {
1585 Some(reason) => format!(
1586 "parked after `{}` ({reason}) — resume to carry on from here",
1587 self.state.status.as_str()
1588 ),
1589 None => format!(
1590 "parked after `{}` — resume to carry on from here",
1591 self.state.status.as_str()
1592 ),
1593 };
1594 self.state.event("park", why);
1595 self.state
1596 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1597 self.state.parked = true;
1598 self.state.save()?;
1599 Ok(true)
1600 }
1601
1602 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1604 if self.state.prep_failure.is_none() {
1605 return Ok(());
1606 }
1607 let failed_at = self
1608 .state
1609 .events
1610 .iter()
1611 .rev()
1612 .find(|event| event.node == "prep")
1613 .map_or(self.state.updated_at, |event| event.at);
1614 anyhow::ensure!(
1615 approval_at > failed_at,
1616 "preparation retry requires a new explicit release after the failure"
1617 );
1618 self.state.prep_failure = None;
1619 self.state.prep_complete = Some(false);
1620 self.state
1621 .event("prep-retry", "operator approved retry of preparation");
1622 Ok(())
1623 }
1624
1625 pub fn on_pause(&mut self, pause: Pause) {
1627 self.pause = pause;
1628 }
1629
1630 pub fn watch_interrupt(&mut self, pause: Pause) {
1636 self.interrupt = pause;
1637 }
1638
1639 fn settle_questions(&mut self) {
1659 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1660 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1661 }
1662 }
1663
1664 async fn finish_after_tally(&mut self) -> Result<()> {
1667 self.graft().await?;
1670 if self.resource_halt()? {
1671 return Ok(());
1672 }
1673 self.fold_losers().await?;
1674 self.sync_to_base().await?;
1679 if self.resource_halt()? {
1680 return Ok(());
1681 }
1682 if self.state.status == RunStatus::AlreadyInBase {
1683 return Ok(());
1684 }
1685 self.review_loop().await?;
1686 if self.resource_halt()? {
1687 return Ok(());
1688 }
1689 self.sync_to_base().await?;
1690 if self.resource_halt()? {
1691 return Ok(());
1692 }
1693 if self.state.status == RunStatus::AlreadyInBase {
1694 return Ok(());
1695 }
1696 self.gate().await?;
1697 self.merge().await?;
1698 if self.resource_halt()? {
1699 return Ok(());
1700 }
1701 self.state.save()?;
1702 Ok(())
1703 }
1704
1705 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1712 let result = match ensure_setup_config(&mut self.state, repo).await {
1713 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1714 Err(e) => Err(e),
1715 };
1716 if let Err(e) = result {
1717 git::worktree_remove_clean(repo, wt).await.ok();
1718 if in_prep {
1719 self.rollback_prep(repo).await;
1720 }
1721 self.state.status = RunStatus::Blocked;
1722 self.state.event(
1723 "setup",
1724 format!("worktree setup failed in {}: {e:#}", wt.display()),
1725 );
1726 self.state.save()?;
1727 return Err(e);
1728 }
1729 Ok(())
1730 }
1731
1732 async fn rollback_prep(&mut self, repo: &Path) {
1734 let root = self.state.worktree_root();
1735 for c in std::mem::take(&mut self.state.candidates) {
1736 if matches!(
1737 git::worktree_remove_clean(repo, &c.worktree).await,
1738 Ok(true)
1739 ) {
1740 git::branch_delete(repo, &c.branch).await.ok();
1741 } else {
1742 self.state.candidates.push(c);
1743 }
1744 }
1745 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1746 for j in 1..=seats {
1747 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1748 .await
1749 .ok();
1750 }
1751 for k in 1..=self.state.config.graph.advisors {
1752 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1753 .await
1754 .ok();
1755 }
1756 }
1757
1758 async fn prep(&mut self) -> Result<()> {
1759 if let Some(reason) = &self.state.prep_failure {
1760 bail!(
1761 "candidate preparation previously failed: {reason}; correct the source before retrying"
1762 );
1763 }
1764 if self
1765 .state
1766 .prep_complete
1767 .unwrap_or(!self.state.candidates.is_empty())
1768 {
1769 return Ok(());
1770 }
1771 let repo = self.state.repo.clone();
1772 let _coordination = self.coordinate(&repo).await?;
1775 let base = self.state.base_commit.clone();
1776 let plan = self.state.sources.plan(&repo, &base).await?;
1777 self.rollback_prep(&repo).await;
1778 if !self.state.candidates.is_empty() {
1779 bail!(
1780 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1781 );
1782 }
1783 self.state.prep_complete = Some(false);
1784 self.state.status = RunStatus::Prep;
1785 self.state.save()?;
1786 let repo = self.state.repo.clone();
1787 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1788 let root = self.state.worktree_root();
1789 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1790
1791 let hooks_dir = self.state.dir().join("hooks");
1794 if self.state.config.blind.commit_msg_hook {
1795 std::fs::create_dir_all(&hooks_dir)
1796 .with_context(|| format!("create {}", hooks_dir.display()))?;
1797 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1798 let path = hooks_dir.join("commit-msg");
1799 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1800 make_executable(&path)?;
1801 git::acquire_worktree_config(&repo).await?;
1809 self.state.enabled_worktree_config = true;
1810 }
1811
1812 ensure_setup_config(&mut self.state, &repo).await?;
1815
1816 for (index, (spec, label)) in self
1817 .roles
1818 .implementers
1819 .clone()
1820 .into_iter()
1821 .zip(labels)
1822 .enumerate()
1823 {
1824 let branch = self.state.branch_for(label);
1825 let worktree = root.join(format!("cand-{label}"));
1826 self.state.event(
1830 "prep-intent",
1831 serde_json::json!({
1832 "branch": branch,
1833 "worktree": worktree,
1834 "start": start,
1835 "picks": plan.picks,
1836 })
1837 .to_string(),
1838 );
1839 self.state.save()?;
1840 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1841 if self.state.config.blind.commit_msg_hook {
1842 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1843 }
1844 git::local_exclude(&worktree, "/.magi/").await?;
1845 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1846 git::worktree_remove_clean(&repo, &worktree).await.ok();
1849 git::branch_delete(&repo, &branch).await.ok();
1850 self.rollback_prep(&repo).await;
1851 self.state.status = RunStatus::Blocked;
1852 self.state
1853 .event("prep", format!("worktree setup failed: {e:#}"));
1854 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1855 self.state.save()?;
1856 return Err(e);
1857 }
1858 self.state.candidates.push(Candidate {
1859 index,
1860 label,
1861 agent: spec.id.clone(),
1862 branch,
1863 worktree,
1864 summary: String::new(),
1865 stat: String::new(),
1866 files: 0,
1867 commits: 0,
1868 empty: false,
1869 failed: None,
1870 verified_noop: None,
1871 duration_ms: 0,
1872 folded: false,
1873 });
1874 self.state.save()?;
1875 for pick in &plan.picks {
1876 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1877 {
1878 self.state.status = RunStatus::Blocked;
1879 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1880 self.state
1881 .event("prep", format!("cannot apply referenced commit: {e}"));
1882 self.state.save()?;
1883 return Err(e);
1884 }
1885 }
1886 }
1887
1888 for j in 1..=self.roles.judges.len() {
1889 let wt = root.join(format!("judge-{j}"));
1890 if !wt.exists() {
1891 git::worktree_add_detached(&repo, &wt, &base).await?;
1892 self.setup_seat_worktree(&repo, &wt, true).await?;
1893 }
1894 }
1895
1896 if self.state.config.graph.advise {
1904 for k in 1..=self.state.config.graph.advisors {
1905 let wt = root.join(format!("advisor-{k}"));
1906 if !wt.exists() {
1907 git::worktree_add_detached(&repo, &wt, &base).await?;
1908 self.setup_seat_worktree(&repo, &wt, true).await?;
1909 }
1910 }
1911 }
1912
1913 let authors: Vec<&str> = self
1918 .roles
1919 .implementers
1920 .iter()
1921 .map(|a| a.id.as_str())
1922 .collect();
1923 let overlap: Vec<String> = self
1924 .roles
1925 .judges
1926 .iter()
1927 .enumerate()
1928 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1929 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1930 .collect();
1931 if !overlap.is_empty() {
1932 let note = format!(
1933 "{} also authored a candidate; blind, but the panel is less \
1934 independent than {} distinct agents would be",
1935 overlap.join(", "),
1936 self.roles.judges.len()
1937 );
1938 self.state.event("prep", note);
1939 }
1940
1941 self.state.event(
1942 "prep",
1943 format!(
1944 "{} candidates, {} judges, base {} ({})",
1945 self.state.candidates.len(),
1946 self.roles.judges.len(),
1947 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1948 self.state.base_branch
1949 ),
1950 );
1951 self.state.prep_complete = Some(true);
1952 self.state.status = RunStatus::Implementing;
1953 self.state.save()?;
1954 Ok(())
1955 }
1956
1957 async fn advise(&mut self) -> Result<()> {
1990 let implement_untouched = self
1991 .state
1992 .candidates
1993 .iter()
1994 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1995 if !self.state.config.graph.advise || self.state.advise_attempted {
1996 return Ok(());
1997 }
1998 if !implement_untouched {
1999 self.state.event(
2000 "advise",
2001 "skipping the design-deliberation stage: at least one \
2002 candidate already shows implementation progress, so this \
2003 run is past the point the stage exists to run before"
2004 .to_owned(),
2005 );
2006 self.state.advise_attempted = true;
2007 self.state.save()?;
2008 return Ok(());
2009 }
2010 let run_id = self.state.id.clone();
2011 let prompts = self.state.config.prompts.clone();
2012 let instruction = self.state.instruction.clone();
2013 let language = self.state.config.graph.language.clone();
2014 let root = self.state.worktree_root();
2015 let n = self.state.config.graph.advisors;
2016 let where_recorded = self.state.dir().join("run.json");
2017
2018 let seats = match self.state.config.advisors() {
2019 Ok(seats) if !seats.is_empty() => seats,
2020 Ok(_) => {
2021 self.state.event(
2022 "advise",
2023 format!(
2024 "[graph] advisors is 0; skipping the design-deliberation \
2025 stage and continuing without a synthesis brief (see {})",
2026 where_recorded.display()
2027 ),
2028 );
2029 self.state.advise_attempted = true;
2030 self.state.save()?;
2031 return Ok(());
2032 }
2033 Err(e) => {
2034 self.state.event(
2035 "advise",
2036 format!(
2037 "could not resolve advisor seats ({e:#}); continuing \
2038 without a design-deliberation brief (see {})",
2039 where_recorded.display()
2040 ),
2041 );
2042 self.state.advise_attempted = true;
2043 self.state.save()?;
2044 return Ok(());
2045 }
2046 };
2047
2048 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2049 let artifacts = agent::artifacts_dir(&self.state.dir());
2050 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2051
2052 let mut jobs = Vec::new();
2053 for (i, spec) in seats.iter().cloned().enumerate() {
2054 let seat_key = format!("advisor-{}", i + 1);
2055 let seat = self.seat(&seat_key, &spec.id);
2056 jobs.push(SeatJob {
2057 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2058 spec,
2059 seat,
2060 cwd: worktrees[i % worktrees.len()].clone(),
2061 timeout,
2062 allow_write: false,
2063 sessions: false,
2064 artifacts: artifacts.clone(),
2065 stem: seat_key,
2066 handover: None,
2067 });
2068 }
2069
2070 self.state.event(
2071 "advise",
2072 format!(
2073 "{} advisor seat(s) sketching a design in parallel",
2074 jobs.len()
2075 ),
2076 );
2077 let mut quota_losses = Vec::new();
2078 let cache = self.state.config.cache_dir();
2079 let ctx = WaveCtx {
2080 carry_seats: false,
2081 pauses: Some((&self.pause, &self.interrupt)),
2082 run: &run_id,
2083 node: "advise",
2084 prompts: &prompts,
2085 cache: cache.as_deref(),
2086 round: None,
2087 };
2088 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2089 let results = ask_json_wave::<Proposal>(
2090 jobs,
2091 Arc::clone(&self.sem),
2092 self.state.config.graph.retries,
2093 &advisor_roster,
2094 &ctx,
2095 &mut quota_losses,
2096 &mut self.state,
2097 &|p: &Proposal| p.validate(),
2098 )
2099 .await;
2100 if self.state.resource_block.is_some() {
2101 return self.stop_for_resource();
2102 }
2103 self.state.quota.extend(quota_losses);
2104
2105 let mut records = Vec::with_capacity(results.len());
2106 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2107 let agent_id = seat.agent.clone();
2108 self.state.seats.insert(seat.key.clone(), seat);
2109 match res {
2110 Ok((proposal, out)) => {
2111 self.state
2112 .event("advise", format!("advisor-{} proposed a design", i + 1));
2113 records.push(advise::AdvisorRecord::proposed(
2114 i + 1,
2115 agent_id,
2116 proposal,
2117 out.duration_ms,
2118 ));
2119 }
2120 Err(e) => {
2121 self.state.event(
2122 "advise",
2123 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2124 );
2125 records.push(advise::AdvisorRecord::failed(
2126 i + 1,
2127 agent_id,
2128 e.to_string(),
2129 ));
2130 }
2131 }
2132 }
2133
2134 let mut advice = advise::Advice {
2135 records,
2136 synthesis: None,
2137 };
2138 if advice.proposals().is_empty() {
2139 self.state.event(
2140 "advise",
2141 "no advisor produced a usable proposal; continuing without a \
2142 synthesis brief"
2143 .to_owned(),
2144 );
2145 } else {
2146 match self
2147 .synthesize_brief(
2148 &advice,
2149 &instruction,
2150 &language,
2151 &worktrees[0],
2152 &artifacts,
2153 &run_id,
2154 &prompts,
2155 cache.as_deref(),
2156 )
2157 .await
2158 {
2159 Ok(Some(text)) => {
2160 self.state.event(
2161 "advise",
2162 "synthesized a design brief for the implementer".to_owned(),
2163 );
2164 advice.synthesis = Some(text);
2165 }
2166 Ok(None) => {
2167 self.state.event(
2168 "advise",
2169 "the synthesis seat produced nothing usable; continuing \
2170 without a design brief"
2171 .to_owned(),
2172 );
2173 }
2174 Err(e) => {
2175 if e.is::<crate::repo_lock::ResourceBlocked>() {
2176 return Err(e);
2177 }
2178 self.state.event(
2179 "advise",
2180 format!("could not synthesize a design brief: {e:#}"),
2181 );
2182 }
2183 }
2184 }
2185 advise::apply_reflection(&mut advice);
2186
2187 self.state.advice = Some(advice);
2188 self.state.advise_attempted = true;
2189 self.state.save()?;
2190 Ok(())
2191 }
2192
2193 #[allow(clippy::too_many_arguments)]
2205 async fn synthesize_brief(
2206 &mut self,
2207 advice: &advise::Advice,
2208 instruction: &str,
2209 language: &str,
2210 cwd: &Path,
2211 artifacts: &Path,
2212 run_id: &str,
2213 prompts: &Prompts,
2214 cache: Option<&Path>,
2215 ) -> Result<Option<String>> {
2216 let chain = agent::pick_chain(
2217 &self.state.config.agents,
2218 self.state.config.roles.synthesizer.as_ref(),
2219 &agent::installed,
2220 "synthesizer",
2221 )?;
2222 let proposals = advice.proposals();
2223 let mut prompt = prompt::with_overlay(
2224 prompt::synthesize_brief(instruction, &proposals, language),
2225 prompts.overlay("advise"),
2226 );
2227 if cache.is_some() {
2228 prompt.push('\n');
2233 prompt.push_str(&prompt::build_cache_note("advise", false));
2234 }
2235 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2236 let mut last = None;
2241 for (n, spec) in chain.iter().enumerate() {
2242 if n > 0 {
2243 self.state
2244 .event("advise", format!("synthesis falling back to {}", spec.id));
2245 }
2246 let mut seat = self.seat("advise-synthesis", &spec.id);
2247 let wait = self.repository_wait();
2248 let outcome = wait
2249 .scope(agent::invoke(
2250 spec,
2251 &mut seat,
2252 &Invocation {
2253 scope: crate::agent::InvocationScope::Repository,
2254 coordination: self.state.config.coordination.as_ref(),
2255 fetched_base: None,
2256 cwd,
2257 prompt: &prompt,
2258 timeout,
2259 allow_write: false,
2260 unsandboxed: false,
2261 sessions: false,
2262 artifacts,
2263 stem: &if n == 0 {
2264 "advise-synthesis".to_owned()
2265 } else {
2266 format!("advise-synthesis-{}", spec.id)
2267 },
2268 run: run_id,
2269 node: "advise",
2270 cache_dir: None,
2271 attachments: &[],
2272 writable: &[],
2273 },
2274 ))
2275 .await;
2276 if outcome.is_ok() {
2277 self.state.seats.insert(seat.key.clone(), seat);
2278 }
2279 let advance = agent::chain_advances(&outcome);
2280 last = Some(outcome);
2281 if !advance {
2282 break;
2283 }
2284 }
2285 let out = last.expect("a chain holds at least one agent")?;
2288 if !out.usable() {
2289 return Ok(None);
2290 }
2291 let text =
2292 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2293 Ok((!text.trim().is_empty()).then_some(text))
2294 }
2295
2296 async fn implement(&mut self) -> Result<()> {
2299 self.persist_candidates().await?;
2300 let run_id = self.state.id.clone();
2305 let prompts = self.state.config.prompts.clone();
2306 let todo: Vec<usize> = self
2307 .state
2308 .candidates
2309 .iter()
2310 .enumerate()
2311 .filter(|(_, c)| {
2312 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2313 && c.failed.is_none()
2314 && !c.empty
2315 })
2316 .map(|(i, _)| i)
2317 .collect();
2318 if todo.is_empty() {
2319 return self.after_implement();
2320 }
2321 self.state.status = RunStatus::Implementing;
2322
2323 let language = self.state.config.graph.language.clone();
2324 let github_language = self.state.config.graph.github_language.clone();
2325 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2326 let sessions = self.state.config.graph.sessions;
2327 let artifacts = agent::artifacts_dir(&self.state.dir());
2328 let brief = self
2332 .state
2333 .advice
2334 .as_ref()
2335 .and_then(|a| a.synthesis.as_deref())
2336 .map(str::to_owned);
2337 let attachments = self.state.attachments.clone();
2338
2339 let mut jobs = Vec::new();
2340 for &i in &todo {
2341 let (index, label, worktree) = {
2342 let c = &self.state.candidates[i];
2343 (c.index, c.label, c.worktree.clone())
2344 };
2345 let spec = self.roles.implementers[index].clone();
2346 let seat_key = format!("impl-{label}");
2347 let seat = self.seat(&seat_key, &spec.id);
2348 let instruction = seeded_instruction(&self.state);
2349 jobs.push(SeatJob {
2350 spec,
2351 seat,
2352 prompt: prompt::implement_in(
2353 &instruction,
2354 &worktree.to_string_lossy(),
2355 &language,
2356 &github_language,
2357 brief.as_deref(),
2358 &attachments,
2359 ),
2360 cwd: worktree,
2361 timeout,
2362 allow_write: true,
2363 sessions,
2364 artifacts: artifacts.clone(),
2365 stem: format!("impl-{label}"),
2366 handover: None,
2367 });
2368 }
2369
2370 self.state.event(
2371 "implement",
2372 format!("{} candidates in parallel", jobs.len()),
2373 );
2374 let mut sent = jobs.clone();
2380 let cache = self.state.config.cache_dir();
2381 let ctx = WaveCtx {
2382 carry_seats: false,
2383 pauses: Some((&self.pause, &self.interrupt)),
2384 run: &run_id,
2385 node: "implement",
2386 prompts: &prompts,
2387 cache: cache.as_deref(),
2388 round: None,
2389 };
2390 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2391 if !wave_blocked(&results) {
2392 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2393 .await;
2394 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2395 .await?;
2396 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2397 .await;
2398 }
2399 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2400
2401 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2402 && results
2403 .iter()
2404 .any(|(_, _, out)| FailClass::of(out).is_some());
2405 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2406 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2407 if self
2411 .state
2412 .candidate_preservation
2413 .contains_key(&self.state.candidates[i].label)
2414 {
2415 self.state.seats.insert(seat.key.clone(), seat);
2416 }
2417 continue;
2418 }
2419 let seat_key = seat.key.clone();
2420 let agent = seat.agent.clone();
2429 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2430 self.state.seats.insert(seat.key.clone(), seat);
2431 let label = self.state.candidates[i].label;
2432 self.state.interrupted_implementations.remove(&label);
2433
2434 let (summary, duration, failed, verified_claim) = match out {
2435 AgentOutcome::Ok(o) => {
2436 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2437 let failed = (!o.usable()).then(|| {
2438 if o.timed_out {
2439 "agent timed out".to_owned()
2440 } else {
2441 format!(
2442 "agent exited with {}",
2443 crate::notices::exit_status(o.exit_code)
2444 )
2445 }
2446 });
2447 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2448 (text, o.duration_ms, failed, verified_claim)
2449 }
2450 AgentOutcome::Dropped(o) => {
2456 let why = o
2457 .dropped
2458 .as_ref()
2459 .map(|d| d.why.as_str())
2460 .unwrap_or("the CLI ended the stream without delivering its answer");
2461 (
2462 String::new(),
2463 o.duration_ms,
2464 Some(format!("the CLI dropped the stream ({why})")),
2465 None,
2466 )
2467 }
2468 AgentOutcome::Quota(o) => {
2469 self.state.quota.push(QuotaLoss {
2470 seat: seat_key,
2471 node: "implement".to_owned(),
2472 at: Timestamp::now(),
2473 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2474 });
2475 (
2476 String::new(),
2477 o.duration_ms,
2478 Some("rate limited (quota); produced no change".to_owned()),
2479 None,
2480 )
2481 }
2482 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2483 (String::new(), 0, Some(e), None)
2484 }
2485 };
2486
2487 let c = &mut self.state.candidates[i];
2490 if !exhausted_the_fallback_chain {
2491 c.agent = agent;
2492 }
2493 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2494 c.duration_ms = duration;
2495 c.failed = failed;
2496 c.verified_noop = verified_claim;
2497 self.state
2498 .candidate_preservation
2499 .insert(label, "awaiting persistence".to_owned());
2500 }
2501 self.state.save()?;
2502 self.persist_candidates().await?;
2503
2504 if resource_blocked {
2505 return self.stop_for_resource();
2506 }
2507 if selected_failed {
2508 self.state.status = RunStatus::Blocked;
2509 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2510 self.state.save()?;
2511 bail!(
2512 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2513 );
2514 }
2515 self.after_implement()
2516 }
2517
2518 async fn persist_candidates(&mut self) -> Result<()> {
2520 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2521 for label in labels {
2522 let i = self
2523 .state
2524 .candidates
2525 .iter()
2526 .position(|c| c.label == label)
2527 .context("pending candidate is missing")?;
2528 let c = self.state.candidates[i].clone();
2529 let interrupted = self.state.interrupted_implementations.contains(&label);
2530 let result = async {
2531 let wait = crate::repo_lock::WaitPolicy::new(
2532 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2533 || false,
2534 );
2535 let _coordination =
2536 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2537 if let Err(e) = git::snapshot_uncommitted(
2540 &c.worktree,
2541 &self
2542 .state
2543 .dir()
2544 .join("artifacts")
2545 .join(format!("candidate-{label}-recovery")),
2546 )
2547 .await
2548 {
2549 self.state.event(
2550 "preserve",
2551 format!("candidate {label} snapshot incomplete: {e:#}"),
2552 );
2553 }
2554 let rescued = git::rescue_commit(
2555 &c.worktree,
2556 &format!("magi: candidate {label} (uncommitted work)"),
2557 )
2558 .await?;
2559 self.state.note_withheld("implement", &rescued.withheld);
2560 let base = &self.state.base_commit;
2561 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2562 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2563 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2564 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2565 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2566 if c.empty && commits > 0 && !patch.trim().is_empty()
2567 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2568 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2569 || self.state.tally.is_some())
2570 {
2571 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2575 }
2576 let c = &mut self.state.candidates[i];
2577 c.commits = commits;
2578 c.stat = stat;
2579 c.files = files;
2580 let no_changes = commits == 0 || patch.trim().is_empty();
2581 c.empty = no_changes && !interrupted;
2582 if interrupted && no_changes {
2583 c.failed = None;
2584 c.verified_noop = None;
2585 }
2586 if !c.empty {
2587 c.failed = None;
2588 c.verified_noop = None;
2589 }
2590 self.state.event(
2591 "preserve",
2592 format!("candidate {label}: {files} files, {commits} commits"),
2593 );
2594 Ok::<_, anyhow::Error>(())
2595 }
2596 .await;
2597 match result {
2598 Ok(()) => {
2599 self.state.candidate_preservation.remove(&label);
2600 if !interrupted || self.state.candidates[i].files > 0 {
2601 self.state.interrupted_implementations.remove(&label);
2602 }
2603 }
2604 Err(e) => {
2605 let reason = format!("candidate {label} preservation failed: {e:#}");
2606 self.state.event("preserve", &reason);
2607 self.state.candidate_preservation.insert(label, reason);
2608 }
2609 }
2610 if self.state.candidate_preservation.is_empty()
2611 && self.state.status == RunStatus::Blocked
2612 && self.state.candidates.iter().all(|c| c.failed.is_none())
2613 {
2614 self.state.status = RunStatus::Implementing;
2618 self.state.event(
2619 "preserve",
2620 "candidate persistence recovered; implementation can continue",
2621 );
2622 }
2623 self.state.save()?;
2624 }
2625 if !self.state.candidate_preservation.is_empty() {
2626 self.state.status = RunStatus::Blocked;
2627 self.state.save()?;
2628 bail!(
2629 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2630 );
2631 }
2632 Ok(())
2633 }
2634
2635 async fn resume_undelivered(
2663 &mut self,
2664 results: &mut [(usize, SeatState, AgentOutcome)],
2665 sent: &[SeatJob],
2666 prompts: &Prompts,
2667 run_id: &str,
2668 ) {
2669 for (wi, seat, out) in results.iter_mut() {
2670 let Some(dropped) = (match &*out {
2671 AgentOutcome::Dropped(o) => o.dropped.clone(),
2672 _ => None,
2673 }) else {
2674 continue;
2675 };
2676 let Some(job) = sent.get(*wi) else { continue };
2677 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2679 self.state.event(
2680 "implement",
2681 format!(
2682 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2683 work is in the tree",
2684 seat.key, dropped.output_tokens, dropped.why
2685 ),
2686 );
2687 continue;
2688 }
2689 if !has_context(&job.spec, seat, job.sessions) {
2697 self.state.event(
2698 "implement",
2699 format!(
2700 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2701 is no session left to resume",
2702 seat.key, dropped.output_tokens, dropped.why
2703 ),
2704 );
2705 continue;
2706 }
2707 self.state.event(
2708 "implement",
2709 format!(
2710 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2711 conversation",
2712 seat.key, dropped.output_tokens, dropped.why
2713 ),
2714 );
2715 let mut retry = job.clone();
2716 retry.seat = seat.clone();
2717 retry.prompt = prompt::resume_after_drop(&dropped.why);
2718 retry.timeout = retry_budget(job.timeout, true);
2719 retry.stem = format!("{}-resume", job.stem);
2720 let cache = self.state.config.cache_dir();
2721 let ctx = WaveCtx {
2722 carry_seats: false,
2723 pauses: Some((&self.pause, &self.interrupt)),
2724 run: run_id,
2725 node: "implement",
2726 prompts,
2727 cache: cache.as_deref(),
2728 round: None,
2729 };
2730 let (resumed_seat, resumed) =
2731 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2732 *seat = resumed_seat;
2733 *out = resumed;
2734 }
2735 }
2736
2737 async fn resume_seat_handovers(
2802 &mut self,
2803 results: &mut [(usize, SeatState, AgentOutcome)],
2804 sent: &mut [SeatJob],
2805 prompts: &Prompts,
2806 run_id: &str,
2807 ) -> Result<()> {
2808 let instruction = seeded_instruction(&self.state);
2809 let language = self.state.config.graph.language.clone();
2810 let github_language = self.state.config.graph.github_language.clone();
2811 let brief = self
2812 .state
2813 .advice
2814 .as_ref()
2815 .and_then(|a| a.synthesis.as_deref())
2816 .map(str::to_owned);
2817 let attachments = self.state.attachments.clone();
2818 for (wi, seat, out) in results.iter_mut() {
2819 let others: BTreeSet<String> = sent
2822 .iter()
2823 .enumerate()
2824 .filter(|(j, _)| j != wi)
2825 .map(|(_, j)| j.spec.id.clone())
2826 .collect();
2827 let Some(job) = sent.get_mut(*wi) else {
2828 continue;
2829 };
2830 let start = self
2835 .roles
2836 .implementer_roster
2837 .iter()
2838 .position(|s| s.id == job.spec.id)
2839 .unwrap_or(0);
2840 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2841 let mut fallback_attempt = 0usize;
2842 let mut prev: Option<FailClass> = None;
2843 while let Some(cur) = FailClass::of(&*out) {
2844 if !should_hand_over(prev.as_ref(), &cur) {
2845 break;
2846 }
2847 let Some(next) =
2848 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2849 .cloned()
2850 else {
2851 break;
2852 };
2853 tried.insert(next.id.clone());
2854 fallback_attempt += 1;
2855
2856 let coordination = self.coordinate(&job.cwd).await?;
2857 if let Ok(r) = git::rescue_commit(
2858 &job.cwd,
2859 &format!(
2860 "magi: candidate {} (uncommitted work before {} fallback)",
2861 seat.key,
2862 if cur == FailClass::Quota {
2863 "quota"
2864 } else {
2865 "handover"
2866 }
2867 ),
2868 )
2869 .await
2870 {
2871 self.state.note_withheld("implement", &r.withheld);
2872 }
2873 drop(coordination);
2874
2875 record_handover(
2876 &mut self.state,
2877 "implement",
2878 &seat.key,
2879 &seat.agent,
2880 &next.id,
2881 &cur,
2882 &fail_reason(&*out),
2883 );
2884 prev = Some(cur.clone());
2885
2886 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2887 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2888 job.spec = next.clone();
2896 let mut retry = job.clone();
2897 retry.seat = new_seat;
2898 retry.prompt = prompt::implement_in(
2899 &instruction,
2900 &job.cwd.to_string_lossy(),
2901 &language,
2902 &github_language,
2903 brief.as_deref(),
2904 &attachments,
2905 );
2906 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2907 let cache = self.state.config.cache_dir();
2908 let ctx = WaveCtx {
2909 carry_seats: false,
2910 pauses: Some((&self.pause, &self.interrupt)),
2911 run: run_id,
2912 node: "implement",
2913 prompts,
2914 cache: cache.as_deref(),
2915 round: None,
2916 };
2917 let (fallback_seat, fallback_out) = run_one(
2918 retry,
2919 Arc::clone(&self.sem),
2920 &ctx,
2921 &mut self.state,
2922 fallback_attempt,
2923 )
2924 .await;
2925 *seat = fallback_seat;
2926 *out = fallback_out;
2927 }
2928 }
2929 Ok(())
2930 }
2931
2932 async fn resume_unconfirmed_commands(
2956 &mut self,
2957 results: &mut [(usize, SeatState, AgentOutcome)],
2958 sent: &[SeatJob],
2959 prompts: &Prompts,
2960 run_id: &str,
2961 ) {
2962 for (wi, seat, out) in results.iter_mut() {
2963 let AgentOutcome::Ok(o) = &*out else {
2964 continue;
2965 };
2966 if !has_unconfirmed_command(&o.commands) {
2967 continue;
2968 }
2969 let Some(job) = sent.get(*wi) else { continue };
2970 if !has_context(&job.spec, seat, job.sessions) {
2971 self.state.event(
2972 "implement",
2973 format!(
2974 "{}: the reply named a command whose own CLI never confirmed the exit \
2975 status of, but there is no session left to resume",
2976 seat.key
2977 ),
2978 );
2979 continue;
2980 }
2981 self.state.event(
2982 "implement",
2983 format!(
2984 "{}: the reply named a command whose own CLI never confirmed the exit \
2985 status of; resuming the conversation",
2986 seat.key
2987 ),
2988 );
2989 let mut retry = job.clone();
2990 retry.seat = seat.clone();
2991 retry.prompt = prompt::resume_incomplete(
2992 "a command in your last reply had no confirmed exit status",
2993 );
2994 retry.timeout = retry_budget(job.timeout, true);
2995 retry.stem = format!("{}-confirm", job.stem);
2996 let cache = self.state.config.cache_dir();
2997 let ctx = WaveCtx {
2998 carry_seats: false,
2999 pauses: Some((&self.pause, &self.interrupt)),
3000 run: run_id,
3001 node: "implement",
3002 prompts,
3003 cache: cache.as_deref(),
3004 round: None,
3005 };
3006 let (resumed_seat, resumed) =
3007 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3008 *seat = resumed_seat;
3009 *out = resumed;
3010 }
3011 }
3012
3013 async fn continue_fix_report(
3034 &mut self,
3035 mut seat: SeatState,
3036 parse_err: String,
3037 job: &SeatJob,
3038 prompts: &Prompts,
3039 run_id: &str,
3040 round: usize,
3041 ) -> (
3042 SeatState,
3043 Option<FixReport>,
3044 Option<String>,
3045 ContinuationRecord,
3046 ) {
3047 let mut last_err = parse_err;
3048 let mut cumulative_wait_ms = 0u64;
3049 let mut attempts = 0usize;
3050 loop {
3051 if !has_context(&job.spec, &seat, job.sessions) {
3052 self.state.event(
3053 "fix",
3054 format!(
3055 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3056 session left to resume into"
3057 ),
3058 );
3059 let outcome = if attempts == 0 {
3060 ContinuationOutcome::NoSession
3061 } else {
3062 ContinuationOutcome::Exhausted
3063 };
3064 return (
3065 seat,
3066 None,
3067 Some(format!("unparsable fix report: {last_err}")),
3068 ContinuationRecord {
3069 attempts,
3070 cumulative_wait_ms,
3071 outcome,
3072 },
3073 );
3074 }
3075 if attempts >= MAX_FIX_CONTINUATIONS {
3076 self.state.event(
3077 "fix",
3078 format!(
3079 "round {round}: fixer's reply still had no adoption report after \
3080 {attempts} continuation(s) ({last_err}); giving up"
3081 ),
3082 );
3083 return (
3084 seat,
3085 None,
3086 Some(format!(
3087 "unparsable fix report after {attempts} continuation(s): {last_err}"
3088 )),
3089 ContinuationRecord {
3090 attempts,
3091 cumulative_wait_ms,
3092 outcome: ContinuationOutcome::Exhausted,
3093 },
3094 );
3095 }
3096 attempts += 1;
3097 self.state.event(
3098 "fix",
3099 format!(
3100 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3101 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3102 ),
3103 );
3104 let mut retry = job.clone();
3105 retry.seat = seat.clone();
3106 retry.prompt = prompt::resume_incomplete(&last_err);
3107 retry.timeout = retry_budget(job.timeout, true);
3108 retry.stem = format!("{}-continue{attempts}", job.stem);
3109 let cache = self.state.config.cache_dir();
3110 let ctx = WaveCtx {
3111 carry_seats: false,
3112 pauses: Some((&self.pause, &self.interrupt)),
3113 run: run_id,
3114 node: "fix",
3115 prompts,
3116 cache: cache.as_deref(),
3117 round: Some(round),
3118 };
3119 let (resumed_seat, resumed_out) = run_one(
3120 retry,
3121 Arc::clone(&self.sem),
3122 &ctx,
3123 &mut self.state,
3124 attempts,
3125 )
3126 .await;
3127 seat = resumed_seat;
3128 match resumed_out {
3129 AgentOutcome::Ok(o) => {
3130 cumulative_wait_ms += o.duration_ms;
3131 match verdict::extract_json::<FixReport>(&o.text) {
3132 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3133 self.state.event(
3134 "fix",
3135 format!(
3136 "round {round}: fixer's adoption report recovered after \
3137 {attempts} continuation(s)"
3138 ),
3139 );
3140 return (
3141 seat,
3142 Some(report),
3143 None,
3144 ContinuationRecord {
3145 attempts,
3146 cumulative_wait_ms,
3147 outcome: ContinuationOutcome::Resumed,
3148 },
3149 );
3150 }
3151 Ok(_) => {
3159 last_err = "the reply parsed, but it reported a command whose own CLI \
3160 never confirmed an exit status"
3161 .to_owned();
3162 }
3163 Err(e) => last_err = e.to_string(),
3164 }
3165 }
3166 AgentOutcome::Quota(o) => {
3167 cumulative_wait_ms += o.duration_ms;
3168 self.state.quota.push(QuotaLoss {
3169 seat: seat.key.clone(),
3170 node: "fix".to_owned(),
3171 at: Timestamp::now(),
3172 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3173 });
3174 self.state.event(
3175 "fix",
3176 format!(
3177 "round {round}: continuation rate limited (quota); not retrying now"
3178 ),
3179 );
3180 return (
3181 seat,
3182 None,
3183 Some("rate limited (quota) while recovering the fix report".to_owned()),
3184 ContinuationRecord {
3185 attempts,
3186 cumulative_wait_ms,
3187 outcome: ContinuationOutcome::QuotaLost,
3188 },
3189 );
3190 }
3191 AgentOutcome::Dropped(o) => {
3192 cumulative_wait_ms += o.duration_ms;
3193 let why = o
3194 .dropped
3195 .as_ref()
3196 .map(|d| d.why.as_str())
3197 .unwrap_or("the CLI ended the stream without delivering its answer");
3198 last_err = format!("the CLI dropped the stream ({why})");
3199 }
3200 AgentOutcome::Failed(e) => last_err = e,
3201 AgentOutcome::ResourceBlocked(e) => {
3204 return (
3205 seat,
3206 None,
3207 Some(format!("the continuation was not started: {e}")),
3208 ContinuationRecord {
3209 attempts: attempts.saturating_sub(1),
3210 cumulative_wait_ms,
3211 outcome: ContinuationOutcome::NoSession,
3212 },
3213 );
3214 }
3215 }
3216 }
3217 }
3218
3219 fn after_implement(&mut self) -> Result<()> {
3220 if !self.state.candidate_preservation.is_empty() {
3221 bail!("candidate preservation incomplete; judging is unsafe");
3222 }
3223 if self.state.leaks.is_empty() {
3225 let cfg = self.state.config.blind.clone();
3226 let mut leaks = Vec::new();
3227 for c in &self.state.candidates {
3228 let Some(patch) =
3229 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3230 else {
3231 continue;
3232 };
3233 leaks.extend(blind::scan(
3234 &format!("candidate {} patch", c.label),
3235 &patch,
3236 &cfg.vendor_tokens,
3237 ));
3238 }
3239 if !leaks.is_empty() {
3240 let summary = leaks
3241 .iter()
3242 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3243 .collect::<Vec<_>>()
3244 .join(", ");
3245 match cfg.on_leak {
3246 LeakPolicy::Fail => {
3247 self.state.status = RunStatus::Failed;
3248 self.state
3249 .event("blind", format!("vendor text in a patch: {summary}"));
3250 self.state.leaks = leaks;
3251 self.state.save()?;
3252 self.settle_questions();
3253 bail!(
3254 "blind.on_leak = \"fail\" and vendor text reached a \
3255 judged patch: {summary}"
3256 );
3257 }
3258 LeakPolicy::Redact => self.state.event(
3259 "blind",
3260 format!("redacting vendor text for judging: {summary}"),
3261 ),
3262 LeakPolicy::Warn => self.state.event(
3263 "blind",
3264 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3265 ),
3266 }
3267 self.state.leaks = leaks;
3268 }
3269 }
3270
3271 if self.state.viable().is_empty() {
3272 if self.state.all_candidates_verified_noop() {
3273 self.state.status = RunStatus::VerifiedNoop;
3284 self.state.save()?;
3285 self.settle_questions();
3286 return Ok(());
3287 }
3288 self.state.status = RunStatus::Failed;
3289 self.state.save()?;
3290 self.settle_questions();
3291 bail!("no candidate produced a change; nothing to judge");
3292 }
3293 self.state.status = RunStatus::Judging;
3294 self.state.save()?;
3295 Ok(())
3296 }
3297
3298 async fn judge(&mut self) -> Result<()> {
3301 let run_id = self.state.id.clone();
3306 let prompts = self.state.config.prompts.clone();
3307 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3308 return Ok(());
3309 }
3310 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3311 if viable.len() == 1 {
3312 self.state.judge_skipped = true;
3319 self.state.event(
3320 "judge",
3321 format!(
3322 "only candidate {} produced a change; judging skipped",
3323 viable[0].label
3324 ),
3325 );
3326 self.state.save()?;
3327 return Ok(());
3328 }
3329 self.state.status = RunStatus::Judging;
3330
3331 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3332 let language = self.state.config.graph.language.clone();
3333 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3334 let sessions = self.state.config.graph.sessions;
3335 let artifacts = agent::artifacts_dir(&self.state.dir());
3336 let root = self.state.worktree_root();
3337 let base_short = short(&self.state.base_commit);
3338
3339 let mut jobs = Vec::new();
3340 let mut orders = Vec::new();
3341 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3342 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3343 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3344 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3345 let seat_key = format!("judge-{}", j + 1);
3346 let seat = self.seat(&seat_key, &spec.id);
3347 jobs.push(SeatJob {
3348 prompt: prompt::judge(
3349 &self.state.instruction,
3350 &views,
3351 self.roles.judges.len(),
3352 &base_short,
3353 &language,
3354 ),
3355 spec,
3356 seat,
3357 cwd: root.join(format!("judge-{}", j + 1)),
3358 timeout,
3359 allow_write: false,
3360 sessions,
3361 artifacts: artifacts.clone(),
3362 stem: format!("judge-{}", j + 1),
3363 handover: None,
3364 });
3365 }
3366
3367 self.state.event(
3368 "judge",
3369 format!(
3370 "{} judges ranking {} candidates blind",
3371 jobs.len(),
3372 viable.len()
3373 ),
3374 );
3375 let labels_for_check = labels.clone();
3376 let mut quota_losses = Vec::new();
3377 let cache = self.state.config.cache_dir();
3378 let ctx = WaveCtx {
3379 carry_seats: false,
3380 pauses: Some((&self.pause, &self.interrupt)),
3381 run: &run_id,
3382 node: "judge",
3383 prompts: &prompts,
3384 cache: cache.as_deref(),
3385 round: None,
3386 };
3387 let results = ask_json_wave::<Ranking>(
3388 jobs,
3389 Arc::clone(&self.sem),
3390 self.state.config.graph.retries,
3391 &self.roles.judge_roster,
3392 &ctx,
3393 &mut quota_losses,
3394 &mut self.state,
3395 &move |r: &Ranking| r.validate(&labels_for_check),
3396 )
3397 .await;
3398 if self.state.resource_block.is_some() {
3399 return self.stop_for_resource();
3400 }
3401 self.state.quota.extend(quota_losses);
3402
3403 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3404 let agent_id = seat.agent.clone();
3405 self.state.seats.insert(seat.key.clone(), seat);
3406 let mut record = Judgement {
3407 judge: j + 1,
3408 seat: format!("judge-{}", j + 1),
3409 agent: agent_id,
3410 ranking: Vec::new(),
3411 reasons: BTreeMap::new(),
3412 confidence: None,
3413 strengths: BTreeMap::new(),
3414 order: orders[j].clone(),
3415 failed: None,
3416 duration_ms: 0,
3417 };
3418 match res {
3419 Ok((ranking, out)) => {
3420 record.ranking = ranking.normalized();
3421 record.reasons = ranking.reasons;
3422 record.strengths = ranking.strengths;
3423 record.confidence = ranking.confidence;
3424 record.duration_ms = out.duration_ms;
3425 self.state.event(
3426 "judge",
3427 format!(
3428 "judge {} ranked {}",
3429 j + 1,
3430 record.ranking.iter().collect::<String>()
3431 ),
3432 );
3433 }
3434 Err(e) => {
3435 record.failed = Some(e.to_string());
3436 self.state
3437 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3438 }
3439 }
3440 self.state.judgements.push(record);
3441 self.state.save()?;
3442 }
3443 Ok(())
3444 }
3445
3446 async fn deliberate(&mut self) -> Result<()> {
3449 let run_id = self.state.id.clone();
3454 let prompts = self.state.config.prompts.clone();
3455 if !self.state.deliberation.is_empty() {
3456 return Ok(());
3457 }
3458 let tops: Vec<char> = self
3459 .state
3460 .judgements
3461 .iter()
3462 .filter_map(|j| j.ranking.first().copied())
3463 .collect();
3464 let rounds = self.state.config.graph.deliberate_rounds;
3465 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3466 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3467 self.state.event(
3468 "deliberate",
3469 format!("judges agreed on {} outright; no deliberation", tops[0]),
3470 );
3471 }
3472 self.state.status = RunStatus::Voting;
3473 self.state.save()?;
3474 return Ok(());
3475 }
3476
3477 self.state.status = RunStatus::Deliberating;
3478 self.state.event(
3479 "deliberate",
3480 format!(
3481 "split: first choices were {} — opening {rounds} round(s)",
3482 tops.iter().collect::<String>()
3483 ),
3484 );
3485
3486 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3487 let language = self.state.config.graph.language.clone();
3488 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3489 let sessions = self.state.config.graph.sessions;
3490 let artifacts = agent::artifacts_dir(&self.state.dir());
3491 let root = self.state.worktree_root();
3492 let base_short = short(&self.state.base_commit);
3493
3494 for round in 1..=rounds {
3498 let mut turns: Vec<DeliberationTurn> = Vec::new();
3499 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3500 if self.state.judgements[j].failed.is_some() {
3501 continue;
3502 }
3503 let seat_key = format!("judge-{}", j + 1);
3504 let spec = self.occupant(&seat_key, spec);
3505 let mut seat = self.seat(&seat_key, &spec.id);
3506 let transcript = self.transcript(&turns, j);
3507 let build = |context: Option<&str>| {
3508 prompt::deliberate(
3509 &self.state.instruction,
3510 context,
3511 &transcript,
3512 round,
3513 rounds,
3514 &language,
3515 )
3516 };
3517 let block = self.candidate_block(&viable, &base_short);
3518 let full = build(Some(&block));
3519 let text = if has_context(&spec, &seat, sessions) {
3520 build(None)
3521 } else {
3522 full.clone()
3523 };
3524 let job = SeatJob {
3525 spec,
3526 seat: seat.clone(),
3527 prompt: text,
3528 cwd: root.join(format!("judge-{}", j + 1)),
3529 timeout,
3530 allow_write: false,
3531 sessions,
3532 artifacts: artifacts.clone(),
3533 stem: format!("delib-{round}-judge-{}", j + 1),
3534 handover: Some(full),
3535 };
3536 let cache = self.state.config.cache_dir();
3537 let ctx = WaveCtx {
3538 carry_seats: false,
3539 pauses: Some((&self.pause, &self.interrupt)),
3540 run: &run_id,
3541 node: "deliberate",
3542 prompts: &prompts,
3543 cache: cache.as_deref(),
3544 round: None,
3545 };
3546 let mut losses = Vec::new();
3550 let mut results = ask_wave_with::<String>(
3551 vec![job],
3552 Arc::clone(&self.sem),
3553 0,
3554 &self.roles.judge_roster,
3555 &ctx,
3556 &mut losses,
3557 &mut self.state,
3558 &|text: &str| {
3559 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3560 },
3561 )
3562 .await;
3563 self.state.quota.extend(losses);
3564 let (updated, res, _) = results.pop().expect("one job in, one result out");
3565 seat = updated;
3566 let agent_id = seat.agent.clone();
3567 self.state.seats.insert(seat.key.clone(), seat);
3568 let body = match res {
3569 Ok((body, _)) => body,
3570 Err(e) => {
3573 self.state
3574 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3575 continue;
3576 }
3577 };
3578 let tentative = verdict::extract_json::<Position>(&body)
3579 .ok()
3580 .and_then(|p| p.tentative)
3581 .and_then(|s| s.trim().chars().next())
3582 .map(|c| c.to_ascii_uppercase());
3583 self.state.event(
3584 "deliberate",
3585 format!(
3586 "round {round}: judge {} now favours {}",
3587 j + 1,
3588 tentative.map_or("—".to_owned(), |c| c.to_string())
3589 ),
3590 );
3591 turns.push(DeliberationTurn {
3592 judge: j + 1,
3593 agent: agent_id,
3594 body: blind::sanitize_prose(&body, &self.state.config.blind),
3595 tentative,
3596 });
3597 }
3598 self.state
3599 .deliberation
3600 .push(DeliberationRound { round, turns });
3601 self.state.save()?;
3602 }
3603
3604 self.state.status = RunStatus::Voting;
3605 self.state.save()?;
3606 Ok(())
3607 }
3608
3609 async fn vote(&mut self) -> Result<()> {
3612 let run_id = self.state.id.clone();
3617 let prompts = self.state.config.prompts.clone();
3618 if !self.state.votes.is_empty() {
3619 return Ok(());
3620 }
3621 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3622 if viable.len() == 1 {
3623 return Ok(());
3624 }
3625 self.state.status = RunStatus::Voting;
3626
3627 let language = self.state.config.graph.language.clone();
3628 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3629 let sessions = self.state.config.graph.sessions;
3630 let artifacts = agent::artifacts_dir(&self.state.dir());
3631 let root = self.state.worktree_root();
3632 let base_short = short(&self.state.base_commit);
3633 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3634
3635 let mut jobs = Vec::new();
3636 let mut seats_at = Vec::new();
3637 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3638 if self
3639 .state
3640 .judgements
3641 .get(j)
3642 .is_some_and(|r| r.failed.is_some())
3643 {
3644 continue;
3645 }
3646 let seat_key = format!("judge-{}", j + 1);
3647 let spec = self.occupant(&seat_key, spec);
3648 let seat = self.seat(&seat_key, &spec.id);
3649 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3650 let text = if has_context(&spec, &seat, sessions) {
3651 prompt::final_vote(&viable, &language)
3652 } else {
3653 full.clone()
3654 };
3655 jobs.push(SeatJob {
3656 spec,
3657 seat,
3658 prompt: text,
3659 cwd: root.join(format!("judge-{}", j + 1)),
3660 timeout,
3661 allow_write: false,
3662 sessions,
3663 artifacts: artifacts.clone(),
3664 stem: format!("vote-judge-{}", j + 1),
3665 handover: Some(full),
3666 });
3667 seats_at.push(j);
3668 }
3669
3670 self.state.event(
3671 "vote",
3672 format!(
3673 "collecting {} final votes one by one, privately",
3674 jobs.len()
3675 ),
3676 );
3677 let allowed = viable.clone();
3678 let mut quota_losses = Vec::new();
3679 let cache = self.state.config.cache_dir();
3680 let ctx = WaveCtx {
3681 carry_seats: false,
3682 pauses: Some((&self.pause, &self.interrupt)),
3683 run: &run_id,
3684 node: "vote",
3685 prompts: &prompts,
3686 cache: cache.as_deref(),
3687 round: None,
3688 };
3689 let results = ask_json_wave::<FinalVote>(
3690 jobs,
3691 Arc::clone(&self.sem),
3692 self.state.config.graph.retries,
3693 &self.roles.judge_roster,
3694 &ctx,
3695 &mut quota_losses,
3696 &mut self.state,
3697 &move |v: &FinalVote| match v.label() {
3698 Some(c) if allowed.contains(&c) => Ok(()),
3699 other => bail!("vote {other:?} is not one of {allowed:?}"),
3700 },
3701 )
3702 .await;
3703 if self.state.resource_block.is_some() {
3704 return self.stop_for_resource();
3705 }
3706 self.state.quota.extend(quota_losses);
3707
3708 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3709 let agent_id = seat.agent.clone();
3710 self.state.seats.insert(seat.key.clone(), seat);
3711 let initial = self
3712 .state
3713 .judgements
3714 .get(j)
3715 .and_then(|r| r.ranking.first().copied());
3716 let mut record = VoteRecord {
3717 judge: j + 1,
3718 agent: agent_id,
3719 vote: None,
3720 reason: String::new(),
3721 changed: false,
3722 };
3723 match res {
3724 Ok((v, _)) => {
3725 record.vote = v.label();
3726 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3727 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3728 self.state.event(
3729 "vote",
3730 format!(
3731 "judge {} voted {}{}",
3732 j + 1,
3733 record.vote.unwrap_or('?'),
3734 if record.changed { " (changed)" } else { "" }
3735 ),
3736 );
3737 }
3738 Err(e) => {
3739 self.state
3740 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3741 }
3742 }
3743 self.state.votes.push(record);
3744 self.state.save()?;
3745 }
3746 Ok(())
3747 }
3748
3749 fn tally(&mut self) -> Result<()> {
3752 if self.state.tally.is_some() {
3753 return Ok(());
3754 }
3755 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3756 let tops: Vec<char> = self
3757 .state
3758 .judgements
3759 .iter()
3760 .filter_map(|j| j.ranking.first().copied())
3761 .collect();
3762 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3763
3764 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3767 let mut cast: Vec<char> = Vec::new();
3768 for (i, j) in self.state.judgements.iter().enumerate() {
3769 let vote = self
3770 .state
3771 .votes
3772 .iter()
3773 .find(|v| v.judge == i + 1)
3774 .and_then(|v| v.vote)
3775 .or_else(|| j.ranking.first().copied());
3776 if let Some(v) = vote {
3777 *first_choice.entry(v).or_insert(0) += 1;
3778 cast.push(v);
3779 }
3780 }
3781
3782 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3783 for j in &self.state.judgements {
3784 let n = j.ranking.len();
3785 for (pos, label) in j.ranking.iter().enumerate() {
3786 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3787 }
3788 }
3789
3790 let best = first_choice.values().copied().max().unwrap_or(0);
3791 let mut leaders: Vec<char> = first_choice
3792 .iter()
3793 .filter(|(_, v)| **v == best)
3794 .map(|(k, _)| *k)
3795 .collect();
3796 let mut tie_break = None;
3797 if leaders.len() > 1 {
3798 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3799 let borda_leaders: Vec<char> = leaders
3800 .iter()
3801 .copied()
3802 .filter(|l| borda[l] == top_borda)
3803 .collect();
3804 tie_break = Some(if borda_leaders.len() == 1 {
3805 format!(
3806 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3807 leaders.len()
3808 )
3809 } else {
3810 format!(
3811 "{} way tie on both first-choice votes and Borda points, broken by label order",
3812 leaders.len()
3813 )
3814 });
3815 leaders = borda_leaders;
3816 leaders.sort_unstable();
3817 }
3818 let winner = *leaders
3819 .first()
3820 .or(viable.first())
3821 .context("no candidate to declare a winner from")?;
3822
3823 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3824 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3825 let deliberated = !self.state.deliberation.is_empty();
3826
3827 let quota_seats: std::collections::BTreeSet<&str> =
3831 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3832 let mut present = 0usize;
3833 for (i, j) in self.state.judgements.iter().enumerate() {
3834 if quota_seats.contains(j.seat.as_str()) {
3835 continue;
3836 }
3837 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3838 let voted = self
3839 .state
3840 .votes
3841 .iter()
3842 .any(|v| v.judge == i + 1 && v.vote.is_some());
3843 if ranked || voted {
3844 present += 1;
3845 }
3846 }
3847 let needs_quorum = viable.len() > 1;
3853 let judges_total = if needs_quorum {
3854 self.roles.judges.len()
3855 } else {
3856 0
3857 };
3858 let quorum = if needs_quorum {
3859 judges_total / 2 + 1
3860 } else {
3861 0
3862 };
3863 let met_quorum = !needs_quorum || present >= quorum;
3864 let uncontested = (!needs_quorum).then(|| {
3865 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3866 });
3867
3868 self.state.event(
3869 "tally",
3870 match &uncontested {
3871 Some(reason) => format!("winner {winner} — {reason}"),
3872 None => format!(
3873 "winner {winner} — votes {} | initial {} | {} changed | \
3874 {present}/{judges_total} judges{}",
3875 first_choice
3876 .iter()
3877 .map(|(k, v)| format!("{k}:{v}"))
3878 .collect::<Vec<_>>()
3879 .join(" "),
3880 if unanimous_initial {
3881 "unanimous"
3882 } else {
3883 "split"
3884 },
3885 changed_votes,
3886 if met_quorum {
3887 String::new()
3888 } else {
3889 format!(" — below quorum ({quorum} required)")
3890 },
3891 ),
3892 },
3893 );
3894 if !met_quorum {
3895 self.state.event(
3896 "stall",
3897 format!(
3898 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3899 the run stops here, resumable"
3900 ),
3901 );
3902 }
3903 self.state.tally = Some(Tally {
3904 first_choice,
3905 borda,
3906 winner,
3907 rankings: tops.len(),
3908 unanimous_initial,
3909 deliberated,
3910 changed_votes,
3911 unanimous_final,
3912 tie_break,
3913 judges: judges_total,
3914 present,
3915 quorum,
3916 met_quorum,
3917 uncontested,
3918 });
3919 self.state.status = if met_quorum {
3920 RunStatus::Reviewing
3921 } else {
3922 RunStatus::Stalled
3923 };
3924 self.state.save()?;
3925 Ok(())
3926 }
3927
3928 #[allow(clippy::too_many_lines)]
3949 async fn recover_stall(&mut self) -> Result<bool> {
3950 let run_id = self.state.id.clone();
3955 let prompts = self.state.config.prompts.clone();
3956 let quota_seats: BTreeSet<&str> =
3961 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3962 let absent: Vec<String> = self
3963 .state
3964 .judgements
3965 .iter()
3966 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3967 .map(|j| j.seat.clone())
3968 .collect();
3969 if absent.is_empty() {
3970 return Ok(false);
3971 }
3972 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3973 if viable.len() <= 1 {
3974 return Ok(false);
3975 }
3976 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3977 let language = self.state.config.graph.language.clone();
3978 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3979 let sessions = self.state.config.graph.sessions;
3980 let artifacts = agent::artifacts_dir(&self.state.dir());
3981 let root = self.state.worktree_root();
3982 let base_short = short(&self.state.base_commit);
3983 let candidates: Vec<Candidate> = viable.clone();
3984
3985 let mut positions: Vec<usize> = absent
3987 .iter()
3988 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3989 .collect();
3990 if positions.is_empty() {
3991 return Ok(false);
3992 }
3993 positions.sort_unstable();
3994 positions.dedup();
3995
3996 let mut judge_jobs = Vec::new();
3998 for &j in &positions {
3999 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4000 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4001 let seat_key = format!("judge-{}", j + 1);
4002 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4003 let seat = self.seat(&seat_key, &spec.id);
4004 judge_jobs.push(SeatJob {
4005 spec,
4006 seat,
4007 prompt: prompt::judge(
4008 &self.state.instruction,
4009 &views,
4010 self.roles.judges.len(),
4011 &base_short,
4012 &language,
4013 ),
4014 cwd: root.join(seat_key),
4015 timeout,
4016 allow_write: false,
4017 sessions,
4018 artifacts: artifacts.clone(),
4019 stem: format!("judge-{}-recover", j + 1),
4020 handover: None,
4021 });
4022 }
4023
4024 let labels_for_check = labels.clone();
4025 let mut judge_losses = Vec::new();
4026 let retries = self.state.config.graph.retries;
4027 let cache = self.state.config.cache_dir();
4028 let ctx = WaveCtx {
4029 carry_seats: false,
4030 pauses: Some((&self.pause, &self.interrupt)),
4031 run: &run_id,
4032 node: "judge",
4033 prompts: &prompts,
4034 cache: cache.as_deref(),
4035 round: None,
4036 };
4037 let results = ask_json_wave::<Ranking>(
4038 judge_jobs,
4039 Arc::clone(&self.sem),
4040 retries,
4041 &self.roles.judge_roster,
4042 &ctx,
4043 &mut judge_losses,
4044 &mut self.state,
4045 &move |r: &Ranking| r.validate(&labels_for_check),
4046 )
4047 .await;
4048 if self.state.resource_block.is_some() {
4049 self.stop_for_resource()?;
4050 return Ok(false);
4051 }
4052
4053 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4055 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4056 let agent_id = seat.agent.clone();
4057 self.state.seats.insert(seat.key.clone(), seat);
4058 let record = &mut self.state.judgements[j];
4059 match res {
4060 Ok((ranking, out)) => {
4061 record.agent = agent_id;
4062 record.ranking = ranking.normalized();
4063 record.reasons = ranking.reasons;
4064 record.strengths = ranking.strengths;
4065 record.confidence = ranking.confidence;
4066 record.failed = None;
4067 record.duration_ms = out.duration_ms;
4068 recovered.insert(j);
4069 self.state.event(
4070 "recover",
4071 format!("judge {} ranked again after the limit", j + 1),
4072 );
4073 }
4074 Err(e) => {
4075 self.state
4076 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4077 }
4078 }
4079 }
4080
4081 let mut vote_jobs = Vec::new();
4083 let mut vote_pos: Vec<usize> = Vec::new();
4084 for &j in &recovered {
4085 let seat_key = format!("judge-{}", j + 1);
4086 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4087 let seat = self.seat(&seat_key, &spec.id);
4088 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4089 let text = if has_context(&spec, &seat, sessions) {
4090 prompt::final_vote(&labels, &language)
4091 } else {
4092 full.clone()
4093 };
4094 vote_jobs.push(SeatJob {
4095 spec,
4096 seat,
4097 prompt: text,
4098 cwd: root.join(seat_key),
4099 timeout,
4100 allow_write: false,
4101 sessions,
4102 artifacts: artifacts.clone(),
4103 stem: format!("vote-judge-{}-recover", j + 1),
4104 handover: Some(full),
4105 });
4106 vote_pos.push(j);
4107 }
4108 let allowed = labels.clone();
4109 let mut vote_losses = Vec::new();
4110 let vote_retries = self.state.config.graph.retries;
4111 let vote_cache = self.state.config.cache_dir();
4112 let ctx = WaveCtx {
4113 carry_seats: false,
4114 pauses: Some((&self.pause, &self.interrupt)),
4115 run: &run_id,
4116 node: "vote",
4117 prompts: &prompts,
4118 cache: vote_cache.as_deref(),
4119 round: None,
4120 };
4121 let votes = ask_json_wave::<FinalVote>(
4122 vote_jobs,
4123 Arc::clone(&self.sem),
4124 vote_retries,
4125 &self.roles.judge_roster,
4126 &ctx,
4127 &mut vote_losses,
4128 &mut self.state,
4129 &move |v: &FinalVote| match v.label() {
4130 Some(c) if allowed.contains(&c) => Ok(()),
4131 other => bail!("vote {other:?} is not one of {allowed:?}"),
4132 },
4133 )
4134 .await;
4135 if self.state.resource_block.is_some() {
4136 self.stop_for_resource()?;
4137 return Ok(false);
4138 }
4139 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4140 let agent_id = seat.agent.clone();
4141 self.state.seats.insert(seat.key.clone(), seat);
4142 match res {
4143 Ok((v, _)) => {
4144 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4145 rec.vote = v.label();
4146 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4147 } else {
4148 self.state.votes.push(VoteRecord {
4149 judge: j + 1,
4150 agent: agent_id,
4151 vote: v.label(),
4152 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4153 changed: false,
4154 });
4155 }
4156 self.state.event(
4157 "recover",
4158 format!("judge {} voted again after the limit", j + 1),
4159 );
4160 }
4161 Err(e) => {
4162 self.state
4163 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4164 }
4165 }
4166 }
4167
4168 let recovered_keys: BTreeSet<String> = recovered
4172 .iter()
4173 .map(|&j| format!("judge-{}", j + 1))
4174 .collect();
4175 self.state
4176 .quota
4177 .retain(|q| !recovered_keys.contains(&q.seat));
4178 for loss in judge_losses.into_iter().chain(vote_losses) {
4182 if recovered_keys.contains(&loss.seat) {
4183 continue;
4184 }
4185 self.state.quota.retain(|q| q.seat != loss.seat);
4186 self.state.quota.push(loss);
4187 }
4188
4189 self.state.tally = None;
4191 self.tally()?;
4192 Ok(self
4193 .state
4194 .tally
4195 .as_ref()
4196 .map(|t| t.met_quorum)
4197 .unwrap_or(false))
4198 }
4199
4200 async fn graft(&mut self) -> Result<()> {
4216 let Err(e) = self.graft_inner().await else {
4217 return Ok(());
4218 };
4219 if e.is::<crate::repo_lock::ResourceBlocked>() {
4220 return Err(e);
4221 }
4222 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4223 self.state.status = RunStatus::Blocked;
4224 self.state.event("graft", format!("blocked: {e:#}"));
4225 self.state.save()?;
4226 return Err(e);
4227 }
4228 if let Some(rec) = self.state.graft.clone()
4233 && !rec.outcome.settled()
4234 {
4235 if let Some(winner) = self.state.winner().cloned() {
4236 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4237 if re.is::<crate::repo_lock::ResourceBlocked>() {
4238 return Err(re);
4239 }
4240 self.state.status = RunStatus::Blocked;
4241 self.state.event("graft", format!("blocked: {re:#}"));
4242 self.state.save()?;
4243 return Err(re);
4244 }
4245 }
4246 if let Some(g) = self.state.graft.as_mut() {
4247 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4248 }
4249 }
4250 self.state.event("graft", format!("skipped: {e:#}"));
4251 if let Err(e) = self.state.save() {
4252 tracing::warn!("save after graft: {e:#}");
4253 }
4254 Ok(())
4255 }
4256
4257 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4260 let _coordination = self.coordinate(wt).await?;
4261 crate::recovery::restore_repair(wt, pre)
4262 .await
4263 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4264 }
4265
4266 fn graft_skipped(&mut self, why: &str) {
4268 let message = format!("skipped: {why}");
4269 if !self
4270 .state
4271 .events
4272 .iter()
4273 .any(|e| e.node == "graft" && e.message == message)
4274 {
4275 self.state.event("graft", message);
4276 }
4277 }
4278
4279 async fn graft_inner(&mut self) -> Result<()> {
4280 if let Some(rec) = self.state.graft.clone()
4283 && !rec.outcome.settled()
4284 {
4285 if self.state.reviews.is_empty()
4289 && let Some(winner) = self.state.winner().cloned()
4290 {
4291 self.graft_restore(&winner.worktree, &rec.pre).await?;
4292 }
4293 if rec.outcome == GraftOutcome::WaitingForRestore {
4294 self.state.graft = None;
4295 self.state.save()?;
4296 } else {
4297 if let Some(g) = self.state.graft.as_mut() {
4298 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4299 }
4300 self.state.event(
4301 "graft",
4302 "rolled back: the previous round was interrupted; the winner is untouched",
4303 );
4304 self.state.save()?;
4305 return Ok(());
4306 }
4307 }
4308 let plan = match graft::plan(&self.state) {
4309 Ok(plan) => plan,
4310 Err(skip) => {
4311 if skip != graft::Skip::AlreadyDone {
4312 self.graft_skipped(skip.reason());
4313 self.state.save()?;
4314 }
4315 return Ok(());
4316 }
4317 };
4318 let Some(winner) = self.state.winner().cloned() else {
4319 return Ok(());
4320 };
4321 let wt = winner.worktree.clone();
4322 if !git::is_clean(&wt).await? {
4323 self.graft_skipped("the winner's worktree has uncommitted changes");
4324 self.state.save()?;
4325 return Ok(());
4326 }
4327 let pre = git::rev_parse(&wt, "HEAD").await?;
4328 self.state.graft = Some(GraftRecord {
4331 pre: pre.clone(),
4332 post: None,
4333 lines: None,
4334 limit: None,
4335 agent: None,
4336 outcome: GraftOutcome::Running,
4337 requested: graft::requested(&plan),
4338 adoption: None,
4339 adoption_error: None,
4340 });
4341 self.state.event(
4342 "graft",
4343 format!(
4344 "porting strengths of candidate(s) {} onto {}",
4345 plan.strengths
4346 .iter()
4347 .map(|(l, _, _)| l.to_string())
4348 .collect::<Vec<_>>()
4349 .join(", "),
4350 plan.winner
4351 ),
4352 );
4353 self.state.save()?;
4354
4355 let fix_prompt = prompt::graft(
4356 &self.state.instruction,
4357 plan.winner,
4358 &plan.strengths,
4359 &self.state.config.graph.language,
4360 );
4361 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4362 let sessions = self.state.config.graph.sessions;
4363 let artifacts = agent::artifacts_dir(&self.state.dir());
4364 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4365 let (_, seat, out) = self
4366 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4367 SeatJob {
4368 prompt: fix_prompt.clone(),
4369 spec,
4370 seat,
4371 cwd: wt.clone(),
4372 timeout,
4373 allow_write: true,
4374 sessions,
4375 artifacts: artifacts.clone(),
4376 stem: "graft".to_owned(),
4377 handover: None,
4378 }
4379 })
4380 .await;
4381 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4382 self.state.graft.as_mut().expect("graft record").outcome =
4385 GraftOutcome::WaitingForRestore;
4386 return self.stop_for_resource();
4387 }
4388 let mut why: Option<String> = None;
4389 match &out {
4390 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4391 why = Some("the fixer produced nothing usable".to_owned());
4392 }
4393 AgentOutcome::Ok(_) => {}
4394 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4395 AgentOutcome::Quota(o) => {
4396 self.state.quota.push(QuotaLoss {
4397 seat: seat.key.clone(),
4398 node: "graft".to_owned(),
4399 at: Timestamp::now(),
4400 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4401 });
4402 why = Some("rate limited (quota)".to_owned());
4403 }
4404 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4405 why = Some(format!(
4406 "the fixer failed: {}",
4407 e.lines()
4408 .next()
4409 .unwrap_or("")
4410 .chars()
4411 .take(160)
4412 .collect::<String>()
4413 ));
4414 }
4415 }
4416 let agent_id = seat.agent.clone();
4417 self.state.seats.insert(seat.key.clone(), seat);
4418
4419 let mut post = None;
4420 let mut lines = None;
4421 let _coordination = self.coordinate(&wt).await?;
4422 if why.is_none() {
4423 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4424 if let Ok(r) = &rescued {
4425 self.state.note_withheld("graft", &r.withheld);
4426 }
4427 let head = git::rev_parse(&wt, "HEAD").await?;
4428 let dirty = match &rescued {
4432 Ok(r) => git::status_porcelain(&wt)
4433 .await
4434 .map(|st| {
4435 st.lines().any(|l| {
4436 !r.withheld
4437 .iter()
4438 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4439 })
4440 })
4441 .unwrap_or(true),
4442 Err(_) => true,
4443 };
4444 if dirty {
4445 why = Some("the fixer left changes that could not be committed".to_owned());
4446 } else if head == pre {
4447 why = Some("the fixer changed nothing".to_owned());
4448 } else {
4449 lines = git::changed_lines(&wt, &pre, &head).await;
4450 post = Some(head);
4451 }
4452 }
4453 let (adoption, adoption_error) = match &out {
4454 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4455 _ => (None, None),
4456 };
4457 let requested = graft::requested(&plan);
4458 let outcome = match why {
4459 None => {
4460 self.state.event(
4461 "graft",
4462 format!(
4463 "applied: {}",
4464 lines.map_or_else(
4465 || "changed lines unavailable".to_owned(),
4466 |n| format!("{n} changed lines")
4467 )
4468 ),
4469 );
4470 GraftOutcome::Applied
4471 }
4472 Some(why) => {
4473 crate::recovery::restore_repair(&wt, &pre)
4476 .await
4477 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4478 self.state.event(
4479 "graft",
4480 format!("rolled back: {why}; the winner is untouched"),
4481 );
4482 GraftOutcome::RolledBack(why)
4483 }
4484 };
4485 self.state.graft = Some(GraftRecord {
4486 pre,
4487 post,
4488 lines,
4489 limit: None,
4490 agent: Some(agent_id),
4491 outcome,
4492 requested,
4493 adoption,
4494 adoption_error,
4495 });
4496 self.state.save()?;
4497 Ok(())
4498 }
4499
4500 async fn fold_losers(&mut self) -> Result<()> {
4503 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4504 return Ok(());
4505 };
4506 if !self.state.candidate_preservation.is_empty() {
4507 bail!("candidate preservation incomplete; refusing cleanup");
4508 }
4509 let repo = self.state.repo.clone();
4510 let _coordination = self.coordinate(&repo).await?;
4511 let mut folded = Vec::new();
4512 for i in 0..self.state.candidates.len() {
4513 let c = &self.state.candidates[i];
4514 if c.label == winner || c.folded {
4515 continue;
4516 }
4517 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4518 remove_preserved_worktree(&repo, &wt).await?;
4519 if git::branch_exists(&repo, &branch).await?
4520 && !git::branch_delete(&repo, &branch).await?
4521 {
4522 bail!("candidate branch removal refused");
4523 }
4524 self.state.candidates[i].folded = true;
4525 self.state.save()?;
4526 folded.push(label.to_string());
4527 }
4528 let root = self.state.worktree_root();
4530 for j in 1..=self.roles.judges.len() {
4531 let wt = root.join(format!("judge-{j}"));
4532 if wt.exists() {
4533 remove_preserved_worktree(&repo, &wt).await?;
4534 }
4535 }
4536 if self.state.config.graph.advise {
4539 for k in 1..=self.state.config.graph.advisors {
4540 let wt = root.join(format!("advisor-{k}"));
4541 if wt.exists() {
4542 remove_preserved_worktree(&repo, &wt).await?;
4543 }
4544 }
4545 }
4546 if !folded.is_empty() {
4547 self.state
4548 .event("fold", format!("folded candidates {}", folded.join(", ")));
4549 self.state.save()?;
4550 }
4551 Ok(())
4552 }
4553
4554 async fn sync_to_base(&mut self) -> Result<()> {
4594 if self.state.status == RunStatus::AlreadyInBase {
4595 return Ok(());
4596 }
4597 let conflicted = self
4598 .state
4599 .base_sync
4600 .as_ref()
4601 .is_some_and(|s| s.conflict.is_some());
4602 let Some(winner) = self.state.winner().cloned() else {
4603 return Ok(());
4604 };
4605
4606 let repo = self.state.repo.clone();
4607 let remote = self.state.config.merge.remote.clone();
4608 let base_branch = self.state.base_branch.clone();
4609 let tracking = format!("{remote}/{base_branch}");
4610
4611 git::fetch(&repo, &remote, &base_branch).await.ok();
4612 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4617 return Ok(());
4618 };
4619
4620 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4621 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4622 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4623
4624 if (behind > 0 || conflicted || head == tip)
4633 && self
4634 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4635 .await?
4636 {
4637 return Ok(());
4638 }
4639 if conflicted {
4640 return Ok(());
4641 }
4642
4643 if behind == 0 {
4644 if let Some(from) = self
4653 .state
4654 .rebase_fixes
4655 .iter()
4656 .rev()
4657 .find_map(|r| r.from.clone())
4658 && from != head
4659 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4660 .await
4661 .is_ok_and(|o| o.ok())
4662 {
4663 git::sync_to_head(&winner.worktree).await?;
4664 }
4665 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4671 if let Some(why) = &conflict {
4672 self.state.status = RunStatus::Blocked;
4673 self.state.event("land", why.clone());
4674 }
4675 self.state.base_sync = Some(BaseSync {
4676 tip,
4677 behind: 0,
4678 attempts,
4679 conflict,
4680 already_in: None,
4681 });
4682 self.state.save()?;
4683 return Ok(());
4684 }
4685
4686 if attempts >= BASE_SYNC_ROUNDS {
4687 let why = format!(
4688 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4689 rebase(s); rebasing again would only race it",
4690 winner.branch
4691 );
4692 self.state.status = RunStatus::Blocked;
4693 self.state.base_sync = Some(BaseSync {
4694 tip,
4695 behind,
4696 attempts,
4697 conflict: Some(why.clone()),
4698 already_in: None,
4699 });
4700 self.state.event("land", why);
4701 self.state.save()?;
4702 return Ok(());
4703 }
4704
4705 self.state.event(
4706 "land",
4707 format!(
4708 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4709 winner.branch
4710 ),
4711 );
4712 self.state.save()?;
4713
4714 let branch_tracking = format!("{remote}/{}", winner.branch);
4720 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4721 let remote_tip = if self.state.task_pr_branch().is_none()
4722 && matches!(&fetched_branch, Ok(o) if o.ok())
4723 {
4724 git::rev_parse(&repo, &branch_tracking).await.ok()
4725 } else {
4726 None
4727 };
4728 if let Some(theirs) = &remote_tip
4732 && !git::is_ancestor(&repo, theirs, &head).await
4733 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4734 .await
4735 .is_ok_and(|missing| missing.is_empty())
4736 {
4737 let why = format!(
4738 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4739 them",
4740 short(theirs),
4741 winner.branch
4742 );
4743 self.state.status = RunStatus::Blocked;
4744 self.state.base_sync = Some(BaseSync {
4745 tip,
4746 behind,
4747 attempts,
4748 conflict: Some(why.clone()),
4749 already_in: None,
4750 });
4751 self.state.event("land", why);
4752 self.state.save()?;
4753 return Ok(());
4754 }
4755
4756 let scratch = self.state.dir().join("base-sync");
4757 let rebased = match crate::rebase::rebase_with_fixer(
4758 &mut self.state,
4759 &scratch,
4760 &winner.branch,
4761 &tracking,
4762 )
4763 .await
4764 {
4765 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4766 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4767 Err(e) => Err(e),
4768 };
4769 let attempts = attempts + 1;
4770 match rebased {
4771 Ok(None) => {
4772 git::sync_to_head(&winner.worktree).await?;
4776 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4777 let mut conflict = None;
4778 if let Some(pinned) = &remote_tip {
4779 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4780 match pushed {
4781 Ok(o) if o.ok() => self.state.event(
4782 "land",
4783 format!("pushed rebased {} to {remote}", winner.branch),
4784 ),
4785 Ok(o) => {
4786 conflict = Some(format!(
4787 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4788 winner.branch,
4789 short(pinned),
4790 o.stderr.chars().take(600).collect::<String>()
4791 ));
4792 }
4793 Err(e) => {
4794 conflict = Some(format!(
4795 "rebased {} locally but could not push it: {e:#}",
4796 winner.branch
4797 ));
4798 }
4799 }
4800 }
4801 if let Some(why) = &conflict {
4802 self.state.status = RunStatus::Blocked;
4803 self.state.event("land", why.clone());
4804 }
4805 self.state.base_sync = Some(BaseSync {
4806 tip: tip.clone(),
4807 behind: 0,
4808 attempts,
4809 conflict,
4810 already_in: None,
4811 });
4812 self.state
4813 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4814 }
4815 Ok(Some(conflict)) => {
4816 let why = format!(
4817 "{} conflicts with {tracking} and did not rebase: {}",
4818 winner.branch,
4819 conflict.chars().take(600).collect::<String>()
4820 );
4821 self.state.status = RunStatus::Blocked;
4822 self.state.base_sync = Some(BaseSync {
4823 tip,
4824 behind,
4825 attempts,
4826 conflict: Some(why.clone()),
4827 already_in: None,
4828 });
4829 self.state.event("land", why);
4830 }
4831 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4835 self.state.save()?;
4836 return Ok(());
4837 }
4838 Err(e) => {
4839 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4840 self.state.status = RunStatus::Blocked;
4841 self.state.base_sync = Some(BaseSync {
4842 tip,
4843 behind,
4844 attempts,
4845 conflict: Some(why.clone()),
4846 already_in: None,
4847 });
4848 self.state.event("land", why);
4849 }
4850 }
4851 self.state.save()?;
4852 Ok(())
4853 }
4854
4855 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4861 if self.state.task_pr_branch().is_some() {
4864 return None;
4865 }
4866 let repo = self.state.repo.clone();
4867 let remote = self.state.config.merge.remote.clone();
4868 let fetched = git::fetch(&repo, &remote, branch).await;
4869 if !matches!(&fetched, Ok(o) if o.ok()) {
4870 return None;
4871 }
4872 let branch_tracking = format!("{remote}/{branch}");
4873 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4874 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4875 return None;
4876 }
4877 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4878 .await
4879 .is_ok_and(|missing| missing.is_empty())
4880 {
4881 return Some(format!(
4882 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4883 them",
4884 short(&theirs)
4885 ));
4886 }
4887 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4888 Ok(o) if o.ok() => {
4889 self.state
4890 .event("land", format!("pushed rebased {branch} to {remote}"));
4891 None
4892 }
4893 Ok(o) => Some(format!(
4894 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4895 someone may have pushed): {}",
4896 short(&theirs),
4897 o.stderr.chars().take(600).collect::<String>()
4898 )),
4899 Err(e) => Some(format!(
4900 "{branch} is rebased locally but could not be pushed: {e:#}"
4901 )),
4902 }
4903 }
4904
4905 async fn settle_already_in(
4922 &mut self,
4923 branch: &str,
4924 tip: &str,
4925 head: &str,
4926 attempts: usize,
4927 behind: usize,
4928 ) -> Result<bool> {
4929 let repo = self.state.repo.clone();
4930 let remote = self.state.config.merge.remote.clone();
4931 let start = self.state.base_commit.clone();
4932 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4933 Ok(Some(e)) => e,
4934 Ok(None) => return Ok(false),
4935 Err(e) => {
4936 tracing::warn!("already-in-base check for {branch}: {e:#}");
4937 return Ok(false);
4938 }
4939 };
4940 let mut verified = vec![head.to_owned()];
4941 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4942 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4943 if matches!(&fetched, Ok(o) if o.ok())
4944 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4945 && theirs != head
4946 {
4947 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4948 Ok(Some(_)) => verified.push(theirs),
4949 _ => return Ok(false),
4950 }
4951 }
4952 let base_branch = self.state.base_branch.clone();
4953 let message = format!(
4954 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4955 land",
4956 evidence.names(),
4957 evidence.proof.as_str()
4958 );
4959 let closed =
4960 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4961 .await;
4962 match closed {
4963 Ok(Ok(url)) => self
4964 .state
4965 .event("land", format!("closed {url}: superseded on {base_branch}")),
4966 Ok(Err(why)) => self
4967 .state
4968 .event("land", format!("did not close a pull request: {why}")),
4969 Err(e) => {
4970 let why = format!(
4971 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4972 not be closed ({e:#}); resume to retry"
4973 );
4974 self.state.status = RunStatus::Blocked;
4975 self.state.base_sync = Some(BaseSync {
4976 tip: tip.to_owned(),
4977 behind,
4978 attempts,
4979 conflict: Some(why.clone()),
4980 already_in: None,
4981 });
4982 self.state.event("land", why);
4983 self.state.save()?;
4984 return Ok(true);
4985 }
4986 }
4987 self.state.status = RunStatus::AlreadyInBase;
4988 self.state.base_sync = Some(BaseSync {
4989 tip: tip.to_owned(),
4990 behind,
4991 attempts,
4992 conflict: None,
4993 already_in: Some(evidence),
4994 });
4995 self.state.event("land", message);
4996 self.state.save()?;
4997 self.settle_questions();
4998 Ok(true)
4999 }
5000
5001 fn landing_base(&self) -> String {
5011 self.state
5012 .base_sync
5013 .as_ref()
5014 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5015 }
5016
5017 pub async fn fix_selected(
5050 &mut self,
5051 ids: &[String],
5052 reason: &str,
5053 allow_stale: bool,
5054 ) -> Result<()> {
5055 let reason = reason.trim();
5056 if reason.is_empty() {
5057 bail!("a fix request needs a reason — that is the operator's own record of why");
5058 }
5059 if ids.is_empty() {
5060 bail!("no finding id given");
5061 }
5062 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5063 bail!(
5064 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5065 has already concluded — can be given a targeted fix. A run still \
5066 in progress should simply be resumed; a `merged` run's branch has \
5067 already landed, so its answer is a fresh `magi review <branch>`, \
5068 not reopening this run's own record",
5069 self.state.id,
5070 self.state.status.as_str()
5071 );
5072 }
5073 let Some(winner) = self.state.winner().cloned() else {
5074 bail!("run {} has no winning candidate to fix", self.state.id);
5075 };
5076 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5077 bail!(
5078 "branch `{}` no longer exists; this run cannot be extended",
5079 winner.branch
5080 );
5081 }
5082 let home = crate::run::home();
5083 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5084 bail!(
5085 "run {} is currently being worked on by another magi process",
5086 self.state.id
5087 );
5088 }
5089 let _claim = FixClaim::acquire(&self.state.dir())?;
5095
5096 let mut seen = BTreeSet::new();
5100 let mut findings = Vec::new();
5101 let mut missing = Vec::new();
5102 for id in ids {
5103 if !seen.insert(id.clone()) {
5104 continue;
5105 }
5106 match self.state.finding(id) {
5107 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5108 id: f.id.clone(),
5109 severity: f.severity,
5110 reviewer_vote: rec.vote,
5111 round: round.round,
5112 round_head: round.head.clone(),
5113 reviewer: rec.reviewer,
5114 agent: rec.agent.clone(),
5115 file: f.file.clone(),
5116 line: f.line,
5117 title: f.title.clone(),
5118 detail: f.detail.clone(),
5119 outcome: OperatorFixOutcome::Pending,
5120 }),
5121 None => missing.push(id.clone()),
5122 }
5123 }
5124 if !missing.is_empty() {
5125 bail!(
5126 "unknown finding id(s): {}; nothing was changed",
5127 missing.join(", ")
5128 );
5129 }
5130
5131 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5132 let stale_details: Vec<(String, String)> = findings
5133 .iter()
5134 .filter(|f| f.round_head != head_at_request)
5135 .map(|f| (f.id.clone(), f.round_head.clone()))
5136 .collect();
5137 let stale = !stale_details.is_empty();
5138 if stale && !allow_stale {
5139 bail!(
5140 "the branch has moved since some finding(s) were raised — {} — now \
5141 at {}; pass --allow-stale to fix anyway, or re-run review first",
5142 stale_details
5143 .iter()
5144 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5145 .collect::<Vec<_>>()
5146 .join(", "),
5147 short(&head_at_request)
5148 );
5149 }
5150
5151 let request = OperatorFixRequest {
5152 requested_at: Timestamp::now(),
5153 reason: reason.to_owned(),
5154 findings,
5155 head_at_request: head_at_request.clone(),
5156 allow_stale,
5157 stale,
5158 fix: None,
5159 result_head: None,
5160 follow_up_review_run: None,
5161 };
5162 self.state.event(
5163 "fix",
5164 format!(
5165 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5166 request.findings.len(),
5167 request
5168 .findings
5169 .iter()
5170 .map(|f| f.id.as_str())
5171 .collect::<Vec<_>>()
5172 .join(", "),
5173 ),
5174 );
5175 self.state.operator_fixes.push(request);
5182 self.state.save()?;
5183 let request_index = self.state.operator_fixes.len() - 1;
5184
5185 let repo = self.state.repo.clone();
5194 let coordination = self.coordinate(&repo).await?;
5195 if winner.worktree.exists() {
5196 let dirty = git::git(
5199 &winner.worktree,
5200 &["status", "--porcelain", "--untracked-files=all"],
5201 )
5202 .await?;
5203 let only_withheld = dirty.lines().all(|l| {
5204 l.strip_prefix("?? ")
5205 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5206 });
5207 if !only_withheld {
5208 bail!(
5209 "`{}` has uncommitted changes; refusing to touch it — commit or \
5210 discard them first",
5211 winner.worktree.display()
5212 );
5213 }
5214 git::worktree_remove(&self.state.repo, &winner.worktree)
5215 .await
5216 .ok();
5217 }
5218 let fix_worktree = self.state.worktree_root().join("operator-fix");
5219 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5220 git::git(
5221 &self.state.repo,
5222 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5223 )
5224 .await
5225 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5226 if !git::is_clean(&fix_worktree).await? {
5227 git::worktree_remove(&self.state.repo, &fix_worktree)
5228 .await
5229 .ok();
5230 bail!(
5231 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5232 winner.branch
5233 );
5234 }
5235 let repo_for_setup = self.state.repo.clone();
5236 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5237 Ok(()) => {
5238 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5239 }
5240 Err(e) => Err(e),
5241 };
5242 if let Err(e) = setup {
5243 git::worktree_remove(&self.state.repo, &fix_worktree)
5244 .await
5245 .ok();
5246 return Err(e);
5247 }
5248
5249 let run_id = self.state.id.clone();
5250 let prompts = self.state.config.prompts.clone();
5251 let language = self.state.config.graph.language.clone();
5252 let sessions = self.state.config.graph.sessions;
5253 let artifacts = agent::artifacts_dir(&self.state.dir());
5254 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5255 .findings
5256 .iter()
5257 .map(|f| Finding {
5258 id: f.id.clone(),
5259 severity: f.severity,
5260 file: f.file.clone(),
5261 line: f.line,
5262 title: f.title.clone(),
5263 detail: f.detail.clone(),
5264 })
5265 .collect();
5266 let fix_prompt = prompt::operator_fix_in(
5267 &self.state.instruction,
5268 &finding_list,
5269 reason,
5270 &stale_details,
5271 &head_at_request,
5272 &language,
5273 &self.state.config.graph.github_language,
5274 );
5275 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5276 drop(coordination);
5277 let (job, seat, out) = self
5278 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5279 prompt: fix_prompt.clone(),
5280 spec,
5281 seat,
5282 cwd: fix_worktree.clone(),
5283 timeout,
5284 allow_write: true,
5285 sessions,
5286 artifacts: artifacts.clone(),
5287 stem: "operator-fix".to_owned(),
5288 handover: None,
5289 })
5290 .await;
5291 let agent_id = seat.agent.clone();
5292
5293 let mut fix = FixRecord {
5294 agent: agent_id,
5295 addressed: Vec::new(),
5296 rejected: Vec::new(),
5297 notes: String::new(),
5298 committed: false,
5299 failed: None,
5300 duration_ms: 0,
5301 continuation: None,
5302 };
5303 let mut final_seat = seat.clone();
5304 match out {
5305 AgentOutcome::Ok(o) => {
5306 fix.duration_ms = o.duration_ms;
5307 let parsed = verdict::extract_json::<FixReport>(&o.text);
5308 let incomplete_reason = match &parsed {
5309 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5310 "the reply parsed, but it reported a command whose own CLI \
5311 never confirmed an exit status"
5312 .to_owned(),
5313 ),
5314 Ok(_) => None,
5315 Err(e) => Some(e.to_string()),
5316 };
5317 match incomplete_reason {
5318 None => {
5319 let report = parsed.expect("checked Ok above");
5320 fix.addressed = report.addressed;
5321 fix.rejected = report.rejected;
5322 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5323 }
5324 Some(reason) => {
5325 let (resumed_seat, resolved, failure, cont) = self
5326 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5327 .await;
5328 fix.duration_ms += cont.cumulative_wait_ms;
5329 fix.continuation = Some(cont);
5330 final_seat = resumed_seat;
5331 match resolved {
5332 Some(report) => {
5333 fix.addressed = report.addressed;
5334 fix.rejected = report.rejected;
5335 fix.notes =
5336 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5337 }
5338 None => fix.failed = failure,
5339 }
5340 }
5341 }
5342 }
5343 AgentOutcome::Dropped(o) => {
5344 fix.duration_ms = o.duration_ms;
5345 let why = o
5346 .dropped
5347 .as_ref()
5348 .map(|d| d.why.as_str())
5349 .unwrap_or("the CLI ended the stream without delivering its answer");
5350 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5351 }
5352 AgentOutcome::Quota(o) => {
5353 self.state.quota.push(QuotaLoss {
5354 seat: final_seat.key.clone(),
5355 node: "fix".to_owned(),
5356 at: Timestamp::now(),
5357 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5358 });
5359 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5360 }
5361 AgentOutcome::Failed(e) => fix.failed = Some(e),
5362 AgentOutcome::ResourceBlocked(e) => {
5363 fix.failed = Some(format!("the fixer was not started: {e}"));
5364 }
5365 }
5366 if fix.continuation.is_none() {
5367 fix.continuation = Some(ContinuationRecord::not_needed());
5368 }
5369 self.state.seats.insert(final_seat.key.clone(), final_seat);
5370
5371 let rescue_message = format!(
5372 "magi: operator-selected fix ({}) (uncommitted work)",
5373 self.state.operator_fixes[request_index]
5374 .findings
5375 .iter()
5376 .map(|f| f.id.as_str())
5377 .collect::<Vec<_>>()
5378 .join(", ")
5379 );
5380 let repo = self.state.repo.clone();
5381 let coordination = self.coordinate(&repo).await?;
5382 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5383 self.state.note_withheld("fix", &r.withheld);
5384 }
5385 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5386 fix.committed = after != head_at_request;
5387 git::worktree_remove(&self.state.repo, &fix_worktree)
5388 .await
5389 .ok();
5390 drop(coordination);
5391
5392 self.state.event(
5393 "fix",
5394 match &fix.failed {
5395 Some(reason) => format!(
5396 "operator fix: adoption report was lost ({reason}); {}",
5397 if fix.committed {
5398 "committed"
5399 } else {
5400 "NO new commit"
5401 }
5402 ),
5403 None => format!(
5404 "operator fix: {} addressed, {} rejected, {}",
5405 fix.addressed.len(),
5406 fix.rejected.len(),
5407 if fix.committed {
5408 "committed"
5409 } else {
5410 "NO new commit"
5411 }
5412 ),
5413 },
5414 );
5415
5416 for f in &mut self.state.operator_fixes[request_index].findings {
5423 f.outcome = if fix.failed.is_some() {
5424 OperatorFixOutcome::Unreported
5425 } else if fix.addressed.contains(&f.id) {
5426 OperatorFixOutcome::Addressed
5427 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5428 OperatorFixOutcome::Rejected { why: r.why.clone() }
5429 } else {
5430 OperatorFixOutcome::Unreported
5431 };
5432 }
5433
5434 let committed = fix.committed;
5435 if committed {
5436 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5437 }
5438 self.state.operator_fixes[request_index].fix = Some(fix);
5439 self.state.save()?;
5442
5443 if committed {
5444 self.state.event(
5445 "fix",
5446 format!(
5447 "operator fix committed {}; opening a follow-up review-only run",
5448 short(&after)
5449 ),
5450 );
5451 let origin =
5454 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5455 match Self::review(
5456 &self.state.repo,
5457 &winner.branch,
5458 self.state.config.clone(),
5459 origin,
5460 )
5461 .await
5462 {
5463 Ok(mut follow_up) => {
5464 follow_up.state.event(
5465 "start",
5466 format!(
5467 "requested by an operator fix on run {} for finding(s) {}",
5468 self.state.id,
5469 self.state.operator_fixes[request_index]
5470 .findings
5471 .iter()
5472 .map(|f| f.id.as_str())
5473 .collect::<Vec<_>>()
5474 .join(", "),
5475 ),
5476 );
5477 follow_up.state.save()?;
5478 let follow_up_id = follow_up.state.id.clone();
5479 let adopted = match crate::direct::adopt(
5482 &follow_up.state,
5483 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5484 ) {
5485 Ok(a) => a,
5486 Err(e) => {
5487 self.state.event(
5490 "fix",
5491 format!(
5492 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5493 ),
5494 );
5495 self.state.save()?;
5496 return Ok(());
5497 }
5498 };
5499 let executed = follow_up.execute().await;
5500 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5501 if let Some(a) = adopted {
5502 a.finish(&follow_up.state, executed);
5503 }
5504 if let Some(e) = failure {
5505 self.state.event(
5506 "fix",
5507 format!(
5508 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5509 ),
5510 );
5511 }
5512 self.state.operator_fixes[request_index].follow_up_review_run =
5513 Some(follow_up_id);
5514 }
5515 Err(e) => {
5516 self.state.event(
5517 "fix",
5518 format!("committed the fix but could not open a follow-up review: {e:#}"),
5519 );
5520 }
5521 }
5522 self.state.save()?;
5523 }
5524
5525 Ok(())
5526 }
5527
5528 async fn ask_fixer(
5548 &mut self,
5549 winner: &Candidate,
5550 node: &'static str,
5551 round: Option<usize>,
5552 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5553 ) -> (SeatJob, SeatState, AgentOutcome) {
5554 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5555 self.ask_fixer_among(attempts, node, round, None, build)
5556 .await
5557 }
5558
5559 async fn ask_fixer_among(
5564 &mut self,
5565 attempts: Vec<(AgentSpec, String)>,
5566 node: &'static str,
5567 round: Option<usize>,
5568 restore: Option<(&Path, &str)>,
5569 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5570 ) -> (SeatJob, SeatState, AgentOutcome) {
5571 let run_id = self.state.id.clone();
5572 let prompts = self.state.config.prompts.clone();
5573 let cache = self.state.config.cache_dir();
5574 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5575 let mut last = None;
5576 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5577 let seat = self.seat(&key, &spec.id);
5578 let mut job = build(spec.clone(), seat);
5579 if i > 0 {
5580 job.stem = format!("{}-{}", job.stem, spec.id);
5581 }
5582 let ctx = WaveCtx {
5583 carry_seats: false,
5584 pauses: Some((&self.pause, &self.interrupt)),
5585 run: &run_id,
5586 node,
5587 prompts: &prompts,
5588 cache: cache.as_deref(),
5589 round,
5590 };
5591 let (seat, out) =
5592 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5593 let advances = match &out {
5596 AgentOutcome::Ok(o) => agent::output_advances(o),
5597 AgentOutcome::ResourceBlocked(_) => false,
5598 _ => true,
5599 };
5600 if let Some(next) = ids.get(i + 1)
5601 && advances
5602 {
5603 self.state.seats.insert(seat.key.clone(), seat.clone());
5604 let class =
5605 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5606 let reason = match &out {
5607 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5608 other => fail_reason(other),
5609 };
5610 if let Some((wt, rev)) = restore
5614 && let Err(e) = self.graft_restore(wt, rev).await
5615 {
5616 let why =
5617 format!("could not restore the worktree before the next fixer: {e:#}");
5618 tracing::warn!("{why}");
5619 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5620 AgentOutcome::ResourceBlocked(why)
5621 } else {
5622 AgentOutcome::Failed(why)
5623 };
5624 last = Some((job, seat, out));
5625 break;
5626 }
5627 record_handover(
5628 &mut self.state,
5629 node,
5630 &seat.key,
5631 &spec.id,
5632 next,
5633 &class,
5634 &reason,
5635 );
5636 continue;
5637 }
5638 last = Some((job, seat, out));
5639 break;
5640 }
5641 last.expect("the fixer chain always has an entry")
5642 }
5643
5644 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5655 use crate::recovery::{Mode, Repair};
5656 if !self.state.reviews.is_empty() {
5657 return Ok(());
5658 }
5659 let Some(mut rec) = self.state.recovery.clone() else {
5660 return Ok(());
5661 };
5662 if rec.mode != Mode::VerifyRepair {
5663 return Ok(());
5664 }
5665 let wt = winner.worktree.clone();
5666 let mut head = git::rev_parse(&wt, "HEAD").await?;
5667 match rec.repair.clone() {
5668 Repair::Done { .. } => return Ok(()),
5669 Repair::WaitingForRestore { before } => {
5670 self.graft_restore(&wt, &before).await?;
5671 head = before;
5672 rec.repair = Repair::NotStarted;
5673 self.state.recovery = Some(rec.clone());
5674 self.state.save()?;
5675 }
5676 Repair::Started { before } => {
5677 if let Err(e) = self.graft_restore(&wt, &before).await {
5681 if e.is::<crate::repo_lock::ResourceBlocked>() {
5682 return Err(e);
5683 }
5684 return self
5685 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5686 .await;
5687 }
5688 rec.repair = Repair::Done {
5689 before: before.clone(),
5690 after: before,
5691 committed: false,
5692 failed: Some("interrupted; rolled back".to_owned()),
5693 };
5694 self.state
5695 .event("recovery", "the repair was interrupted and rolled back");
5696 self.state.recovery = Some(rec);
5697 self.state.save()?;
5698 return Ok(());
5699 }
5700 Repair::NotStarted => {}
5701 }
5702 if crate::recovery::recheck(&mut rec, &head) {
5703 self.state.event("recovery", rec.summary());
5704 self.state.recovery = Some(rec);
5705 self.state.save()?;
5706 return Ok(());
5707 }
5708 rec.repair = Repair::Started {
5709 before: head.clone(),
5710 };
5711 self.state.recovery = Some(rec.clone());
5712 self.state.save()?;
5713
5714 let language = self.state.config.graph.language.clone();
5715 let sessions = self.state.config.graph.sessions;
5716 let artifacts = agent::artifacts_dir(&self.state.dir());
5717 let fix_prompt = prompt::recovery_fix(
5718 &self.state.instruction,
5719 &rec,
5720 self.state.config.graph.review_rounds,
5721 &language,
5722 );
5723 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5724 self.state.event(
5725 "recovery",
5726 format!(
5727 "repair: the fixer starts from the failed verification on {}",
5728 short(&head)
5729 ),
5730 );
5731 let (_job, seat, out) = self
5732 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5733 prompt: fix_prompt.clone(),
5734 spec,
5735 seat,
5736 cwd: wt.clone(),
5737 timeout,
5738 allow_write: true,
5739 sessions,
5740 artifacts: artifacts.clone(),
5741 stem: "fix-recovery".to_owned(),
5742 handover: None,
5743 })
5744 .await;
5745 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5746 rec.repair = Repair::WaitingForRestore { before: head };
5747 self.state.recovery = Some(rec);
5748 return self.stop_for_resource();
5749 }
5750 let failed = match &out {
5751 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5752 AgentOutcome::Quota(o) => {
5753 self.state.quota.push(QuotaLoss {
5754 seat: seat.key.clone(),
5755 node: "fix".to_owned(),
5756 at: Timestamp::now(),
5757 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5758 });
5759 Some("rate limited (quota); the fixer could not run".to_owned())
5760 }
5761 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5762 AgentOutcome::Failed(e) => Some(e.clone()),
5763 };
5764 self.state.seats.insert(seat.key.clone(), seat);
5765 let coordination = self.coordinate(&wt).await?;
5766 let failed = match failed {
5767 Some(why) => {
5769 if let Err(e) = git::restore_to(&wt, &head).await {
5770 return self
5771 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5772 .await;
5773 }
5774 Some(why)
5775 }
5776 None => {
5777 if let Ok(r) = git::rescue_commit(
5778 &wt,
5779 "magi: recovery repair of the failed verification (uncommitted work)",
5780 )
5781 .await
5782 {
5783 self.state.note_withheld("fix", &r.withheld);
5784 }
5785 None
5786 }
5787 };
5788 let after = git::rev_parse(&wt, "HEAD").await?;
5789 drop(coordination);
5790 self.state.event(
5791 "recovery",
5792 match &failed {
5793 Some(why) => format!("repair: nothing kept ({why})"),
5794 None if after != head => {
5795 format!("repair: committed {}", short(&after))
5796 }
5797 None => "repair: the fixer made no commit".to_owned(),
5798 },
5799 );
5800 rec.repair = Repair::Done {
5801 before: head.clone(),
5802 committed: after != head,
5803 after,
5804 failed,
5805 };
5806 self.state.recovery = Some(rec);
5807 self.state.save()?;
5808 Ok(())
5809 }
5810
5811 async fn stop_recovery(
5814 &mut self,
5815 mut rec: crate::recovery::RecoveryRecord,
5816 head: &str,
5817 why: String,
5818 ) -> Result<()> {
5819 rec.repair = crate::recovery::Repair::Started {
5821 before: head.to_owned(),
5822 };
5823 self.state.recovery = Some(rec);
5824 self.state
5825 .event("recovery", format!("repair could not be undone: {why}"));
5826 self.state.status = RunStatus::Blocked;
5827 self.state.save()?;
5828 bail!("recovery repair could not be rolled back: {why}")
5829 }
5830
5831 async fn review_loop(&mut self) -> Result<()> {
5832 if self
5837 .state
5838 .base_sync
5839 .as_ref()
5840 .is_some_and(|s| s.conflict.is_some())
5841 {
5842 return Ok(());
5843 }
5844 let run_id = self.state.id.clone();
5849 let prompts = self.state.config.prompts.clone();
5850 let Some(winner) = self.state.winner().cloned() else {
5851 return Ok(());
5852 };
5853 let max_rounds = self.state.config.graph.review_rounds;
5854 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5864 if status == RunStatus::Gating {
5867 self.record_contested_handoff();
5868 }
5869 self.state.status = status;
5870 self.state.save()?;
5871 return Ok(());
5872 }
5873 self.state.status = RunStatus::Reviewing;
5874 if self
5884 .state
5885 .reviews
5886 .last()
5887 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5888 {
5889 let shell = self.state.config.shell();
5890 return self
5891 .stop_reviewing(
5892 "the last round's own verification never resolved",
5893 &shell,
5894 &winner.worktree,
5895 )
5896 .await;
5897 }
5898
5899 self.recovery_repair(&winner).await?;
5900 if self.resource_halt()? {
5901 return Ok(());
5902 }
5903
5904 let repo = self.state.repo.clone();
5905 let root = self.state.worktree_root();
5906 let language = self.state.config.graph.language.clone();
5907 let sessions = self.state.config.graph.sessions;
5908 let artifacts = self.merge_repair_artifacts();
5909 let base = self.landing_base();
5910 let base_short = short(&base);
5911 let reviewers = self.roles.reviewers.clone();
5912 let shell = self.state.config.shell();
5913
5914 for round in (self.state.reviews.len() + 1)..=max_rounds {
5915 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5916 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5917 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5918 let prev_verification = self
5927 .state
5928 .reviews
5929 .last()
5930 .and_then(|r| r.verification_summary(&head));
5931 let prior_section = match (&self.state.recovery, round) {
5934 (Some(rec), 1) if self.state.reviews.is_empty() => {
5935 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5936 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5937 Some(crate::recovery::head_of(
5938 &git::diff(&winner.worktree, reviewed, "HEAD")
5939 .await
5940 .unwrap_or_default(),
5941 crate::recovery::SINCE_DIFF_MAX,
5942 ))
5943 }
5944 _ => None,
5945 };
5946 Some(prompt::recovery_review(rec, since.as_deref()))
5947 }
5948 _ => None,
5949 };
5950
5951 let mut jobs = Vec::new();
5955 let coordination = self.coordinate(&repo).await?;
5956 for (r, spec) in reviewers.iter().cloned().enumerate() {
5957 let wt = root.join(format!("review-{}", r + 1));
5958 if wt.exists() {
5961 git::reset_detached(&wt, &head).await?;
5962 } else {
5963 git::worktree_add_detached(&repo, &wt, &head).await?;
5964 }
5965 self.setup_seat_worktree(&repo, &wt, false).await?;
5966 let seat_key = format!("review-{}", r + 1);
5967 let spec = pick_start_spec(
5970 &self.roles.reviewer_roster,
5971 spec,
5972 self.state.seat_history.get(&seat_key),
5973 );
5974 let seat = self.seat(&seat_key, &spec.id);
5975 jobs.push(SeatJob {
5976 prompt: prompt::review(&prompt::ReviewCtx {
5977 instruction: &self.state.instruction,
5978 branch: &winner.branch,
5979 base_short: &base_short,
5980 stat: &stat,
5981 patch: &patch,
5982 verification: prev_verification.as_ref(),
5983 prior: prior_section.as_deref(),
5984 reviewers: reviewers.len(),
5985 round,
5986 rounds: max_rounds,
5987 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5990 lens: Lens::for_seat(r),
5991 language: &language,
5992 github_language: &self.state.config.graph.github_language,
5993 }),
5994 spec,
5995 seat,
5996 cwd: wt,
5997 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
5998 allow_write: false,
5999 sessions,
6000 artifacts: artifacts.clone(),
6001 stem: format!("review-{round}-{}", r + 1),
6002 handover: None,
6003 });
6004 }
6005
6006 drop(coordination);
6007 self.state.event(
6008 "review",
6009 format!(
6010 "round {round}: {} reviewers on {}",
6011 jobs.len(),
6012 short(&head)
6013 ),
6014 );
6015 let mut quota_losses = Vec::new();
6016 let review_retries = self.state.config.graph.retries;
6017 let review_cache = self.state.config.cache_dir();
6018 let ctx = WaveCtx {
6019 carry_seats: true,
6020 pauses: Some((&self.pause, &self.interrupt)),
6021 run: &run_id,
6022 node: "review",
6023 prompts: &prompts,
6024 cache: review_cache.as_deref(),
6025 round: Some(round),
6026 };
6027 let results = ask_json_wave::<Review>(
6028 jobs,
6029 Arc::clone(&self.sem),
6030 review_retries,
6031 &self.roles.reviewer_roster,
6032 &ctx,
6033 &mut quota_losses,
6034 &mut self.state,
6035 &|_: &Review| Ok(()),
6036 )
6037 .await;
6038 if self.state.resource_block.is_some() {
6039 return self.stop_for_resource();
6040 }
6041 let round_quota_missing = quota_losses.len();
6045 self.state.quota.extend(quota_losses);
6046
6047 let mut records = Vec::new();
6048 let mut all_findings = Vec::new();
6049 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6050 let agent_id = seat.agent.clone();
6051 self.state.seats.insert(seat.key.clone(), seat);
6052 let mut record = ReviewRecord {
6053 reviewer: r + 1,
6054 agent: agent_id,
6055 summary: String::new(),
6056 findings: Vec::new(),
6057 vote: None,
6058 failed: None,
6059 duration_ms: 0,
6060 attempts,
6066 };
6067 match res {
6068 Ok((review, out)) => {
6069 record.summary =
6077 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6078 record.vote = Some(review.vote);
6079 record.duration_ms = out.duration_ms;
6080 for (n, mut f) in review.findings.into_iter().enumerate() {
6081 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6084 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6085 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6086 f.file = f
6092 .file
6093 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6094 all_findings.push(f.clone());
6095 record.findings.push(f);
6096 }
6097 self.state.event(
6098 "review",
6099 format!(
6100 "round {round}: reviewer {} voted {} with {} finding(s)",
6101 r + 1,
6102 review.vote.label(),
6103 record.findings.len()
6104 ),
6105 );
6106 }
6107 Err(e) => {
6108 record.failed = Some(e.to_string());
6109 self.state.event(
6110 "review",
6111 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6112 );
6113 }
6114 }
6115 records.push(record);
6116 }
6117
6118 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6125 let vote_split =
6126 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6127 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6128 if vote_split {
6129 self.state.event(
6130 "review",
6131 format!(
6132 "round {round}: votes split ({}) — one round of reconsideration",
6133 initial_votes
6134 .iter()
6135 .map(|v| v.label())
6136 .collect::<Vec<_>>()
6137 .join(", ")
6138 ),
6139 );
6140 let panel: Vec<ReviewSeatReport<'_>> = records
6143 .iter()
6144 .filter_map(|r| {
6145 r.vote.map(|vote| ReviewSeatReport {
6146 reviewer: r.reviewer,
6147 vote,
6148 summary: &r.summary,
6149 findings: &r.findings,
6150 })
6151 })
6152 .collect();
6153
6154 let mut jobs = Vec::new();
6155 let mut seats_at = Vec::new();
6156 for (r, spec) in reviewers.iter().cloned().enumerate() {
6157 if records[r].vote.is_none() {
6161 continue;
6162 }
6163 let wt = root.join(format!("review-{}", r + 1));
6164 let seat_key = format!("review-{}", r + 1);
6165 let spec = self.occupant(&seat_key, spec);
6166 let seat = self.seat(&seat_key, &spec.id);
6167 let build = |with_patch: bool| {
6175 prompt::review_reconsider(&ReviewReconsiderCtx {
6176 instruction: &self.state.instruction,
6177 reviewer: r + 1,
6178 lens: Lens::for_seat(r),
6179 panel: &panel,
6180 patch: with_patch.then_some(ReviewPatch {
6181 branch: &winner.branch,
6182 base_short: &base_short,
6183 stat: &stat,
6184 patch: &patch,
6185 }),
6186 round,
6187 rounds: max_rounds,
6188 language: &language,
6189 })
6190 };
6191 let full = build(true);
6192 let prompt = if has_context(&spec, &seat, sessions) {
6193 build(false)
6194 } else {
6195 full.clone()
6196 };
6197 jobs.push(SeatJob {
6198 prompt,
6199 spec,
6200 seat,
6201 cwd: wt,
6202 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6203 allow_write: false,
6204 sessions,
6205 artifacts: artifacts.clone(),
6206 stem: format!("review-{round}-reconsider-{}", r + 1),
6207 handover: Some(full),
6208 });
6209 seats_at.push(r);
6210 }
6211
6212 let mut recon_quota_losses = Vec::new();
6213 let recon_cache = self.state.config.cache_dir();
6214 let recon_ctx = WaveCtx {
6215 carry_seats: true,
6216 pauses: Some((&self.pause, &self.interrupt)),
6217 run: &run_id,
6218 node: "review",
6219 prompts: &prompts,
6220 cache: recon_cache.as_deref(),
6221 round: Some(round),
6222 };
6223 let recon_results = ask_json_wave::<ReviewRevote>(
6224 jobs,
6225 Arc::clone(&self.sem),
6226 review_retries,
6227 &self.roles.reviewer_roster,
6228 &recon_ctx,
6229 &mut recon_quota_losses,
6230 &mut self.state,
6231 &|_: &ReviewRevote| Ok(()),
6232 )
6233 .await;
6234 if self.state.resource_block.is_some() {
6235 return self.stop_for_resource();
6236 }
6237 self.state.quota.extend(recon_quota_losses);
6238
6239 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6240 let agent_id = seat.agent.clone();
6241 self.state.seats.insert(seat.key.clone(), seat);
6242 let mut rec = ReviewRevoteRecord {
6243 reviewer: r + 1,
6244 agent: agent_id,
6245 vote: None,
6246 reason: String::new(),
6247 failed: None,
6248 };
6249 match res {
6250 Ok((rv, _)) => {
6251 rec.vote = Some(rv.vote);
6252 rec.reason =
6253 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6254 self.state.event(
6255 "review",
6256 format!(
6257 "round {round}: reviewer {} revoted {}",
6258 r + 1,
6259 rv.vote.label()
6260 ),
6261 );
6262 }
6263 Err(e) => {
6264 rec.failed = Some(e.to_string());
6265 self.state.event(
6266 "review",
6267 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6268 );
6269 }
6270 }
6271 reconsideration.push(rec);
6272 }
6273 } else if initial_votes.len() > 1 {
6274 self.state.event(
6275 "review",
6276 format!(
6277 "round {round}: votes agreed ({}) — no reconsideration",
6278 initial_votes[0].label()
6279 ),
6280 );
6281 }
6282
6283 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6284 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6285 let defer_e2e =
6296 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6297 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6298 let reason =
6299 format!("{blocking} blocking finding(s) already required a fix this round");
6300 self.state.event(
6301 "verify",
6302 format!(
6303 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6304 {}); it will run once a round has none left",
6305 short(&head)
6306 ),
6307 );
6308 (Vec::new(), false, true, Some(reason))
6309 } else {
6310 let e2e_commands = self.state.config.verify.e2e.clone();
6311 let cache_dir = self.state.config.cache_dir();
6312 let context = format!("round {round}");
6313 let wait = self.repository_wait();
6314 let (e2e, verify_retried) = with_cache_lease(
6315 &mut self.state,
6316 cache_dir.as_deref(),
6317 "e2e",
6318 "e2e",
6319 &winner.worktree,
6320 &head,
6321 verify_timeout,
6322 &context,
6323 &wait,
6324 |state, budget| {
6325 let shell = shell.clone();
6326 let e2e_commands = e2e_commands.clone();
6327 let worktree = winner.worktree.clone();
6328 let context = context.clone();
6329 async move {
6330 run_e2e_with_retry(
6331 state,
6332 &shell,
6333 &e2e_commands,
6334 &worktree,
6335 budget,
6336 &context,
6337 )
6338 .await
6339 }
6340 },
6341 )
6342 .await?;
6343 (e2e, verify_retried, false, None)
6344 };
6345
6346 let expected = records.len();
6347 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6348 let incomplete = answered < expected;
6349 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6350 let policy = self.state.config.graph.incomplete_review;
6351 let clean = round_is_clean(
6352 blocking,
6353 e2e_ok,
6354 answered,
6355 expected,
6356 round_quota_missing,
6357 policy,
6358 );
6359
6360 let mut round_record = ReviewRound {
6361 round,
6362 head: head.clone(),
6363 verified_head: None,
6364 verified_at: None,
6365 reviews: records,
6366 e2e,
6367 verify_retried,
6368 e2e_deferred,
6369 e2e_defer_reason,
6370 fix: None,
6371 blocking,
6372 answered,
6373 expected,
6374 clean,
6375 progressed: false,
6376 vote_split,
6377 reconsideration,
6378 verdict: None,
6379 };
6380 round_record.verdict = ReviewVote::worst(
6384 round_record
6385 .final_votes()
6386 .into_iter()
6387 .map(|(_, _, vote)| vote),
6388 );
6389 if !matches!(
6400 round_record.e2e_status(),
6401 E2eStatus::Deferred | E2eStatus::NotConfigured
6402 ) {
6403 round_record.verified_head = Some(head.clone());
6404 round_record.verified_at = Some(Timestamp::now());
6405 }
6406 let this_round_verification = round_record.verification_summary(&head);
6407
6408 if incomplete {
6409 let missing: Vec<String> = round_record
6410 .reviews
6411 .iter()
6412 .filter(|r| r.failed.is_some())
6413 .map(|r| format!("review-{}", r.reviewer))
6414 .collect();
6415 self.state.event(
6416 "review",
6417 format!(
6418 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6419 missing.join(", ")
6420 ),
6421 );
6422 }
6423
6424 if clean {
6425 self.state.event(
6426 "review",
6427 if incomplete && policy == IncompleteReviewPolicy::Warn {
6428 format!(
6429 "round {round}: clean (warn policy, incomplete panel) — no \
6430 blocking findings from the seats that answered, verification green"
6431 )
6432 } else if incomplete {
6433 format!(
6434 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6435 quorum) — no blocking findings from the seats that answered, \
6436 verification green",
6437 expected - answered
6438 )
6439 } else {
6440 format!("round {round}: clean — no blocking findings, verification green")
6441 },
6442 );
6443 self.state.reviews.push(round_record);
6444 self.state.status = RunStatus::Gating;
6445 self.state.save()?;
6446 return Ok(());
6447 }
6448
6449 if incomplete && blocking == 0 && e2e_ok {
6457 self.state.reviews.push(round_record);
6458 self.state.save()?;
6459 if round == max_rounds {
6460 self.state.status = RunStatus::Blocked;
6461 self.state.event(
6462 "review",
6463 format!(
6464 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6465 refusing to call it clean",
6466 expected - answered
6467 ),
6468 );
6469 return Ok(());
6470 }
6471 continue;
6472 }
6473
6474 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6486 self.state.reviews.push(round_record);
6487 return self
6488 .stop_reviewing(
6489 "the round's own verification could not run",
6490 &shell,
6491 &winner.worktree,
6492 )
6493 .await;
6494 }
6495
6496 if round == max_rounds {
6497 self.state.reviews.push(round_record);
6498 return self
6499 .stop_reviewing(
6500 &format!(
6501 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6502 ),
6503 &shell,
6504 &winner.worktree,
6505 )
6506 .await;
6507 }
6508
6509 let blocking_findings: Vec<_> = all_findings
6512 .iter()
6513 .filter(|f| f.severity.blocks())
6514 .cloned()
6515 .collect();
6516 let fix_prompt = prompt::fix_in(
6517 &self.state.instruction,
6518 &blocking_findings,
6519 this_round_verification.as_ref(),
6520 round,
6521 max_rounds,
6522 &language,
6523 &self.state.config.graph.github_language,
6524 );
6525 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6526 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6527 let (job, seat, out) = self
6528 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6529 prompt: fix_prompt.clone(),
6530 spec,
6531 seat,
6532 cwd: winner.worktree.clone(),
6533 timeout,
6534 allow_write: true,
6535 sessions,
6536 artifacts: artifacts.clone(),
6537 stem: format!("fix-{round}"),
6538 handover: None,
6539 })
6540 .await;
6541 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6544 return self.stop_for_resource();
6545 }
6546 let agent_id = seat.agent.clone();
6547
6548 let mut fix = FixRecord {
6549 agent: agent_id,
6550 addressed: Vec::new(),
6551 rejected: Vec::new(),
6552 notes: String::new(),
6553 committed: false,
6554 failed: None,
6555 duration_ms: 0,
6556 continuation: None,
6557 };
6558 let mut continuation = ContinuationRecord::not_needed();
6559 let mut final_seat = seat.clone();
6560 match out {
6561 AgentOutcome::Ok(o) => {
6562 fix.duration_ms = o.duration_ms;
6563 let parsed = verdict::extract_json::<FixReport>(&o.text);
6564 let incomplete_reason = match &parsed {
6571 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6572 "the reply parsed, but it reported a command whose own CLI never \
6573 confirmed an exit status"
6574 .to_owned(),
6575 ),
6576 Ok(_) => None,
6577 Err(e) => Some(e.to_string()),
6578 };
6579 match incomplete_reason {
6580 None => {
6581 let report = parsed.expect("checked Ok above");
6582 fix.addressed = report.addressed;
6583 fix.rejected = report.rejected;
6584 fix.notes =
6585 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6586 }
6587 Some(reason) => {
6588 let (resumed_seat, resolved, failure, cont) = self
6589 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6590 .await;
6591 fix.duration_ms += cont.cumulative_wait_ms;
6592 continuation = cont;
6593 final_seat = resumed_seat;
6594 match resolved {
6595 Some(report) => {
6596 fix.addressed = report.addressed;
6597 fix.rejected = report.rejected;
6598 fix.notes = blind::sanitize_prose(
6599 &report.notes,
6600 &self.state.config.blind,
6601 );
6602 }
6603 None => fix.failed = failure,
6604 }
6605 }
6606 }
6607 }
6608 AgentOutcome::Dropped(o) => {
6610 fix.duration_ms = o.duration_ms;
6611 let why = o
6612 .dropped
6613 .as_ref()
6614 .map(|d| d.why.as_str())
6615 .unwrap_or("the CLI ended the stream without delivering its answer");
6616 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6617 }
6618 AgentOutcome::Quota(o) => {
6619 self.state.quota.push(QuotaLoss {
6620 seat: final_seat.key.clone(),
6621 node: "fix".to_owned(),
6622 at: Timestamp::now(),
6623 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6624 });
6625 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6626 }
6627 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6628 }
6629 fix.continuation = Some(continuation);
6630 self.state.seats.insert(final_seat.key.clone(), final_seat);
6631 let coordination = self.coordinate(&winner.worktree).await?;
6632 if let Ok(r) = git::rescue_commit(
6633 &winner.worktree,
6634 &format!("magi: review round {round} fixes (uncommitted work)"),
6635 )
6636 .await
6637 {
6638 self.state.note_withheld("fix", &r.withheld);
6639 }
6640 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6641 fix.committed = after != before;
6642 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6650 drop(coordination);
6651 let progressed = diff_after != patch;
6652 let commit_note = if fix.committed {
6653 "committed"
6654 } else {
6655 "NO new commit"
6656 };
6657 let tree_note = if progressed {
6658 "changed vs base"
6659 } else {
6660 "unchanged vs base"
6661 };
6662 self.state.event(
6663 "fix",
6664 match &fix.failed {
6665 Some(reason) => {
6671 format!(
6672 "round {round}: fixer's adoption report was lost ({reason}); \
6673 {commit_note}, tree {tree_note}"
6674 )
6675 }
6676 None => format!(
6677 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6678 {tree_note}{}",
6679 fix.addressed.len(),
6680 fix.rejected.len(),
6681 if continuation.outcome == ContinuationOutcome::Resumed {
6682 format!(
6683 " (adoption report recovered after {} continuation(s))",
6684 continuation.attempts
6685 )
6686 } else {
6687 String::new()
6688 },
6689 ),
6690 },
6691 );
6692 round_record.fix = Some(fix);
6693 round_record.progressed = progressed;
6694 self.state.reviews.push(round_record);
6695 self.state.save()?;
6696
6697 if matches!(
6710 continuation.outcome,
6711 ContinuationOutcome::Exhausted
6712 | ContinuationOutcome::QuotaLost
6713 | ContinuationOutcome::NoSession
6714 ) {
6715 return self
6716 .stop_reviewing(
6717 "the fixer's adoption report never came back, even after resuming its \
6718 own seat; refusing to start another round against the same worktree \
6719 while that is unresolved",
6720 &shell,
6721 &winner.worktree,
6722 )
6723 .await;
6724 }
6725
6726 let streak = self
6727 .state
6728 .reviews
6729 .iter()
6730 .rev()
6731 .take_while(|r| !r.progressed)
6732 .count();
6733 if streak >= STAGNANT_LIMIT {
6734 return self
6735 .stop_reviewing(
6736 &format!(
6737 "the tree has not moved against base for {streak} round(s) in a row"
6738 ),
6739 &shell,
6740 &winner.worktree,
6741 )
6742 .await;
6743 }
6744 }
6745 Ok(())
6746 }
6747
6748 fn record_contested_handoff(&mut self) {
6782 if self.state.contested_handoff.is_some() {
6783 return;
6784 }
6785 let Some(contested) = self
6786 .state
6787 .reviews
6788 .last()
6789 .and_then(ReviewRound::contested_handoff)
6790 else {
6791 return;
6792 };
6793 self.state.event(
6794 "review",
6795 format!(
6796 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6797 wait for the owner's approval",
6798 contested.findings.len(),
6799 contested.rejecters.len()
6800 ),
6801 );
6802 self.state.contested_handoff = Some(contested);
6803 }
6804
6805 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6806 let round_idx = self.state.reviews.len() - 1;
6807 let needs_catchup_run = matches!(
6815 self.state.reviews[round_idx].e2e_status(),
6816 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6817 );
6818 if needs_catchup_run {
6819 let round = self.state.reviews[round_idx].round;
6820 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6821 let commands = self.state.config.verify.e2e.clone();
6822 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6823 let cache_dir = self.state.config.cache_dir();
6824 let context = format!(
6825 "round {round}: verification unresolved, catching up before the final decision"
6826 );
6827 let wait = self.repository_wait();
6828 let (outcomes, verify_retried) = with_cache_lease(
6829 &mut self.state,
6830 cache_dir.as_deref(),
6831 "e2e",
6832 "e2e",
6833 worktree,
6834 &attempted_head,
6835 timeout,
6836 &context,
6837 &wait,
6838 |state, budget| {
6839 let shell = shell.to_vec();
6840 let commands = commands.clone();
6841 let context = context.clone();
6842 async move {
6843 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6844 .await
6845 }
6846 },
6847 )
6848 .await?;
6849 let last = &mut self.state.reviews[round_idx];
6850 last.e2e = outcomes;
6851 last.verify_retried = verify_retried;
6852 last.verified_head = Some(attempted_head);
6859 last.verified_at = Some(Timestamp::now());
6860 if verify_inconclusive(&last.e2e) {
6861 self.state.save()?;
6868 return Ok(());
6869 }
6870 last.e2e_deferred = false;
6871 }
6872 let last = &self.state.reviews[round_idx];
6873 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6874
6875 match last.e2e_status() {
6876 E2eStatus::Failed => {
6877 let red: Vec<String> = last
6878 .e2e
6879 .iter()
6880 .filter(|o| !o.ok())
6881 .map(|o| {
6882 format!(
6883 "`{}` -> {:?}\n{}",
6884 o.command,
6885 o.code,
6886 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6887 )
6888 })
6889 .collect();
6890 self.state
6891 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6892 self.state.status = RunStatus::Blocked;
6893 }
6894 E2eStatus::ResourceBlocked => {
6899 self.state.event(
6900 "review",
6901 format!(
6902 "{why}; e2e could not run (shared build cache unavailable); not \
6903 deciding yet"
6904 ),
6905 );
6906 }
6907 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6908 self.state.event(
6909 "review",
6910 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6911 );
6912 self.record_contested_handoff();
6913 self.state.status = RunStatus::Gating;
6914 }
6915 }
6916 self.state.save()?;
6917 Ok(())
6918 }
6919
6920 async fn gate(&mut self) -> Result<()> {
6923 if self.state.status == RunStatus::Failed
6935 || self
6936 .state
6937 .base_sync
6938 .as_ref()
6939 .is_some_and(|s| s.conflict.is_some())
6940 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6941 != Some(RunStatus::Gating)
6942 {
6943 return Ok(());
6944 }
6945 if self.state.gate_ran {
6946 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6957 self.state.status = RunStatus::Blocked;
6958 self.state.save()?;
6959 }
6960 return Ok(());
6961 }
6962 let Some(winner) = self.state.winner().cloned() else {
6963 return Ok(());
6964 };
6965 self.state.status = RunStatus::Gating;
6966 let mut outcomes = self.run_gate(&winner).await?;
6967 loop {
6968 if verify_inconclusive(&outcomes) {
6979 self.state.save()?;
6980 return Ok(());
6981 }
6982 if outcomes.iter().all(CommandOutcome::ok) {
6983 break;
6984 }
6985 match self.gate_fix_round(&winner, &outcomes).await? {
6986 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6987 GateFix::Stop => break,
6988 GateFix::Defer => {
6989 self.state.save()?;
6990 return Ok(());
6991 }
6992 }
6993 }
6994 let passed = outcomes.iter().all(CommandOutcome::ok);
6995 self.state.gate = outcomes;
6996 self.state.gate_ran = true;
6997 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
6998 if !passed {
6999 self.state.status = RunStatus::Blocked;
7000 let spent = self.state.gate_fixes.len();
7001 self.state.event(
7002 "gate",
7003 if spent == 0 {
7004 "gate failed; not merging".to_owned()
7005 } else {
7006 format!("gate failed after {spent} gate-fix round(s); not merging")
7007 },
7008 );
7009 }
7010 self.state.save()?;
7011 Ok(())
7012 }
7013
7014 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7024 let commands = state.config.verify.pre_gate.clone();
7025 if commands.is_empty() {
7026 return;
7027 }
7028 let shell = state.config.shell();
7029 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7030 let (outcomes, _) = run_commands(
7031 state,
7032 "pre_gate",
7033 "pre_gate",
7034 0,
7035 &shell,
7036 &commands,
7037 &winner.worktree,
7038 timeout,
7039 )
7040 .await;
7041 for o in &outcomes {
7042 if !o.ok() {
7043 tracing::warn!(
7044 "pre_gate `{}` failed ({:?}); the gate decides",
7045 o.command,
7046 o.code
7047 );
7048 }
7049 state.event(
7050 "pre_gate",
7051 format!(
7052 "`{}` -> {}",
7053 o.command,
7054 if o.ok() {
7055 "pass".to_owned()
7056 } else {
7057 format!(
7058 "FAIL ({:?})\n{}",
7059 o.code,
7060 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7061 )
7062 }
7063 ),
7064 );
7065 }
7066 state.pre_gate = outcomes;
7067 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7068 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7069 Ok(head) => {
7070 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7071 state.pre_gate_commit = Some(head);
7072 }
7073 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7074 },
7075 Ok(false) => {}
7076 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7077 }
7078 if let Err(e) = state.save() {
7079 tracing::warn!("could not persist the pre_gate record: {e:#}");
7080 }
7081 }
7082
7083 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7086 let wait = self.repository_wait();
7087 run_landing_gate(&mut self.state, winner, &wait).await
7088 }
7089
7090 async fn gate_fix_round(
7103 &mut self,
7104 winner: &Candidate,
7105 outcomes: &[CommandOutcome],
7106 ) -> Result<GateFix> {
7107 let cap = self.state.config.graph.gate_fix_rounds;
7108 let spent = self.state.gate_fixes.len();
7109 if spent >= cap {
7110 if cap > 0 {
7111 self.state.event(
7112 "gate",
7113 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7114 );
7115 }
7116 return Ok(GateFix::Stop);
7117 }
7118 if !gate_fixable(outcomes) {
7119 self.state.event(
7120 "gate",
7121 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7122 command or similar); not spending a fix round on it",
7123 );
7124 return Ok(GateFix::Stop);
7125 }
7126 let min_free = self.state.config.disk.min_free_bytes;
7127 if min_free > 0 {
7128 match crate::disk::free_bytes(&winner.worktree) {
7129 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7130 Ok(free) => {
7131 self.state.event(
7132 "gate",
7133 format!(
7134 "only {free} bytes free ({min_free} required by `[disk] \
7135 min_free_bytes`); not spending a fix round on a failure the disk \
7136 may explain"
7137 ),
7138 );
7139 return Ok(GateFix::Stop);
7140 }
7141 Err(e) => {
7142 self.state.event(
7143 "gate",
7144 format!("free disk space could not be measured ({e:#}); no fix round"),
7145 );
7146 return Ok(GateFix::Stop);
7147 }
7148 }
7149 }
7150
7151 let attempt = spent + 1;
7152 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7153 let base = self.landing_base();
7154 let fix_prompt = prompt::gate_fix_in(
7155 &self.state.instruction,
7156 &failed,
7157 attempt,
7158 cap,
7159 &self.state.config.graph.language,
7160 &self.state.config.graph.github_language,
7161 );
7162 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7163 let sessions = self.state.config.graph.sessions;
7164 let artifacts = self.merge_repair_artifacts();
7165 self.state.event(
7166 "gate",
7167 format!("gate failed; gate-fix round {attempt} of {cap}"),
7168 );
7169 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7170 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7171 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7174 head: before.clone(),
7175 passed: false,
7176 outcomes: Vec::new(),
7177 });
7178 self.state.save()?;
7179 let (_, seat, out) = self
7180 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7181 prompt: fix_prompt.clone(),
7182 spec,
7183 seat,
7184 cwd: winner.worktree.clone(),
7185 timeout,
7186 allow_write: true,
7187 sessions,
7188 artifacts: artifacts.clone(),
7189 stem: format!("gate-fix-{attempt}"),
7190 handover: None,
7191 })
7192 .await;
7193 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7196 return Ok(GateFix::Defer);
7197 }
7198 let mut record = GateFixRecord {
7199 agent: seat.agent.clone(),
7200 failed,
7201 notes: String::new(),
7202 committed: false,
7203 error: None,
7204 };
7205 match out {
7206 AgentOutcome::Ok(o) => {
7207 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7210 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7211 }
7212 }
7213 AgentOutcome::Dropped(_) => {
7214 record.error = Some("the CLI dropped the stream".to_owned());
7215 }
7216 AgentOutcome::Quota(o) => {
7217 self.state.quota.push(QuotaLoss {
7218 seat: seat.key.clone(),
7219 node: "gate-fix".to_owned(),
7220 at: Timestamp::now(),
7221 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7222 });
7223 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7224 }
7225 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7226 }
7227 self.state.seats.insert(seat.key.clone(), seat);
7228 let coordination = self.coordinate(&winner.worktree).await?;
7229 if let Ok(r) = git::rescue_commit(
7230 &winner.worktree,
7231 &format!("magi: gate fix {attempt} (uncommitted work)"),
7232 )
7233 .await
7234 {
7235 self.state.note_withheld("gate-fix", &r.withheld);
7236 }
7237 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7238 record.committed = after != before;
7239 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7240 let note = record.error.clone();
7241 self.state.gate_fixes.push(record);
7242 self.state.save()?;
7243 if !changed {
7244 self.state.event(
7245 "gate",
7246 match note {
7247 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7248 None => format!("gate-fix round {attempt}: the tree did not change"),
7249 },
7250 );
7251 return Ok(GateFix::Stop);
7252 }
7253 self.state.event(
7254 "gate",
7255 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7256 );
7257
7258 drop(coordination);
7259 let wait = self.repository_wait();
7260 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7261 }
7262
7263 async fn guarded_pr_message(
7270 &mut self,
7271 winner: &Candidate,
7272 facts: Option<&BranchFacts>,
7273 posting: bool,
7274 ) -> Result<Option<PrMessage>> {
7275 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7276 let head = match final_head.as_deref() {
7277 Some(h) => crate::pr_outcome::FinalHead::At(h),
7278 None => crate::pr_outcome::FinalHead::Unknown,
7279 };
7280 let summarized = if posting {
7282 Some(self.summarize_change(winner, final_head.as_deref()).await)
7283 } else {
7284 None
7285 };
7286 if self.state.resource_block.is_some() {
7291 self.state.event(
7292 "pr-summary",
7293 "a shared resource was busy; nothing was published, resume to summarize again",
7294 );
7295 return Ok(None);
7296 }
7297 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7298 anyhow::bail!(
7299 "the final commit moved during public description generation; retry before publishing"
7300 );
7301 }
7302 let note = match (&summarized, &self.state.change_summary) {
7303 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7304 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7305 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7306 (None, _) => ChangeNote::NotAsked,
7307 };
7308 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7309 if !posting {
7310 let identity = crate::scrub::Identity::current();
7311 return Ok(Some(PrMessage {
7312 title: crate::scrub::scrub(&pr.title, &identity),
7313 body: crate::scrub::scrub(&pr.body, &identity),
7314 }));
7315 }
7316 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7319 if let Some(known) = self
7320 .state
7321 .github_text
7322 .clone()
7323 .filter(|g| g.fingerprint == raw_fp)
7324 {
7325 return self.settle_withheld(winner, &pr, known).await;
7326 }
7327 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7328 let mut violations =
7329 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7330 let identity = crate::scrub::Identity::current();
7331 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7332 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7333 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7334 {
7335 violations.push(crate::github_text::Violation::SensitiveData);
7336 }
7337 let has_language = violations
7341 .iter()
7342 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7343 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7344 self.state.event(
7345 "github-text",
7346 if has_language {
7347 format!("description rejected: {violations:?}; requesting one rewrite")
7348 } else {
7349 format!("description rejected: {violations:?}; asking the owner")
7350 },
7351 );
7352 let mut rewritten = false;
7353 let mut rejected_rewrite = None;
7354 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7355 {
7356 let output = text;
7357 if let Some(title) = summary_title(&output) {
7358 let raw_body = summary_without_title(&output);
7361 let clean_title = crate::scrub::scrub(&title, &identity);
7362 let mut body = crate::scrub::scrub(&raw_body, &identity);
7363 let redacted = clean_title != title || body != raw_body;
7364 let title = clean_title;
7365 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7366 if !complete {
7367 self.state.event(
7368 "github-text",
7369 "description rewrite omitted required findings",
7370 );
7371 }
7372 if !body.trim().is_empty() {
7373 body.push_str("\n\n");
7374 body.push_str(&public_pr_evidence(&self.state, head));
7375 }
7376 let marker = format!("magi:run/{}", self.state.id);
7377 if !body.trim().is_empty() && !body.contains(&marker) {
7378 body.push_str(&format!("\n\n{marker}"));
7379 }
7380 let rewrite = PrMessage {
7381 title: title.clone(),
7382 body: body.clone(),
7383 };
7384 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7385 let mut failed = crate::github_text::check_for(
7386 &self.state.config.graph,
7387 &title,
7388 &body,
7389 redecision,
7390 );
7391 if (!crate::github_text::shareable(&title)
7392 || !crate::github_text::shareable(&body))
7393 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7394 {
7395 failed.push(crate::github_text::Violation::SensitiveData);
7396 }
7397 if complete
7398 && !title.trim().is_empty()
7399 && !body.trim().is_empty()
7400 && !failed.is_empty()
7401 {
7402 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7403 title: title.clone(),
7404 body: body.clone(),
7405 failed: failed
7406 .iter()
7407 .map(|v| crate::github_text::category(*v).to_owned())
7408 .collect(),
7409 });
7410 }
7411 if complete
7412 && !title.trim().is_empty()
7413 && !raw_body.trim().is_empty()
7414 && failed.is_empty()
7415 {
7416 decision = redecision;
7417 pr = PrMessage { title, body };
7418 rewritten = true;
7419 self.state
7420 .event("github-text", "description rewrite passed");
7421 if redacted {
7422 self.state.event(
7423 "github-text",
7424 "sensitive data removed from description rewrite",
7425 );
7426 }
7427 }
7428 }
7429 }
7430 if !rewritten {
7431 self.state.event(
7432 "github-text",
7433 "description rewrite unavailable or rejected; asking the owner",
7434 );
7435 if let Some(w) =
7436 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7437 {
7438 self.retire_github_text_question("the withheld text changed");
7441 let record = crate::run::GithubTextAsk {
7442 rewrite: rejected_rewrite,
7443 fingerprint: raw_fp,
7444 title: w.title,
7445 body: w.body,
7446 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7447 question: None,
7448 asks: 0,
7449 resolved: false,
7450 chosen_title: None,
7451 choice: None,
7452 };
7453 return self.settle_withheld(winner, &pr, record).await;
7454 }
7455 }
7456 }
7457 let (title, body) =
7458 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7459 Ok(Some(PrMessage { title, body }))
7460 }
7461
7462 async fn summarize_change(
7473 &mut self,
7474 winner: &Candidate,
7475 head: Option<&str>,
7476 ) -> std::result::Result<(), String> {
7477 let Some(head) = head else {
7478 return Err("the final commit could not be read".to_owned());
7479 };
7480 let remote = self.state.config.merge.remote.clone();
7481 let tracking = format!("{remote}/{}", self.state.base_branch);
7482 if !git::rev_exists(&winner.worktree, &tracking).await {
7483 return Err(format!("the remote base `{tracking}` could not be read"));
7484 }
7485 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7486 Ok(b) if !b.is_empty() => b,
7487 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7488 };
7489 let evidence = crate::pr_outcome::summary_evidence(
7490 &self.state,
7491 crate::pr_outcome::FinalHead::At(head),
7492 );
7493 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7494 if self
7495 .state
7496 .change_summary
7497 .as_ref()
7498 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7499 {
7500 return Ok(());
7501 }
7502 if self.state.change_summary.take().is_some() {
7503 self.state.event(
7504 "pr-summary",
7505 "the final diff or evidence changed; the earlier change summary was discarded",
7506 );
7507 }
7508 if let Err(e) = write_artifact(
7510 &self.state,
7511 &format!("public-description-{fingerprint}.md"),
7512 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7513 ) {
7514 tracing::warn!("could not save public description evidence: {e:#}");
7515 }
7516 let chain = match agent::pick_chain(
7517 &self.state.config.agents,
7518 self.state.config.roles.synthesizer.as_ref(),
7519 &agent::installed,
7520 "synthesizer",
7521 ) {
7522 Ok(chain) => chain,
7523 Err(e) => {
7524 tracing::warn!("no agent for the change summary: {e:#}");
7525 self.state.event("pr-summary", "no summary agent available");
7526 return Err("no agent was available to summarize it".to_owned());
7527 }
7528 };
7529 let subjects = git::subjects(&winner.worktree, &base, head)
7530 .await
7531 .unwrap_or_default();
7532 let stat = git::diff_stat(&winner.worktree, &base, head)
7533 .await
7534 .unwrap_or_default();
7535 let stat: String = stat.chars().take(6_000).collect();
7536 let mut prompt = crate::prompt::change_summary_in(
7537 &base,
7538 head,
7539 &subjects,
7540 stat.trim(),
7541 &self.state.config.graph.github_language,
7542 );
7543 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7544 let sessions = self.state.config.graph.sessions;
7545 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7546 let prompts = self.state.config.prompts.clone();
7547 let cache = self.state.config.cache_dir();
7548 let run = self.state.id.clone();
7549 for (i, spec) in chain.into_iter().enumerate() {
7550 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7551 let job = SeatJob {
7552 spec: spec.clone(),
7553 seat,
7554 cwd: winner.worktree.clone(),
7555 prompt: prompt.clone(),
7556 timeout,
7557 allow_write: false,
7558 sessions,
7559 artifacts: agent::artifacts_dir(&self.state.dir()),
7560 stem: if i == 0 {
7561 "change-summary".to_owned()
7562 } else {
7563 format!("change-summary-{}", spec.id)
7564 },
7565 handover: None,
7566 };
7567 let ctx = WaveCtx {
7568 run: &run,
7569 node: "pr-summary",
7570 prompts: &prompts,
7571 cache: cache.as_deref(),
7572 round: None,
7573 carry_seats: false,
7574 pauses: Some((&self.pause, &self.interrupt)),
7575 };
7576 let (seat, outcome) =
7577 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7578 self.state.seats.insert(seat.key.clone(), seat);
7579 let text = match outcome {
7580 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7581 AgentOutcome::ResourceBlocked(why) => {
7582 if self.state.resource_block.is_none() {
7585 self.state.resource_block = Some(crate::run::ResourceBlock {
7586 node: "pr-summary".to_owned(),
7587 kind: "shared resource".to_owned(),
7588 detail: why,
7589 at: Timestamp::now(),
7590 });
7591 }
7592 return Err("a shared resource was busy".to_owned());
7593 }
7594 _ => continue,
7595 };
7596 let blind = &self.state.config.blind;
7597 let Some(title) = summary_title(&text) else {
7598 continue;
7599 };
7600 let body: String = summary_without_title(&text)
7601 .lines()
7602 .filter(|l| !l.contains("magi:run/"))
7603 .collect::<Vec<_>>()
7604 .join("\n");
7605 let body = blind::sanitize_prose(body.trim(), blind);
7606 let title = blind::sanitize_prose(&title, blind);
7607 if body.trim().is_empty() || title.trim().is_empty() {
7608 continue;
7609 }
7610 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7611 continue;
7612 }
7613 if git::rev_parse(&winner.worktree, "HEAD")
7615 .await
7616 .ok()
7617 .as_deref()
7618 != Some(head)
7619 {
7620 return Err("the branch moved while the summary was written".to_owned());
7621 }
7622 if git::merge_base(&winner.worktree, &tracking, head)
7623 .await
7624 .ok()
7625 .as_deref()
7626 != Some(base.as_str())
7627 || crate::pr_outcome::summary_evidence(
7628 &self.state,
7629 crate::pr_outcome::FinalHead::At(head),
7630 ) != evidence
7631 {
7632 return Err("the summary evidence changed while it was written".to_owned());
7633 }
7634 self.state.change_summary = Some(crate::run::ChangeSummary {
7635 head: head.to_owned(),
7636 base,
7637 evidence: fingerprint,
7638 title,
7639 body,
7640 });
7641 self.state.event(
7642 "pr-summary",
7643 format!("change summary written by {} for the final diff", spec.id),
7644 );
7645 if let Err(e) = self.state.save() {
7646 tracing::warn!("could not save the change summary: {e:#}");
7647 }
7648 return Ok(());
7649 }
7650 self.state
7651 .event("pr-summary", "no agent produced a change summary");
7652 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7653 }
7654
7655 async fn rewrite_pr_text(
7663 &mut self,
7664 winner: &Candidate,
7665 pr: &PrMessage,
7666 violations: &[crate::github_text::Violation],
7667 ) -> Option<String> {
7668 let prompt = format!(
7669 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7670 pr.title, pr.body
7671 );
7672 let sessions = self.state.config.graph.sessions;
7673 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7674 let mut attempts: Vec<(AgentSpec, SeatState, bool)> = Vec::new();
7676 let key = format!("impl-{}", winner.label);
7677 if let Some(spec) = self
7678 .state
7679 .config
7680 .agents
7681 .iter()
7682 .find(|a| a.id == winner.agent)
7683 .cloned()
7684 && let Some(existing) = self.state.seats.get(&key)
7685 && existing.agent == spec.id
7686 && agent::has_session(spec.kind, existing, sessions)
7687 {
7688 attempts.push((spec, existing.clone(), true));
7689 } else {
7690 let chain = agent::pick_chain(
7691 &self.state.config.agents,
7692 self.state.config.roles.language_judge.as_ref(),
7693 &agent::installed,
7694 "language_judge",
7695 );
7696 match chain {
7697 Ok(chain) => {
7698 for spec in chain {
7699 let seat = crate::fixer::seat_for(
7700 &mut self.state,
7701 "github-text-rewrite",
7702 &spec.id,
7703 );
7704 attempts.push((spec, seat, false));
7705 }
7706 }
7707 Err(e) => {
7708 tracing::warn!("no agent for the description rewrite: {e:#}");
7709 self.state
7710 .event("github-text", "no rewrite agent available");
7711 return None;
7712 }
7713 }
7714 }
7715 let prompts = self.state.config.prompts.clone();
7716 let cache = self.state.config.cache_dir();
7717 let run = self.state.id.clone();
7718 for (i, (spec, seat, resumed)) in attempts.into_iter().enumerate() {
7719 let job = SeatJob {
7720 spec: spec.clone(),
7721 seat,
7722 cwd: winner.worktree.clone(),
7723 prompt: prompt.clone(),
7724 timeout: retry_budget(timeout_full, resumed),
7725 allow_write: false,
7726 sessions,
7727 artifacts: agent::artifacts_dir(&self.state.dir()),
7728 stem: if i == 0 {
7729 "github-text-rewrite".to_owned()
7730 } else {
7731 format!("github-text-rewrite-{}", spec.id)
7732 },
7733 handover: None,
7734 };
7735 let ctx = WaveCtx {
7736 run: &run,
7737 node: "github-text",
7738 prompts: &prompts,
7739 cache: cache.as_deref(),
7740 round: None,
7741 carry_seats: false,
7742 pauses: Some((&self.pause, &self.interrupt)),
7743 };
7744 let (seat, outcome) =
7745 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7746 self.state.seats.insert(seat.key.clone(), seat);
7747 match outcome {
7748 AgentOutcome::Ok(o)
7749 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7750 {
7751 return Some(o.text);
7752 }
7753 AgentOutcome::ResourceBlocked(_) => return None,
7754 _ => {}
7755 }
7756 }
7757 None
7758 }
7759
7760 fn retire_github_text_question(&mut self, why: &str) {
7762 let Some(id) = self
7763 .state
7764 .github_text
7765 .as_ref()
7766 .and_then(|g| g.question.clone())
7767 else {
7768 return;
7769 };
7770 let _ = ask::Questions::open().update(&id, |q| {
7771 q.abandon(why);
7772 Ok(())
7773 });
7774 }
7775
7776 async fn settle_withheld(
7780 &mut self,
7781 winner: &Candidate,
7782 pr: &PrMessage,
7783 mut g: crate::run::GithubTextAsk,
7784 ) -> Result<Option<PrMessage>> {
7785 use crate::github_text as gt;
7786 let store = ask::Questions::open();
7787 let timeout = self.state.config.graph.answer_timeout;
7788 if !g.resolved {
7789 let mut standing = g
7790 .question
7791 .as_ref()
7792 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7793 if let Some(q) = standing.as_ref()
7794 && q.status.open()
7795 && gt::expired(q, timeout)
7796 {
7797 let why = format!("no answer within {}s of asking", timeout.max(1));
7798 standing = store
7799 .update(&q.id, |q| {
7800 q.abandon(&why);
7801 Ok(())
7802 })
7803 .ok()
7804 .map(|(q, ())| q);
7805 }
7806 let mut ask_again: Option<Option<Vec<String>>> = None;
7807 match standing {
7808 None if g.asks <= 1 => {
7811 g.asks = 0;
7812 ask_again = Some(None);
7813 }
7814 None => {
7815 g.resolved = true;
7816 g.chosen_title = None;
7817 g.choice = Some(GithubTextChoice::Fallback);
7818 }
7819 Some(q) if q.status.open() => {
7820 self.park_on_github_text(&g)?;
7821 return Ok(None);
7822 }
7823 Some(q) => match gt::read_reply(&q) {
7824 gt::Reply::Fallback => {
7825 g.resolved = true;
7826 g.chosen_title = None;
7827 g.choice = Some(GithubTextChoice::Fallback);
7828 }
7829 gt::Reply::Original => {
7830 g.resolved = true;
7831 g.chosen_title = None;
7832 g.choice = Some(GithubTextChoice::Original);
7833 }
7834 gt::Reply::Rewrite => {
7835 g.resolved = true;
7836 g.chosen_title = None;
7837 g.choice = Some(if g.rewrite.is_some() {
7838 GithubTextChoice::Rewrite
7839 } else {
7840 GithubTextChoice::Fallback
7841 });
7842 }
7843 gt::Reply::Title(text) => {
7844 let first = text
7845 .lines()
7846 .map(str::trim)
7847 .find(|l| !l.is_empty())
7848 .unwrap_or("");
7849 let decision = if first.is_empty() {
7850 None
7851 } else {
7852 let candidate = PrMessage {
7853 title: first.to_owned(),
7854 body: String::new(),
7855 };
7856 self.judge_pr_language(&winner.worktree, &candidate).await
7857 };
7858 match gt::vet_title(
7859 &text,
7860 decision,
7861 self.state.config.graph.github_english(),
7862 ) {
7863 Ok(title) => {
7864 g.resolved = true;
7865 g.chosen_title = Some(title);
7866 g.choice = Some(GithubTextChoice::MyTitle);
7867 }
7868 Err(cats) if g.asks < 2 => {
7869 ask_again = Some(Some(
7870 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7871 ))
7872 }
7873 Err(_) => {
7874 g.resolved = true;
7875 g.chosen_title = None;
7876 g.choice = Some(GithubTextChoice::Fallback);
7877 }
7878 }
7879 }
7880 },
7881 }
7882 if let Some(rejected) = ask_again {
7883 let w = gt::Withheld {
7886 title: g.title,
7887 body: g.body,
7888 categories: g.categories.clone(),
7889 };
7890 let artifact = match crate::run::write_artifact(
7893 &self.state,
7894 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7895 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7896 ) {
7897 Ok(p) => Some(p),
7898 Err(e) => {
7899 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7900 None
7901 }
7902 };
7903 let (detail, comparison) = gt::question_content(
7904 &w,
7905 &pr.title,
7906 &pr.body,
7907 &self.state.id,
7908 rejected.as_deref(),
7909 artifact.as_deref(),
7910 g.rewrite.as_ref(),
7911 );
7912 let mut q = ask::Question::new(
7913 self.state.id.clone(),
7914 gt::ASK_NODE.to_owned(),
7915 gt::ASK_SEAT.to_owned(),
7916 gt::question_summary(&self.state.config.graph.language, &w),
7917 detail,
7918 gt::question_choices(&w, g.rewrite.as_ref()),
7919 );
7920 q.posting_comparison = Some(comparison);
7921 q.answer_timeout = timeout;
7922 g.question = Some(q.id.clone());
7925 g.asks += 1;
7926 self.state.github_text = Some(g.clone());
7927 self.state.event(
7928 "github-text",
7929 format!("asking the owner about the withheld text ({})", q.short()),
7930 );
7931 self.state.save()?;
7932 store
7933 .put_executable(&mut q)
7934 .context("file the posting-gate question")?;
7935 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7936 tracing::warn!(
7937 "could not notify about posting-gate question {}: {e:#}",
7938 q.short()
7939 );
7940 }
7941 self.park_on_github_text(&g)?;
7942 return Ok(None);
7943 }
7944 }
7945 let resolved = g.chosen_title.clone();
7946 let original = g.choice == Some(GithubTextChoice::Original);
7947 self.state.event(
7949 "github-text",
7950 if original {
7951 format!(
7952 "posting the original text for: {}",
7953 match (g.title, g.body) {
7954 (true, true) => "title, description",
7955 (true, false) => "title",
7956 _ => "description",
7957 }
7958 )
7959 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7960 "posting the saved rewrite title and description".to_owned()
7961 } else if resolved.is_some() {
7962 "posting the owner's replacement title".to_owned()
7963 } else {
7964 "posting the neutral text for the withheld fields".to_owned()
7965 },
7966 );
7967 self.state.github_text = Some(g.clone());
7968 self.state.save()?;
7969 if g.choice == Some(GithubTextChoice::Rewrite)
7970 && let Some(r) = g.rewrite.as_ref()
7971 {
7972 return Ok(Some(PrMessage {
7973 title: r.title.clone(),
7974 body: r.body.clone(),
7975 }));
7976 }
7977 let w = gt::Withheld {
7978 title: g.title,
7979 body: g.body,
7980 categories: g.categories.clone(),
7981 };
7982 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
7983 if let Some(t) = resolved {
7984 title = t;
7985 }
7986 let english = gt::LanguageDecision {
7988 title_english: true,
7989 body_english: true,
7990 body_complete: true,
7991 };
7992 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
7993 if original {
7994 if g.title {
7997 title = pr.title.clone();
7998 }
7999 if g.body {
8000 body = pr.body.clone();
8001 }
8002 }
8003 Ok(Some(PrMessage { title, body }))
8004 }
8005
8006 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8007 self.state.github_text = Some(g.clone());
8008 self.state.end_pass_parked(ParkCause::Question);
8009 self.state.parked = true;
8010 self.state.event(
8011 "github-text",
8012 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8013 );
8014 self.state.save()
8015 }
8016
8017 async fn judge_pr_language(
8020 &mut self,
8021 cwd: &Path,
8022 pr: &PrMessage,
8023 ) -> Option<crate::github_text::LanguageDecision> {
8024 self.state.config.roles.language_judge.as_ref()?;
8025 if !self.state.config.graph.language_gate() {
8026 return None;
8027 }
8028 let decision =
8029 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8030 self.state.event(
8031 "github-text",
8032 if decision.is_some() {
8033 "language decided by the language judge"
8034 } else {
8035 "language judge unavailable; using built-in heuristics"
8036 },
8037 );
8038 decision
8039 }
8040
8041 async fn merge(&mut self) -> Result<()> {
8042 if self
8057 .state
8058 .base_sync
8059 .as_ref()
8060 .is_some_and(|s| s.conflict.is_some())
8061 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8062 != Some(RunStatus::Gating)
8063 || !self.state.gate_allows_progress()
8071 {
8072 return Ok(());
8073 }
8074 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8083 return Ok(());
8084 }
8085 let Some(winner) = self.state.winner().cloned() else {
8086 return Ok(());
8087 };
8088 let repo = self.state.repo.clone();
8089 let base = self.state.base_branch.clone();
8090 let mode = self.state.config.merge.mode;
8091 let style = self.state.config.merge.style;
8092 let facts = if is_review_run(&self.state) {
8093 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8094 let start = review_base(
8095 &repo,
8096 &self.state.config.merge.remote,
8097 &base,
8098 &self.state.base_commit,
8099 &winner.branch,
8100 )
8101 .await;
8102 branch_facts(&repo, &start, &winner.branch).await
8103 } else {
8104 None
8105 };
8106 let leaked = if is_review_run(&self.state) {
8109 leaked_subjects(&self.state, &winner.branch).await
8110 } else {
8111 Some(Vec::new())
8112 };
8113 let Some(pr) = self
8117 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8118 .await?
8119 else {
8120 return Ok(());
8121 };
8122 let message = pr.commit_message();
8123
8124 let outcome = match mode {
8125 MergeMode::None => MergeOutcome {
8126 mode,
8127 ok: true,
8128 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8129 empty: false,
8130 },
8131 MergeMode::Pr | MergeMode::Local
8132 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8133 {
8134 MergeOutcome {
8135 mode,
8136 ok: false,
8137 detail: empty_candidate_detail(&self.state, &base),
8138 empty: true,
8139 }
8140 }
8141 MergeMode::Local => {
8142 let on = git::current_branch(&repo).await?;
8143 if on.as_deref() != Some(base.as_str()) {
8144 MergeOutcome {
8145 mode,
8146 ok: false,
8147 detail: format!(
8148 "{} has {} checked out, not the base branch {base}",
8149 repo.display(),
8150 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8151 ),
8152 empty: false,
8153 }
8154 } else if !git::is_clean(&repo).await? {
8155 MergeOutcome {
8156 mode,
8157 ok: false,
8158 detail: format!("{} is dirty; refusing to merge", repo.display()),
8159 empty: false,
8160 }
8161 } else {
8162 let out = match style {
8163 MergeStyle::Merge => {
8164 git::merge_no_ff(&repo, &winner.branch, &message).await?
8165 }
8166 MergeStyle::Squash => {
8167 git::merge_squash(&repo, &winner.branch, &message).await?
8168 }
8169 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8170 };
8171 MergeOutcome {
8172 mode,
8173 ok: out.ok(),
8174 detail: if out.ok() { out.stdout } else { out.stderr },
8175 empty: false,
8176 }
8177 }
8178 }
8179 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8180 self.state.pr_publication_pending = true;
8181 self.state.save()?;
8182 let out =
8183 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8184 self.state.pr_publication_pending = out.is_err();
8185 MergeOutcome {
8186 mode,
8187 ok: out.is_ok(),
8188 detail: match out {
8189 Ok(url) => url,
8190 Err(e) => format!("{e:#}"),
8191 },
8192 empty: false,
8193 }
8194 }
8195 MergeMode::Pr => {
8196 let remote = self.state.config.merge.remote.clone();
8197 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8198 if !pushed.ok() {
8199 MergeOutcome {
8200 mode,
8201 ok: false,
8202 detail: pushed.stderr,
8203 empty: false,
8204 }
8205 } else {
8206 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8212 let out = match pr_merge_plan(found) {
8213 PrPlan::Create => {
8214 gh_pr_create(
8215 &winner.worktree,
8216 &base,
8217 &winner.branch,
8218 &pr.title,
8219 &pr.body,
8220 )
8221 .await
8222 }
8223 PrPlan::Adopt { url, title } => {
8224 self.state
8225 .event("merge", format!("Pr: adopted open pull request {url}"));
8226 if self.state.github_text.as_ref().is_some_and(|g| {
8227 g.resolved
8228 && g.choice == Some(GithubTextChoice::Rewrite)
8229 && g.rewrite.is_some()
8230 }) {
8231 land::set_pr_text_verbatim(
8234 &winner.worktree,
8235 &url,
8236 &pr.title,
8237 &pr.body,
8238 )
8239 .await
8240 .map(|()| url)
8241 } else {
8242 if title != pr.title
8243 && (!is_review_run(&self.state)
8244 || leaked
8245 .as_deref()
8246 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8247 && let Err(e) =
8248 if self.state.github_text.as_ref().is_some_and(|g| {
8249 g.resolved
8250 && g.title
8251 && g.choice == Some(GithubTextChoice::Original)
8252 }) {
8253 land::set_pr_title_verbatim(
8255 &winner.worktree,
8256 &url,
8257 &pr.title,
8258 )
8259 .await
8260 } else {
8261 land::set_pr_title(
8262 &mut self.state,
8263 &winner.worktree,
8264 &url,
8265 &pr.title,
8266 )
8267 .await
8268 }
8269 {
8270 tracing::warn!("could not refresh title of {url}: {e:#}");
8271 self.state
8272 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8273 }
8274 Ok(url)
8275 }
8276 }
8277 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8278 };
8279 match out {
8280 Ok(url) => MergeOutcome {
8281 mode,
8282 ok: true,
8283 detail: url,
8284 empty: false,
8285 },
8286 Err(e) => MergeOutcome {
8287 mode,
8288 ok: false,
8289 detail: e.to_string(),
8290 empty: false,
8291 },
8292 }
8293 }
8294 }
8295 };
8296
8297 self.state.status = match (mode, outcome.ok) {
8298 (MergeMode::None, _) => RunStatus::Ready,
8299 (_, true) if outcome.empty => RunStatus::Ready,
8300 (MergeMode::Pr, true) => RunStatus::Ready,
8301 (_, true) => RunStatus::Merged,
8302 (_, false) => RunStatus::Blocked,
8303 };
8304 self.state.event(
8305 "merge",
8306 format!(
8307 "{:?}: {}",
8308 mode,
8309 outcome.detail.lines().next().unwrap_or("")
8310 ),
8311 );
8312 self.state.merge = Some(outcome);
8313 self.state.save()?;
8314
8315 if self.state.config.graph.land
8321 && mode == MergeMode::Pr
8322 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8323 {
8324 self.run_land().await?;
8325 }
8326 self.settle_questions();
8331 Ok(())
8332 }
8333
8334 async fn run_land(&mut self) -> Result<()> {
8345 crate::pr_head::ensure_owner(&self.state)?;
8346 let url = self
8347 .state
8348 .pr
8349 .as_ref()
8350 .map(|pr| pr.url.clone())
8351 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8352 .unwrap_or_default();
8353 let url = url.lines().next().unwrap_or("").trim().to_owned();
8354 if !url.starts_with("http") {
8355 return Ok(());
8356 }
8357 loop {
8360 match land::land(&mut self.state, &url).await {
8361 Ok(_)
8362 if self
8363 .state
8364 .merge_repairs
8365 .last()
8366 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8367 {
8368 if self.repair_merge_review().await? {
8369 continue;
8370 }
8371 }
8372 Ok(pr) if self.state.parked => {
8373 let _ = pr;
8377 }
8378 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8379 self.state.status = RunStatus::Superseded;
8380 self.state.save()?;
8381 }
8382 Ok(pr) => {
8383 self.state.status = match pr.state {
8384 land::PrLifecycle::Merged => RunStatus::Merged,
8385 _ => RunStatus::Blocked,
8386 };
8387 if bump::should_release_bump(self.state.status)
8394 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8395 {
8396 self.state
8402 .event("bump", format!("release bump skipped: {e:#}"));
8403 }
8404 if self.state.status == RunStatus::Merged {
8407 crate::followup::after_merge(&mut self.state, &pr.url).await;
8408 }
8409 self.state.save()?;
8410 }
8411 Err(e) => {
8412 if e.is::<crate::repo_lock::ResourceBlocked>() {
8413 return Err(e);
8414 }
8415 self.state.status = RunStatus::Blocked;
8416 self.state.event("land", format!("gave up: {e}"));
8417 self.state.save()?;
8418 }
8419 }
8420 break;
8421 }
8422 Ok(())
8423 }
8424
8425 async fn repair_merge_review(&mut self) -> Result<bool> {
8428 use crate::run::MergeRepairPhase as Phase;
8429 crate::pr_head::ensure_owner(&self.state)?;
8430 let i = self.state.merge_repairs.len() - 1;
8431 let request = self.state.merge_repairs[i].clone();
8432 if request.phase == Phase::Fixing {
8433 let restored = match self.state.winner().cloned() {
8434 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8435 None => Err(anyhow::anyhow!("repair has no winning branch")),
8436 };
8437 let why = match restored {
8438 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8439 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8440 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8441 };
8442 self.hold_merge_repair(i, why)?;
8443 return Ok(false);
8444 }
8445 let failure = if i >= self.state.config.graph.land_rounds
8446 || self.state.config.graph.review_rounds == 0
8447 {
8448 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8449 } else {
8450 None
8451 };
8452 if let Some(why) = failure {
8453 self.hold_merge_repair(i, why)?;
8454 return Ok(false);
8455 }
8456 if request.phase == Phase::Requested {
8457 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8458 Ok(pr) => pr,
8459 Err(e) => {
8460 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8461 return Ok(false);
8462 }
8463 };
8464 self.state.merge_repairs[i].phase = Phase::Fixing;
8465 self.state.land_approval = None;
8466 self.state.land_armed_head = None;
8467 self.state.save()?;
8468 let evidence = serde_json::to_string_pretty(&request)?;
8471 let reason = format!(
8472 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8473 );
8474 let budget = self.state.config.graph.land_rounds;
8475 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8476 match result {
8477 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8478 Ok(land::Fixed::Failed(why)) => {
8479 self.hold_merge_repair(
8480 i,
8481 format!(
8482 "repair failed: {why}; inspect the branch and run an explicit review"
8483 ),
8484 )?;
8485 return Ok(false);
8486 }
8487 Err(e) => {
8488 self.hold_merge_repair(
8489 i,
8490 format!(
8491 "repair failed: {e:#}; inspect the branch and run an explicit review"
8492 ),
8493 )?;
8494 return Ok(false);
8495 }
8496 }
8497 self.begin_merge_repair_review(i)?;
8501 }
8502 self.review_loop().await?;
8503 self.gate().await?;
8504 if self.resource_halt()? || self.state.parked {
8505 return Ok(false);
8506 }
8507 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8508 != Some(RunStatus::Gating)
8509 || !self.state.gate_allows_progress()
8510 {
8511 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8512 return Ok(false);
8513 }
8514 let winner = self
8517 .state
8518 .winner()
8519 .cloned()
8520 .context("repair has no winning branch")?;
8521 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8522 if self.state.gate_head.as_deref() != Some(after.as_str())
8523 || !git::is_clean(&winner.worktree).await?
8524 {
8525 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8526 return Ok(false);
8527 }
8528 let expected = self
8529 .state
8530 .pr_pushed_head
8531 .as_deref()
8532 .unwrap_or(&request.head)
8533 .to_owned();
8534 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8535 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8538 let refspec = format!("{after}:refs/heads/{public}");
8539 let push = git::git_raw(
8540 &winner.worktree,
8541 &["push", &lease, &self.state.config.merge.remote, &refspec],
8542 )
8543 .await;
8544 match push {
8545 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8546 Ok(out) => {
8547 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8548 return Ok(false);
8549 }
8550 Err(e) => {
8551 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8552 return Ok(false);
8553 }
8554 }
8555 self.state.merge_repairs[i].phase = Phase::Complete;
8556 self.state.status = RunStatus::Landing;
8557 self.state.event(
8558 "repair",
8559 "new review and gate completed; fresh owner merge decision required",
8560 );
8561 self.state.save()?;
8562 Ok(true)
8563 }
8564
8565 fn merge_repair_artifacts(&self) -> PathBuf {
8566 let artifacts = agent::artifacts_dir(&self.state.dir());
8567 match self
8568 .state
8569 .merge_repairs
8570 .last()
8571 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8572 {
8573 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8574 None => artifacts,
8575 }
8576 }
8577
8578 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8579 self.state.reviews.clear();
8580 self.state.gate.clear();
8581 self.state.gate_ran = false;
8582 self.state.gate_head = None;
8583 self.state.gate_fixes.clear();
8584 self.state.gate_repair_e2e = None;
8585 self.state.pre_gate.clear();
8586 self.state.pre_gate_commit = None;
8587 self.state.contested_handoff = None;
8588 self.state.land_reruns = None;
8589 self.state.land_approval = None;
8590 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8591 self.state.status = RunStatus::Reviewing;
8592 self.state.save()
8593 }
8594
8595 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8596 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8597 self.state.merge_repairs[index].failure = Some(why.clone());
8598 self.state.status = RunStatus::Blocked;
8599 self.state.event("repair", why);
8600 self.state.save()
8601 }
8602
8603 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8607 if let Some(existing) = self.state.seats.get(key)
8608 && existing.agent == agent
8609 {
8610 return existing.clone();
8611 }
8612 let fresh = if self.state.seats.contains_key(key) {
8615 handover_seat(key, agent, self.state.next_seat_seed())
8616 } else {
8617 SeatState::new(key, agent, self.state.seed)
8618 };
8619 self.state.seats.insert(key.to_owned(), fresh.clone());
8620 fresh
8621 }
8622
8623 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8629 match self.state.seats.get(key) {
8630 Some(s) if s.agent != spec.id => {
8631 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8632 }
8633 _ => spec,
8634 }
8635 }
8636
8637 fn view(&self, c: &Candidate) -> CandidateView {
8639 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8640 .unwrap_or_default();
8641 let (patch, _) = blind::sanitize_patch(
8642 &format!("candidate {} patch", c.label),
8643 &raw,
8644 &self.state.config.blind,
8645 );
8646 CandidateView {
8647 label: c.label,
8648 branch: c.branch.clone(),
8649 summary: c.summary.clone(),
8650 stat: c.stat.clone(),
8651 patch,
8652 }
8653 }
8654
8655 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8657 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8658 prompt::judge(
8659 "(see above)",
8660 &views,
8661 self.roles.judges.len(),
8662 base_short,
8663 "en",
8664 )
8665 }
8666
8667 fn vote_prompt_full(
8673 &self,
8674 j: usize,
8675 labels: &[char],
8676 language: &str,
8677 candidates: &[Candidate],
8678 base_short: &str,
8679 ) -> String {
8680 let mut text = format!(
8681 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8682 prompt::final_vote(labels, language),
8683 self.state.instruction,
8684 self.candidate_block(candidates, base_short)
8685 );
8686 if let Some(own) = self
8687 .state
8688 .judgements
8689 .get(j)
8690 .filter(|r| !r.ranking.is_empty())
8691 {
8692 let reasons = own
8693 .reasons
8694 .iter()
8695 .map(|(k, v)| format!("- {k}: {v}"))
8696 .collect::<Vec<_>>()
8697 .join("\n");
8698 text.push_str(&format!(
8699 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8700 own.ranking.iter().collect::<String>(),
8701 if reasons.is_empty() {
8702 ""
8703 } else {
8704 ", because:\n"
8705 }
8706 ));
8707 }
8708 if !self.state.deliberation.is_empty() {
8709 text.push_str("\n# What was argued before this vote\n");
8710 for t in self.transcript(&[], j) {
8711 text.push_str(&format!(
8712 "\n## {}{}\n\n{}\n",
8713 t.who,
8714 if t.is_self { " (you)" } else { "" },
8715 t.body.trim()
8716 ));
8717 }
8718 }
8719 text
8720 }
8721
8722 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8729 let mut turns = Vec::new();
8730 for j in &self.state.judgements {
8731 if j.ranking.is_empty() {
8732 continue;
8733 }
8734 let reasons = j
8735 .reasons
8736 .iter()
8737 .map(|(k, v)| format!("- {k}: {v}"))
8738 .collect::<Vec<_>>()
8739 .join("\n");
8740 turns.push(Turn {
8741 who: format!("Judge {} (opening ranking)", j.judge),
8742 is_self: j.judge == self_idx + 1,
8743 body: format!(
8744 "Ranked {}{}{reasons}",
8745 j.ranking.iter().collect::<String>(),
8746 if reasons.is_empty() {
8747 ""
8748 } else {
8749 ", because:\n"
8750 }
8751 ),
8752 });
8753 }
8754 for t in self
8755 .state
8756 .deliberation
8757 .iter()
8758 .flat_map(|r| r.turns.iter())
8759 .chain(current)
8760 {
8761 turns.push(Turn {
8762 who: format!("Judge {}", t.judge),
8763 is_self: t.judge == self_idx + 1,
8764 body: t.body.clone(),
8765 });
8766 }
8767 turns
8768 }
8769}
8770
8771fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8773 agent::has_session(spec.kind, seat, sessions)
8774}
8775
8776fn next_untried_in_roster<'a>(
8797 roster: &'a [AgentSpec],
8798 start: usize,
8799 tried: &BTreeSet<String>,
8800) -> Option<&'a AgentSpec> {
8801 roster
8802 .get(start + 1..)?
8803 .iter()
8804 .find(|s| !tried.contains(&s.id))
8805}
8806
8807fn pick_successor<'a>(
8828 roster: &'a [AgentSpec],
8829 start: usize,
8830 tried: &BTreeSet<String>,
8831 carried: Option<&BTreeSet<String>>,
8832 others: &BTreeSet<String>,
8833) -> Option<&'a AgentSpec> {
8834 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8835 roster
8836 .get(start + 1..)
8837 .and_then(|tail| {
8838 tail.iter()
8839 .filter(free)
8840 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8841 })
8842 .or_else(|| {
8843 carried
8844 .is_some()
8845 .then(|| roster.iter().find(free))
8846 .flatten()
8847 })
8848 .or_else(|| match carried {
8849 Some(c) => next_for_seat(roster, start, tried, c),
8850 None => next_untried_in_roster(roster, start, tried),
8851 })
8852}
8853
8854fn next_for_seat<'a>(
8864 roster: &'a [AgentSpec],
8865 start: usize,
8866 round_tried: &BTreeSet<String>,
8867 carried_failed: &BTreeSet<String>,
8868) -> Option<&'a AgentSpec> {
8869 roster
8870 .get(start + 1..)?
8871 .iter()
8872 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8873 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8874}
8875
8876fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8882 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8883 return spec;
8884 };
8885 let ok = |id: &str| !h.failed.contains(id);
8886 if let Some(last) = h.last_ok.as_deref()
8887 && ok(last)
8888 && let Some(s) = roster.iter().find(|s| s.id == last)
8889 {
8890 return s.clone();
8891 }
8892 if ok(&spec.id) {
8893 return spec;
8894 }
8895 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8896 roster
8897 .get(from + 1..)
8898 .into_iter()
8899 .flatten()
8900 .chain(roster.iter())
8901 .find(|s| ok(&s.id))
8902 .cloned()
8903 .unwrap_or(spec)
8904}
8905
8906pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8910 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8911}
8912
8913const TIMED_OUT: &str = "timed out";
8916
8917impl FailClass {
8918 fn of(out: &AgentOutcome) -> Option<Self> {
8920 match out {
8921 AgentOutcome::Ok(_) => None,
8922 AgentOutcome::Quota(_) => Some(Self::Quota),
8923 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8924 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8925 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8926 AgentOutcome::ResourceBlocked(_) => None,
8929 }
8930 }
8931
8932 fn stem_word(&self) -> &'static str {
8934 match self {
8935 Self::Quota => "quota",
8936 _ => "handover",
8937 }
8938 }
8939}
8940
8941fn failure_signature(msg: &str) -> String {
8945 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8946 if let Some(code) = line
8949 .strip_prefix("process exited with no usable output (exit code ")
8950 .and_then(|s| s.strip_suffix(')'))
8951 .and_then(|s| s.parse::<i32>().ok())
8952 {
8953 line = format!("exited with some({code}) and no usable output");
8954 } else if matches!(
8955 line.as_str(),
8956 "process exited with no usable output (no exit code reported)"
8957 | "process exited with no usable output (exit code unavailable)"
8958 ) {
8959 line = "exited with none and no usable output".to_owned();
8960 }
8961 let mut out = Vec::new();
8962 for word in line.split_whitespace() {
8963 if word.contains('/') || word.contains('\\') {
8964 out.push("<path>".to_owned());
8965 continue;
8966 }
8967 let mut w = String::new();
8968 let mut in_digits = false;
8969 for c in word.chars() {
8970 if c.is_ascii_digit() {
8971 if !in_digits {
8972 w.push('#');
8973 }
8974 in_digits = true;
8975 } else {
8976 in_digits = false;
8977 w.push(c);
8978 }
8979 }
8980 out.push(w);
8981 }
8982 out.join(" ").chars().take(120).collect()
8983}
8984
8985fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
8992 match cur {
8993 FailClass::Quota | FailClass::Timeout => true,
8994 FailClass::Other(_) => prev != Some(cur),
8995 }
8996}
8997
8998fn fail_reason(out: &AgentOutcome) -> String {
9000 match out {
9001 AgentOutcome::Ok(_) => String::new(),
9002 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9003 AgentOutcome::Dropped(o) => format!(
9004 "the CLI dropped the stream ({})",
9005 o.dropped
9006 .as_ref()
9007 .map(|d| d.why.as_str())
9008 .unwrap_or("it ended without delivering its answer")
9009 ),
9010 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9011 e.lines().next().unwrap_or("").chars().take(160).collect()
9012 }
9013 }
9014}
9015
9016pub(crate) fn record_handover(
9019 state: &mut RunState,
9020 node: &str,
9021 seat: &str,
9022 from: &str,
9023 to: &str,
9024 class: &FailClass,
9025 reason: &str,
9026) {
9027 let message = if *class == FailClass::Quota {
9028 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9029 } else {
9030 format!("{seat}: handed over {from} -> {to} ({reason})")
9031 };
9032 state.event(node, message);
9033 state.handovers.push(Handover {
9034 at: Timestamp::now(),
9035 node: node.to_owned(),
9036 seat: seat.to_owned(),
9037 from: from.to_owned(),
9038 to: to.to_owned(),
9039 reason: reason.to_owned(),
9040 });
9041}
9042
9043fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9057 commands.iter().any(|c| c.exit_code.is_none())
9058}
9059
9060fn verified_noop_claim(
9073 usable: bool,
9074 commands: &[agent::CommandEvidence],
9075 text: &str,
9076) -> Option<String> {
9077 (usable && !has_unconfirmed_command(commands))
9078 .then(|| verdict::verified_noop(text))
9079 .flatten()
9080}
9081
9082fn short(commit: &str) -> String {
9083 commit.chars().take(7).collect()
9084}
9085
9086fn make_executable(path: &Path) -> Result<()> {
9087 #[cfg(unix)]
9088 {
9089 use std::os::unix::fs::PermissionsExt as _;
9090 let mut perms = std::fs::metadata(path)?.permissions();
9091 perms.set_mode(0o755);
9092 std::fs::set_permissions(path, perms)?;
9093 }
9094 #[cfg(not(unix))]
9095 {
9096 let _ = path;
9097 }
9098 Ok(())
9099}
9100
9101struct WaveCtx<'a> {
9108 run: &'a str,
9111 node: &'a str,
9113 prompts: &'a Prompts,
9114 cache: Option<&'a Path>,
9116 round: Option<usize>,
9119 carry_seats: bool,
9123 pauses: Option<(&'a Pause, &'a Pause)>,
9126}
9127
9128fn repository_wait(
9130 state: &RunState,
9131 pauses: Option<(&Pause, &Pause)>,
9132) -> crate::repo_lock::WaitPolicy {
9133 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9134 let inherited = crate::repo_lock::WaitPolicy::current();
9135 crate::repo_lock::WaitPolicy::new(
9136 Duration::from_secs(state.config.graph.timeout_resource_wait),
9137 move || {
9138 inherited.cancelled()
9139 || pauses
9140 .as_ref()
9141 .is_some_and(|(a, b)| a.parked() || b.parked())
9142 },
9143 )
9144}
9145
9146fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9147 if error.is::<crate::repo_lock::ResourceBlocked>() {
9148 state.resource_block = Some(crate::run::ResourceBlock {
9149 node: "repository".into(),
9150 kind: if error.to_string().contains("disk pressure") {
9151 "disk pressure"
9152 } else {
9153 "repository coordination"
9154 }
9155 .into(),
9156 detail: error.to_string(),
9157 at: Timestamp::now(),
9158 });
9159 state.event("resource", error.to_string());
9160 }
9161}
9162
9163async fn coordinate_operation(
9164 state: &mut RunState,
9165 cwd: &Path,
9166 wait: &crate::repo_lock::WaitPolicy,
9167) -> Result<Option<crate::repo_lock::SeatLease>> {
9168 let result =
9169 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9170 .await;
9171 if let Err(e) = &result {
9172 record_repository_block(state, e);
9173 }
9174 result
9175}
9176
9177async fn run_one(
9179 job: SeatJob,
9180 sem: Arc<Semaphore>,
9181 ctx: &WaveCtx<'_>,
9182 state: &mut RunState,
9183 attempt: usize,
9184) -> (SeatState, AgentOutcome) {
9185 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9186 .await
9187 .pop()
9188 .expect("one job in, one result out");
9189 (seat, out)
9190}
9191
9192async fn wave(
9198 jobs: Vec<SeatJob>,
9199 sem: Arc<Semaphore>,
9200 ctx: &WaveCtx<'_>,
9201 state: &mut RunState,
9202 attempt: usize,
9203) -> Vec<(usize, SeatState, AgentOutcome)> {
9204 let WaveCtx {
9205 run,
9206 node,
9207 prompts,
9208 cache,
9209 round,
9210 carry_seats: _,
9211 pauses,
9212 } = *ctx;
9213 for job in &jobs {
9214 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9215 }
9216 if let Err(e) = state.save() {
9217 tracing::warn!("could not persist in-progress seats: {e:#}");
9222 }
9223 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9239 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9243 let cache_guard = if let Some(cache_dir) = cache {
9244 if jobs_had_a_writer {
9245 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9246 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9247 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9248 {
9249 Ok(Some(g)) => Some(g),
9250 Ok(None) => {
9253 state.resource_block = Some(crate::run::ResourceBlock {
9254 node: node.to_owned(),
9255 kind: "shared build cache".to_owned(),
9256 detail: "the run is parking".to_owned(),
9257 at: Timestamp::now(),
9258 });
9259 return block_wave(
9260 jobs,
9261 state,
9262 node,
9263 attempt,
9264 "parked while waiting for the shared build cache",
9265 );
9266 }
9267 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9270 Err(e) => {
9271 let why = format!("{e:#}");
9275 state.resource_block = Some(crate::run::ResourceBlock {
9276 node: node.to_owned(),
9277 kind: "shared build cache".to_owned(),
9278 detail: why.clone(),
9279 at: Timestamp::now(),
9280 });
9281 state.event(
9282 "resource",
9283 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9284 );
9285 return block_wave(jobs, state, node, attempt, &why);
9286 }
9287 }
9288 } else {
9289 None
9290 }
9291 } else {
9292 None
9293 };
9294 state.seats_spawning(node, attempt);
9297 if let Err(e) = state.save() {
9298 tracing::warn!("could not persist the start of a wave: {e:#}");
9299 }
9300 let slot_limit = resource_limit
9304 + jobs
9305 .iter()
9306 .map(|j| j.timeout)
9307 .max()
9308 .unwrap_or_default()
9309 .saturating_mul(jobs.len() as u32);
9310 let mut set = tokio::task::JoinSet::new();
9311 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9314 let overlay = prompts.overlay(node);
9315 let repository_wait = repository_wait(state, pauses);
9316 for (i, mut job) in jobs.into_iter().enumerate() {
9317 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9318 if cache.is_some() {
9319 job.prompt.push('\n');
9320 job.prompt
9321 .push_str(&prompt::build_cache_note(node, job.allow_write));
9322 }
9323 let sem = Arc::clone(&sem);
9324 let run = run.to_owned();
9325 let node = node.to_owned();
9326 let attachments = if node == "implement" {
9329 state.attachments.clone()
9330 } else {
9331 Vec::new()
9332 };
9333 let cache = cache
9344 .filter(|_| job.allow_write && cache_guard.is_some())
9345 .map(Path::to_path_buf);
9346 let spawned_tx = spawned_tx.clone();
9347 let repository_wait = repository_wait.clone();
9348 let coordination = state.config.coordination.clone();
9349 let fetched_base = state.fetched_base.clone();
9350 set.spawn(async move {
9351 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9355 Ok(p) => p,
9356 Err(_) => {
9357 return (
9358 i,
9359 job.seat,
9360 AgentOutcome::ResourceBlocked(format!(
9361 "no parallel slot became free within {}s",
9362 slot_limit.as_secs()
9363 )),
9364 false,
9365 );
9366 }
9367 };
9368 spawned_tx.send(job.seat.key.clone()).ok();
9369 let mut seat = job.seat;
9370 let out = repository_wait
9371 .scope(agent::invoke(
9372 &job.spec,
9373 &mut seat,
9374 &Invocation {
9375 scope: crate::agent::InvocationScope::Repository,
9376 coordination: coordination.as_ref(),
9377 fetched_base: fetched_base.as_ref(),
9378 cwd: &job.cwd,
9379 prompt: &job.prompt,
9380 timeout: job.timeout,
9381 allow_write: job.allow_write,
9382 unsandboxed: false,
9383 sessions: job.sessions,
9384 artifacts: &job.artifacts,
9385 stem: &job.stem,
9386 run: &run,
9387 node: &node,
9388 cache_dir: cache.as_deref(),
9389 attachments: &attachments,
9390 writable: &[],
9391 },
9392 ))
9393 .await;
9394 let interrupted = out
9395 .as_ref()
9396 .err()
9397 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9398 let out = match out {
9399 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9400 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9401 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9409 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9410 Ok(o) => AgentOutcome::Failed(format!(
9411 "process exited with no usable output ({})",
9412 crate::notices::exit_status(o.exit_code)
9413 )),
9414 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9415 AgentOutcome::ResourceBlocked(e.to_string())
9416 }
9417 Err(e) => AgentOutcome::Failed(e.to_string()),
9418 };
9419 (i, seat, out, interrupted)
9420 });
9421 }
9422 drop(spawned_tx);
9423 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9424 loop {
9425 let joined = tokio::select! {
9426 Some(key) = spawned_rx.recv() => {
9427 state.seat_spawned(&key);
9429 if let Err(e) = state.save() {
9430 tracing::warn!("could not persist a seat's start: {e:#}");
9431 }
9432 continue;
9433 }
9434 joined = set.join_next() => joined,
9435 };
9436 let Some(joined) = joined else {
9437 break;
9438 };
9439 let (i, seat, out, interrupted) = match joined {
9440 Ok(v) => v,
9441 Err(e) => {
9445 tracing::error!("agent task panicked: {e}");
9446 continue;
9447 }
9448 };
9449 state.seat_finished(&seat.key);
9450 if interrupted
9451 && node == "implement"
9452 && let Some(label) = state
9453 .candidates
9454 .iter()
9455 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9456 .map(|candidate| candidate.label)
9457 {
9458 state.candidate_preservation.insert(
9459 label,
9460 "interrupted implementation awaiting persistence".into(),
9461 );
9462 state.interrupted_implementations.insert(label);
9463 }
9464 if let AgentOutcome::ResourceBlocked(why) = &out
9465 && state.resource_block.is_none()
9466 {
9467 state.resource_block = Some(crate::run::ResourceBlock {
9468 node: node.to_owned(),
9469 kind: if why.contains("disk pressure") {
9470 "disk pressure"
9471 } else if why.contains("repository coordination") {
9472 "repository coordination"
9473 } else {
9474 "parallel slot"
9475 }
9476 .to_owned(),
9477 detail: why.clone(),
9478 at: Timestamp::now(),
9479 });
9480 let activity = if interrupted {
9481 "agent execution interrupted"
9482 } else {
9483 "no agent was started"
9484 };
9485 state.event("resource", format!("{node}: {activity} - {why}"));
9486 }
9487 record_jobs(state, node, round, &seat.key, &out);
9488 if let Err(e) = state.save() {
9489 tracing::warn!("could not persist a seat's completion: {e:#}");
9490 }
9491 if collected.len() <= i {
9492 collected.resize_with(i + 1, || None);
9493 }
9494 collected[i] = Some((i, seat, out));
9495 }
9496 if state
9502 .active
9503 .values()
9504 .any(|a| a.node == node && a.attempt == attempt)
9505 {
9506 state
9507 .active
9508 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9509 if let Err(e) = state.save() {
9510 tracing::warn!("could not persist the end of a wave: {e:#}");
9511 }
9512 }
9513 if let Some(cache_dir) = cache
9520 && jobs_had_a_writer
9521 {
9522 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9523 }
9524 if let Some(guard) = cache_guard {
9525 guard.release();
9526 }
9527 collected.into_iter().flatten().collect()
9528}
9529
9530fn record_jobs(
9541 state: &mut RunState,
9542 node: &str,
9543 round: Option<usize>,
9544 seat: &str,
9545 out: &AgentOutcome,
9546) {
9547 let commands: &[agent::CommandEvidence] = match out {
9548 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9549 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9550 };
9551 let checked_at = Timestamp::now();
9552 for c in commands {
9553 state.jobs.push(JobRecord {
9554 node: node.to_owned(),
9555 round,
9556 seat: seat.to_owned(),
9557 id: c.id.clone(),
9558 description: c.description.clone(),
9559 checked_at,
9560 status: match c.exit_code {
9561 Some(0) => JobStatus::Completed,
9562 Some(_) => JobStatus::Failed,
9563 None => JobStatus::Unknown,
9564 },
9565 exit_code: c.exit_code,
9566 result_summary: c.result_summary.clone(),
9567 source: c.source.clone(),
9568 });
9569 }
9570}
9571
9572fn round_is_clean(
9593 blocking: usize,
9594 e2e_ok: bool,
9595 answered: usize,
9596 expected: usize,
9597 quota_missing: usize,
9598 policy: IncompleteReviewPolicy,
9599) -> bool {
9600 if blocking != 0 || !e2e_ok {
9601 return false;
9602 }
9603 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9604 return true;
9605 }
9606 answered > 0 && expected - answered <= quota_missing
9607}
9608
9609pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9633 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9634 return Some(RunStatus::Gating);
9635 }
9636 let last = reviews.last()?;
9637 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9638 if reviews.len() < max_rounds && !stagnant {
9639 return None;
9640 }
9641 if last.incomplete() && last.blocking == 0 {
9642 return Some(RunStatus::Blocked);
9643 }
9644 if last.e2e_status() == E2eStatus::ResourceBlocked {
9645 return None;
9646 }
9647 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9648 RunStatus::Gating
9649 } else {
9650 RunStatus::Blocked
9651 })
9652}
9653
9654pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9669 if nudged {
9670 (full / 4).max(Duration::from_secs(120)).min(full)
9671 } else {
9672 full
9673 }
9674}
9675
9676#[allow(clippy::too_many_arguments)]
9705async fn ask_json_wave<T>(
9706 jobs: Vec<SeatJob>,
9707 sem: Arc<Semaphore>,
9708 retries: usize,
9709 roster: &[AgentSpec],
9710 ctx: &WaveCtx<'_>,
9711 losses: &mut Vec<QuotaLoss>,
9712 state: &mut RunState,
9713 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9714) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9715where
9716 T: serde::de::DeserializeOwned + Send + 'static,
9717{
9718 ask_wave_with(
9719 jobs,
9720 sem,
9721 retries,
9722 roster,
9723 ctx,
9724 losses,
9725 state,
9726 &|text: &str| {
9727 let v = verdict::extract_json::<T>(text)?;
9728 validate(&v)?;
9729 Ok(v)
9730 },
9731 )
9732 .await
9733}
9734
9735#[allow(clippy::too_many_arguments)]
9744async fn ask_wave_with<T>(
9745 jobs: Vec<SeatJob>,
9746 sem: Arc<Semaphore>,
9747 retries: usize,
9748 roster: &[AgentSpec],
9749 ctx: &WaveCtx<'_>,
9750 losses: &mut Vec<QuotaLoss>,
9751 state: &mut RunState,
9752 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9753) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9754where
9755 T: Send + 'static,
9756{
9757 let n = jobs.len();
9758 let originals: Vec<SeatJob> = jobs;
9759 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9760 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9761 let mut nudges: Vec<usize> = vec![0; n];
9768 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9772 let mut tried: Vec<BTreeSet<String>> = specs
9773 .iter()
9774 .map(|s| BTreeSet::from([s.id.clone()]))
9775 .collect();
9776 let starts: Vec<usize> = specs
9777 .iter()
9778 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9779 .collect();
9780 let carry = ctx.carry_seats && !roster.is_empty();
9781 let carried: Vec<BTreeSet<String>> = originals
9784 .iter()
9785 .map(|j| {
9786 state
9787 .seat_history
9788 .get(&j.seat.key)
9789 .filter(|_| carry)
9790 .map(|h| h.failed.clone())
9791 .unwrap_or_default()
9792 })
9793 .collect();
9794 let mut prev: Vec<Option<FailClass>> = originals
9795 .iter()
9796 .map(|j| {
9797 state
9798 .seat_history
9799 .get(&j.seat.key)
9800 .filter(|_| carry)
9801 .and_then(|h| h.last_fail.clone())
9802 })
9803 .collect();
9804 let next_agent =
9805 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9806 let others: BTreeSet<String> = specs
9807 .iter()
9808 .enumerate()
9809 .filter(|(j, _)| *j != i)
9810 .map(|(_, s)| s.id.clone())
9811 .collect();
9812 pick_successor(
9813 roster,
9814 starts[i],
9815 tried,
9816 carry.then(|| &carried[i]),
9817 &others,
9818 )
9819 .cloned()
9820 };
9821 let mut fresh: Vec<Option<String>> = vec![None; n];
9822 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9823 let mut pending: Vec<usize> = (0..n).collect();
9824
9825 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9830 for round in 0..max_rounds {
9831 if pending.is_empty() {
9832 break;
9833 }
9834 let mut batch = Vec::with_capacity(pending.len());
9835 let mut renudged: Vec<&str> = Vec::new();
9836 for &i in &pending {
9837 let src = &originals[i];
9838 let full: &str = match &src.handover {
9843 Some(h) if specs[i].id != src.spec.id => h,
9844 _ => &src.prompt,
9845 };
9846 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9849 (
9850 full.to_owned(),
9851 src.timeout,
9852 format!("{}-{word}-{}", src.stem, specs[i].id),
9853 )
9854 } else if nudges[i] == 0 {
9855 (full.to_owned(), src.timeout, src.stem.clone())
9856 } else {
9857 renudged.push(src.seat.key.as_str());
9858 let why = done[i]
9859 .as_ref()
9860 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9861 .unwrap_or_else(|| "no parsable answer".to_owned());
9862 let nudge = prompt::nudge(&why);
9863 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9864 let prompt = if nudged {
9865 nudge
9866 } else {
9867 format!("{full}\n\n---\n\n{nudge}")
9868 };
9869 (
9870 prompt,
9871 retry_budget(src.timeout, nudged),
9872 format!("{}-retry{}", src.stem, nudges[i]),
9873 )
9874 };
9875 batch.push(SeatJob {
9876 spec: specs[i].clone(),
9877 seat: seats[i].clone(),
9878 cwd: src.cwd.clone(),
9879 prompt,
9880 timeout,
9881 allow_write: src.allow_write,
9882 sessions: src.sessions,
9883 artifacts: src.artifacts.clone(),
9884 stem,
9885 handover: None,
9886 });
9887 }
9888
9889 if !renudged.is_empty() {
9890 state.event(
9891 ctx.node,
9892 format!("retry {round}: re-asking {}", renudged.join(", ")),
9893 );
9894 }
9895 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9896 let mut still = Vec::new();
9897 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9898 seats[i] = seat;
9899 if let AgentOutcome::ResourceBlocked(why) = &out {
9902 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9903 continue;
9904 }
9905 let class = FailClass::of(&out);
9906 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9909 && nudges[i] < retries
9910 && !roster.is_empty();
9911 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9912 let next = should_hand_over(prev[i].as_ref(), &cur)
9913 .then(|| next_agent(i, &tried[i], &specs))
9914 .flatten();
9915 if carry {
9916 let h = state
9917 .seat_history
9918 .entry(originals[i].seat.key.clone())
9919 .or_default();
9920 h.failed.insert(specs[i].id.clone());
9921 h.last_fail = Some(cur.clone());
9922 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9923 h.last_ok = None;
9924 }
9925 if let Err(e) = state.save() {
9928 tracing::warn!("could not persist a seat's failure history: {e:#}");
9929 }
9930 }
9931 if let Some(next) = next {
9932 record_handover(
9933 state,
9934 ctx.node,
9935 &originals[i].seat.key,
9936 &specs[i].id,
9937 &next.id,
9938 &cur,
9939 &fail_reason(&out),
9940 );
9941 tried[i].insert(next.id.clone());
9942 prev[i] = Some(cur.clone());
9943 seats[i] =
9944 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9945 specs[i] = next;
9946 fresh[i] = Some(cur.stem_word().to_owned());
9947 nudges[i] = 0;
9948 done[i] = Some(Err(anyhow::anyhow!(
9949 "handed over after: {}",
9950 fail_reason(&out)
9951 )));
9952 still.push(i);
9953 continue;
9954 }
9955 }
9956 if carry
9957 && !nudge_first
9958 && let Some(cur) = class.clone()
9959 {
9960 let h = state
9962 .seat_history
9963 .entry(originals[i].seat.key.clone())
9964 .or_default();
9965 h.failed.insert(specs[i].id.clone());
9966 h.last_fail = Some(cur);
9967 }
9968 let parsed = match out {
9969 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
9970 AgentOutcome::Quota(o) => {
9971 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
9972 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
9973 }
9974 AgentOutcome::Dropped(o) => {
9978 let why = o
9979 .dropped
9980 .as_ref()
9981 .map(|d| d.why.as_str())
9982 .unwrap_or("the CLI ended the stream without delivering its answer");
9983 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
9984 }
9985 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9986 Err(anyhow::anyhow!(e))
9987 }
9988 };
9989 let quota = class == Some(FailClass::Quota);
9990 let failed = parsed.is_err();
9991 done[i] = Some(parsed);
9992 if carry && !failed {
9993 let h = state
9994 .seat_history
9995 .entry(originals[i].seat.key.clone())
9996 .or_default();
9997 h.failed.remove(&specs[i].id);
9998 h.last_ok = Some(specs[i].id.clone());
9999 h.last_fail = None;
10000 }
10001 let agent_failure = class.is_some()
10009 && !nudge_first
10010 && !roster.is_empty()
10011 && next_agent(i, &tried[i], &specs).is_some();
10012 if failed && !quota && !agent_failure && nudges[i] < retries {
10013 nudges[i] += 1;
10014 still.push(i);
10015 }
10016 }
10017 pending = still;
10018 }
10019
10020 for (i, q) in last_quota.into_iter().enumerate() {
10023 if let Some(reset) = q {
10024 losses.push(QuotaLoss {
10025 seat: originals[i].seat.key.clone(),
10026 node: ctx.node.to_owned(),
10027 at: Timestamp::now(),
10028 reset,
10029 });
10030 }
10031 }
10032
10033 seats
10034 .into_iter()
10035 .zip(done)
10036 .zip(nudges)
10037 .map(|((seat, res), attempts)| {
10038 (
10039 seat,
10040 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10041 attempts,
10042 )
10043 })
10044 .collect()
10045}
10046
10047fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10049 results
10050 .iter()
10051 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10052}
10053
10054fn block_wave(
10058 jobs: Vec<SeatJob>,
10059 state: &mut RunState,
10060 node: &str,
10061 attempt: usize,
10062 why: &str,
10063) -> Vec<(usize, SeatState, AgentOutcome)> {
10064 state.resource_wait = None;
10065 state
10066 .active
10067 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10068 if let Err(e) = state.save() {
10069 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10070 }
10071 jobs.into_iter()
10072 .enumerate()
10073 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10074 .collect()
10075}
10076
10077async fn acquire_cache_lease(
10090 state: &mut RunState,
10091 cache_dir: &Path,
10092 owner: &crate::cache::Owner,
10093 budget: Duration,
10094 context: &str,
10095 cancel: &(dyn Fn() -> bool + Sync),
10096) -> Result<Option<crate::cache::Guard>> {
10097 let home = crate::run::home();
10098 let started = Instant::now();
10099 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10100 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10101 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10102 Err(e) => {
10103 state.event(
10104 "verify",
10105 format!("{context}: could not check the shared build cache: {e:#}"),
10106 );
10107 if let Err(e2) = state.save() {
10108 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10109 }
10110 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10111 }
10112 };
10113 state.event(
10114 "verify",
10115 format!(
10116 "{context}: waiting for the shared build cache at {} ({})",
10117 cache_dir.display(),
10118 busy.describe()
10119 ),
10120 );
10121 state.resource_wait = Some(crate::run::ResourceWait {
10124 node: context.to_owned(),
10125 kind: "shared build cache".to_owned(),
10126 holder: busy.describe(),
10127 since: Timestamp::now(),
10128 limit_secs: budget.as_secs(),
10129 });
10130 if let Err(e) = state.save() {
10131 tracing::warn!("could not persist a cache wait: {e:#}");
10132 }
10133 let remaining = budget.saturating_sub(started.elapsed());
10134 let waited = crate::cache::wait_for_unless(
10135 &home,
10136 cache_dir,
10137 owner,
10138 remaining,
10139 Duration::from_secs(5),
10140 cancel,
10141 )
10142 .await;
10143 state.resource_wait = None;
10144 match waited {
10145 Ok(g) => Ok(g),
10146 Err(e) => {
10147 state.event("verify", format!("{context}: {e:#}"));
10148 if let Err(e2) = state.save() {
10149 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10150 }
10151 Err(e)
10152 }
10153 }
10154}
10155
10156#[derive(Debug)]
10161struct CacheCheckError(String);
10162
10163impl std::fmt::Display for CacheCheckError {
10164 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10165 f.write_str(&self.0)
10166 }
10167}
10168
10169impl std::error::Error for CacheCheckError {}
10170
10171#[allow(clippy::too_many_arguments)]
10192async fn with_cache_lease<'s, F, Fut>(
10193 state: &'s mut RunState,
10194 cache_dir: Option<&Path>,
10195 node: &str,
10196 seat: &str,
10197 worktree: &Path,
10198 head: &str,
10199 budget: Duration,
10200 context: &str,
10201 wait: &crate::repo_lock::WaitPolicy,
10202 body: F,
10203) -> Result<(Vec<CommandOutcome>, bool)>
10204where
10205 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10206 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10207{
10208 let started = Instant::now();
10211 let home = crate::run::home();
10212 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10213 let guard = if let Some(cache_dir) = cache_dir {
10214 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10215 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10216 wait.cancelled()
10217 })
10218 .await
10219 .and_then(|g| {
10220 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10221 }) {
10222 Ok(g) => Some(g),
10223 Err(e) => {
10224 return Ok((
10225 vec![CommandOutcome {
10226 command: "(waiting for the shared build cache)".into(),
10227 code: None,
10228 output_tail: e.to_string(),
10229 duration_ms: started.elapsed().as_millis() as u64,
10230 resource_blocked: true,
10231 }],
10232 false,
10233 ));
10234 }
10235 }
10236 } else {
10237 None
10238 };
10239 let _coordination = coordinate_operation(state, worktree, wait).await?;
10240 if let Some(cache_dir) = cache_dir {
10241 let identity = crate::cache::Identity::new(worktree, head);
10242 if let Err(e) = crate::cache::ensure_fresh(
10243 &home,
10244 cache_dir,
10245 &identity,
10246 &state.config.verify.cargo_manifests(),
10247 ) {
10248 state.event(
10249 "verify",
10250 format!(
10251 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10252 worktree.display(),
10253 short(head)
10254 ),
10255 );
10256 if let Some(guard) = guard {
10257 guard.release();
10258 }
10259 return Ok((
10260 vec![CommandOutcome {
10261 command: "(confirming the shared build cache is fresh)".into(),
10262 code: None,
10263 output_tail: e.to_string(),
10264 duration_ms: started.elapsed().as_millis() as u64,
10265 resource_blocked: true,
10266 }],
10267 false,
10268 ));
10269 }
10270 }
10271 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10272 if !timed_out_pids.is_empty() {
10273 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10274 }
10275 drop(_coordination);
10278 if let Some(guard) = guard {
10279 guard.release();
10280 }
10281 Ok((outcomes, retried))
10282}
10283
10284async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10296 wait_for_pids_with(
10297 pids,
10298 crate::proc::pid_alive,
10299 LEASE_RELEASE_POLL,
10300 LEASE_RELEASE_MAX_WAIT,
10301 )
10302 .await;
10303}
10304
10305async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10311 pids: &[u32],
10312 alive: F,
10313 poll: Duration,
10314 max_wait: Duration,
10315) {
10316 let deadline = Instant::now() + max_wait;
10317 loop {
10318 if pids.iter().all(|&pid| !alive(pid)) {
10319 return;
10320 }
10321 if Instant::now() >= deadline {
10322 return;
10323 }
10324 tokio::time::sleep(poll).await;
10325 }
10326}
10327
10328fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10335 outcomes.iter().any(|o| o.resource_blocked)
10336}
10337
10338pub(crate) enum GateFix {
10340 Retry,
10342 Stop,
10345 Defer,
10348}
10349
10350fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10358 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10359 red.peek().is_some()
10360 && red.all(|o| {
10361 !o.resource_blocked
10362 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10363 && !o.output_tail.trim().is_empty()
10364 })
10365}
10366
10367fn e2e_outcome_label(o: &CommandOutcome) -> String {
10371 if o.ok() {
10372 return "pass".to_owned();
10373 }
10374 let reason = if o.build_failed() {
10375 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10376 } else {
10377 format!("FAIL ({:?})", o.code)
10378 };
10379 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10380}
10381
10382async fn run_e2e_with_retry(
10390 state: &mut RunState,
10391 shell: &[String],
10392 commands: &[String],
10393 worktree: &Path,
10394 timeout: Duration,
10395 context: &str,
10396) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10397 let (mut e2e, mut timed_out_pids) = run_commands(
10398 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10399 )
10400 .await;
10401 for o in &e2e {
10402 state.event(
10403 "verify",
10404 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10405 );
10406 }
10407 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10411 if verify_retried {
10412 state.event(
10413 "verify",
10414 format!(
10415 "{context}: verify could not build/link, not a test result — retrying once \
10416 before concluding"
10417 ),
10418 );
10419 let retried = run_commands(
10420 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10421 )
10422 .await;
10423 e2e = retried.0;
10424 timed_out_pids.extend(retried.1);
10427 for o in &e2e {
10428 state.event(
10429 "verify",
10430 format!(
10431 "{context}: retry `{}` -> {}",
10432 o.command,
10433 e2e_outcome_label(o)
10434 ),
10435 );
10436 }
10437 }
10438 (e2e, verify_retried, timed_out_pids)
10439}
10440
10441#[allow(clippy::too_many_arguments)]
10457async fn run_commands(
10458 state: &mut RunState,
10459 node: &str,
10460 task: &str,
10461 attempt: usize,
10462 shell: &[String],
10463 commands: &[String],
10464 cwd: &Path,
10465 timeout: Duration,
10466) -> (Vec<CommandOutcome>, Vec<u32>) {
10467 if commands.is_empty() {
10468 return (Vec::new(), Vec::new());
10473 }
10474 let mut out = Vec::new();
10475 let mut timed_out_pids = Vec::new();
10476 let total = commands.len();
10477 for (idx, command) in commands.iter().enumerate() {
10478 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10479 if let Err(e) = state.save() {
10480 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10481 }
10482 let started = Instant::now();
10483 let mut cmd = tokio::process::Command::new(&shell[0]);
10484 cmd.quiet();
10485 cmd.args(&shell[1..])
10486 .arg(command)
10487 .current_dir(cwd)
10488 .stdin(std::process::Stdio::null())
10489 .stdout(std::process::Stdio::piped())
10490 .stderr(std::process::Stdio::piped())
10491 .kill_on_drop(true);
10492 let mut outputs = Vec::new();
10493 let protected = (|| -> Result<()> {
10494 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10495 outputs.push(guard);
10496 }
10497 if let Some(cache) = state.config.cache_dir() {
10498 outputs.push(crate::retention::OutputLease::cache(
10499 &crate::run::home(),
10500 &cache,
10501 false,
10502 )?);
10503 }
10504 Ok(())
10505 })();
10506 if let Err(e) = protected {
10507 out.push(CommandOutcome {
10508 command: command.clone(),
10509 code: None,
10510 output_tail: format!("output ownership unavailable: {e:#}"),
10511 duration_ms: 0,
10512 resource_blocked: true,
10513 });
10514 continue;
10515 }
10516 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10517 let (code, body) = match spawned {
10518 Ok(mut running) => {
10519 let pid = running.child().id();
10520 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10521 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10522 let mut paths = vec![cwd.to_owned()];
10523 if let Some(cache) = state.config.cache_dir() {
10524 paths.push(cache);
10525 }
10526 let mut pressure = None;
10527 let status = tokio::select! {
10528 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10529 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10530 pressure = Some(reason); None
10531 }
10532 };
10533 let (code, failure) = match status {
10534 Some(Ok(Ok(status))) => (
10535 status.code(),
10536 status
10537 .code()
10538 .is_none()
10539 .then(|| format!("command terminated: {status}")),
10540 ),
10541 Some(Ok(Err(error))) => {
10542 (None, Some(format!("failed waiting for command: {error}")))
10543 }
10544 Some(Err(_)) => {
10545 if !running.coordinated()
10549 && let Some(pid) = pid
10550 {
10551 timed_out_pids.push(pid);
10552 }
10553 (
10554 None,
10555 Some(format!("timed out after {}s", timeout.as_secs())),
10556 )
10557 }
10558 None => (None, None),
10559 };
10560 running.finish().await;
10561 if let Some(reason) = pressure {
10562 state.resource_block = Some(crate::run::ResourceBlock {
10563 node: node.to_owned(),
10564 kind: "disk pressure".to_owned(),
10565 detail: reason.clone(),
10566 at: Timestamp::now(),
10567 });
10568 out.push(CommandOutcome {
10569 command: command.clone(),
10570 code: None,
10571 output_tail: reason,
10572 duration_ms: started.elapsed().as_millis() as u64,
10573 resource_blocked: true,
10574 });
10575 break;
10576 }
10577 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10578 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10579 if let Some(failure) = failure {
10580 body.push('\n');
10581 body.push_str(&failure);
10582 }
10583 (code, body)
10584 }
10585 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10586 };
10587 out.push(CommandOutcome {
10588 command: command.clone(),
10589 code,
10590 output_tail: tail(&body, OUTPUT_TAIL),
10591 duration_ms: started.elapsed().as_millis() as u64,
10592 resource_blocked: false,
10593 });
10594 }
10595 state.task_finished(task);
10596 if let Err(e) = state.save() {
10597 tracing::warn!("could not persist the end of {task}: {e:#}");
10598 }
10599 (out, timed_out_pids)
10600}
10601
10602fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10614 let repo = repo.display();
10615 match style {
10616 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10617 MergeStyle::Squash => {
10618 let subject = message
10621 .lines()
10622 .next()
10623 .unwrap_or(branch)
10624 .replace(['\\', '"', '$', '`'], "");
10625 format!(
10626 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10627 )
10628 }
10629 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10630 }
10631}
10632
10633const PR_TITLE_MAX: usize = 240;
10644
10645fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10656 let line = queue::first_line(text)?;
10657 let chars: Vec<char> = line.chars().collect();
10658 let (kept, mark) = if chars.len() <= max {
10659 (line.to_owned(), "")
10660 } else if let Some(end) = sentence_end(&chars, max) {
10661 (chars[..end].iter().collect::<String>(), "")
10662 } else {
10663 let room = max.saturating_sub(3);
10664 let cut = if chars[room].is_whitespace() {
10667 room
10668 } else {
10669 chars[..room]
10670 .iter()
10671 .rposition(|c| c.is_whitespace())
10672 .unwrap_or(room)
10673 };
10674 let head: String = chars[..cut].iter().collect();
10675 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10676 (head.to_owned(), "...")
10677 };
10678 if kept.is_empty() {
10679 return None;
10680 }
10681 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10682 return None;
10683 }
10684 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10685 return None;
10686 }
10687 Some(format!("{kept}{mark}"))
10688}
10689
10690fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10694 const MIN: usize = 20;
10695 for i in MIN..max.min(chars.len()) {
10696 if !matches!(chars[i], '.' | '!' | '?') {
10697 continue;
10698 }
10699 let Some(&next) = chars.get(i + 1) else {
10700 continue;
10701 };
10702 if !next.is_whitespace() {
10703 continue;
10704 }
10705 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10706 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10707 continue;
10708 }
10709 let word: String = chars[..i]
10710 .iter()
10711 .rev()
10712 .take_while(|c| !c.is_whitespace())
10713 .collect::<Vec<_>>()
10714 .into_iter()
10715 .rev()
10716 .collect();
10717 let word = word.to_ascii_lowercase();
10718 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10719 continue;
10720 }
10721 let end = chars[..i]
10722 .iter()
10723 .rposition(|c| !c.is_whitespace())
10724 .map_or(i, |p| p + 1);
10725 return Some(end);
10726 }
10727 None
10728}
10729
10730struct PrMessage {
10735 title: String,
10736 body: String,
10737}
10738
10739impl PrMessage {
10740 fn commit_message(&self) -> String {
10744 format!("{}\n\n{}", self.title, self.body)
10745 }
10746}
10747
10748fn title_marker(line: &str) -> Option<&str> {
10750 let line = line.trim();
10751 let head = line.get(..6)?;
10752 head.eq_ignore_ascii_case("title:")
10753 .then(|| line[6..].trim())
10754}
10755
10756fn summary_title(summary: &str) -> Option<String> {
10761 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10762 let raw = title_marker(first)?;
10763 if raw.is_empty() {
10764 return None;
10765 }
10766 let title = queue::title_from(raw, PR_TITLE_MAX);
10767 let lower = title.to_ascii_lowercase();
10768 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10769 return None;
10770 }
10771 Some(title)
10772}
10773
10774const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10776
10777pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10779
10780pub(crate) fn is_review_run(state: &RunState) -> bool {
10784 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10785}
10786
10787fn review_title(state: &RunState) -> Option<String> {
10792 english_subject(
10793 state.reviewed_commits.as_ref()?.first()?,
10794 state.config.graph.github_english(),
10795 )
10796}
10797
10798fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10801 let raw = raw.trim();
10802 if raw.is_empty() {
10803 return None;
10804 }
10805 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10806 return None;
10807 }
10808 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10809 return None;
10810 }
10811 let title = queue::title_from(raw, PR_TITLE_MAX);
10812 let lower = title.to_ascii_lowercase();
10813 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10814 return None;
10815 }
10816 Some(title)
10817}
10818
10819#[derive(Debug, Clone, PartialEq, Eq)]
10822struct BranchFacts {
10823 commits: Vec<(String, String)>,
10825 stat: String,
10827}
10828
10829const STAT_FILE_LINES: usize = 25;
10831const COMMITS_MAX_CHARS: usize = 20_000;
10833
10834async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10837 let commits = git::commit_log(repo, base, branch).await.ok()?;
10838 if commits.is_empty() {
10839 return None;
10840 }
10841 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10842 let lines: Vec<&str> = stat.lines().collect();
10843 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10844 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10845 let more = format!(" ... {omitted} more file(s)");
10846 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10847 kept.push(&more);
10848 kept.push(lines[lines.len() - 1]);
10849 kept.join("\n")
10850 } else {
10851 lines.join("\n")
10852 };
10853 Some(BranchFacts { commits, stat })
10854}
10855
10856fn markdown_safe(text: &str) -> String {
10859 text.replace("</details>", "</details>")
10860 .replace("\x60\x60\x60", "~~~")
10861}
10862
10863fn neutral_title(state: &RunState, winner: char) -> String {
10864 format!(
10865 "chore: land candidate {} of run {}",
10866 winner.to_ascii_uppercase(),
10867 state.id
10868 )
10869}
10870
10871pub(crate) async fn repair_landing_gate(
10874 state: &mut RunState,
10875 winner: &Candidate,
10876 outcomes: &[CommandOutcome],
10877) -> Result<GateFix> {
10878 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10879 return Ok(GateFix::Stop);
10880 }
10881 let mut runner = Runner {
10882 roles: state.config.resolve_roles()?,
10883 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10884 state: state.clone(),
10885 pause: Pause::new(),
10886 interrupt: Pause::new(),
10887 };
10888 let result = runner.gate_fix_round(winner, outcomes).await;
10889 *state = runner.state;
10890 result
10891}
10892
10893pub(crate) async fn check_gate_repair_e2e(
10896 state: &mut RunState,
10897 winner: &Candidate,
10898 wait: &crate::repo_lock::WaitPolicy,
10899) -> Result<GateFix> {
10900 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10901 return Ok(GateFix::Retry);
10902 }
10903 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10904 if state
10905 .gate_repair_e2e
10906 .as_ref()
10907 .is_some_and(|r| r.head == head && r.passed)
10908 {
10909 return Ok(GateFix::Retry);
10910 }
10911 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10912 head: head.clone(),
10913 passed: false,
10914 outcomes: Vec::new(),
10915 });
10916 state.save()?;
10917 let commands = state.config.verify.e2e.clone();
10918 let outcomes = if commands.is_empty() {
10919 Vec::new()
10920 } else {
10921 let shell = state.config.shell();
10922 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10923 let cache_dir = state.config.cache_dir();
10924 let (outcomes, _) = with_cache_lease(
10925 state,
10926 cache_dir.as_deref(),
10927 "e2e",
10928 "e2e",
10929 &winner.worktree,
10930 &head,
10931 timeout,
10932 "gate repair e2e",
10933 wait,
10934 |state, budget| async move {
10935 run_e2e_with_retry(
10936 state,
10937 &shell,
10938 &commands,
10939 &winner.worktree,
10940 budget,
10941 "gate repair e2e",
10942 )
10943 .await
10944 },
10945 )
10946 .await?;
10947 outcomes
10948 };
10949 let deferred = verify_inconclusive(&outcomes);
10950 let passed = outcomes.iter().all(CommandOutcome::ok)
10951 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10952 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10953 head,
10954 passed,
10955 outcomes,
10956 });
10957 state.save()?;
10958 if deferred {
10959 Ok(GateFix::Defer)
10960 } else if !passed {
10961 state.event(
10962 "gate",
10963 "verify.e2e failed after gate repair; nothing published",
10964 );
10965 Ok(GateFix::Stop)
10966 } else {
10967 Ok(GateFix::Retry)
10968 }
10969}
10970
10971pub(crate) async fn run_landing_gate(
10973 state: &mut RunState,
10974 winner: &Candidate,
10975 wait: &crate::repo_lock::WaitPolicy,
10976) -> Result<Vec<CommandOutcome>> {
10977 let shell = state.config.shell();
10978 let gate_commands = state.config.verify.gate.clone();
10979 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
10980 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
10981 Vec::new()
10982 } else {
10983 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10984 let cache_dir = state.config.cache_dir();
10985 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10986 let (outcomes, _) = with_cache_lease(
10987 state,
10988 cache_dir.as_deref(),
10989 "gate",
10990 "gate",
10991 &winner.worktree,
10992 &head,
10993 timeout,
10994 "final gate",
10995 wait,
10996 |state, budget| {
10997 let shell = shell.clone();
10998 let gate_commands = gate_commands.clone();
10999 let winner = winner.clone();
11000 async move {
11001 Runner::run_pre_gate(state, &winner).await;
11002 let (outcomes, timed_out_pids) = run_commands(
11003 state,
11004 "gate",
11005 "gate",
11006 0,
11007 &shell,
11008 &gate_commands,
11009 &winner.worktree,
11010 budget,
11011 )
11012 .await;
11013 (outcomes, false, timed_out_pids)
11014 }
11015 },
11016 )
11017 .await?;
11018 outcomes
11019 };
11020 if outcomes.is_empty() {
11021 state.event("gate", "no gate commands configured; not tested");
11026 }
11027 for o in &outcomes {
11028 state.event(
11029 "gate",
11030 format!(
11031 "`{}` -> {}",
11032 o.command,
11033 if o.ok() {
11034 "pass".to_owned()
11035 } else {
11036 format!(
11037 "FAIL ({:?})\n{}",
11038 o.code,
11039 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11040 )
11041 }
11042 ),
11043 );
11044 }
11045 Ok(outcomes)
11046}
11047
11048pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11054 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11055 ""
11056 } else {
11057 pr_title
11058 }
11059}
11060
11061pub fn landing_subject_source(state: &RunState) -> String {
11065 if is_review_run(state) {
11066 let winner = state.candidates.first().map_or('A', |c| c.label);
11067 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11068 }
11069 state.instruction.clone()
11070}
11071
11072fn summary_without_title(summary: &str) -> String {
11075 let mut lines = summary.trim().lines().peekable();
11076 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11077 lines.next();
11078 }
11079 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11080}
11081
11082#[cfg(test)]
11094fn pr_message(state: &RunState, winner: char) -> PrMessage {
11095 pr_message_with(state, winner, None)
11096}
11097
11098#[cfg(test)]
11101fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11102 let raw = pr_message_raw(
11103 state,
11104 winner,
11105 facts,
11106 crate::pr_outcome::FinalHead::Unreported,
11107 ChangeNote::NotAsked,
11108 );
11109 let id = crate::scrub::Identity::current();
11110 PrMessage {
11111 title: crate::scrub::scrub(&raw.title, &id),
11112 body: crate::scrub::scrub(&raw.body, &id),
11113 }
11114}
11115
11116#[derive(Debug, Clone, Copy)]
11118enum ChangeNote<'a> {
11119 NotAsked,
11121 Summary(&'a crate::run::ChangeSummary),
11123 Unavailable(&'a str),
11125}
11126
11127fn pr_message_raw(
11128 state: &RunState,
11129 winner: char,
11130 facts: Option<&BranchFacts>,
11131 head: crate::pr_outcome::FinalHead<'_>,
11132 change: ChangeNote<'_>,
11133) -> PrMessage {
11134 if !matches!(change, ChangeNote::NotAsked) {
11135 let (title, mut body) = match change {
11136 ChangeNote::Summary(c) => (
11137 c.title.clone(),
11138 format!("## Summary\n\n{}\n\n", c.body.trim()),
11139 ),
11140 ChangeNote::Unavailable(why) => (
11141 neutral_title(state, winner),
11142 format!(
11143 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11144 ),
11145 ),
11146 ChangeNote::NotAsked => unreachable!(),
11147 };
11148 body.push_str(&public_pr_evidence(state, head));
11149 body.push_str(&format!(
11150 "\n---\nmagi:run/{} magi:candidate-{}\n",
11151 state.id,
11152 winner.to_ascii_lowercase()
11153 ));
11154 return PrMessage { title, body };
11155 }
11156 pr_preview(state, winner, facts, head)
11157}
11158
11159fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11162 let mut body = crate::pr_outcome::review_rounds(state);
11163 body.push_str(&crate::pr_outcome::final_verification(state, head));
11164 let open = state.last_round_findings();
11165 if !open.is_empty() {
11166 body.push_str("## Open review findings\n\n");
11167 for f in open {
11168 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11169 }
11170 body.push('\n');
11171 }
11172 if let Some(c) = state.change_summary.as_ref()
11173 && head == crate::pr_outcome::FinalHead::At(&c.head)
11174 {
11175 body.push_str(&format!(
11176 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11177 c.base, c.head
11178 ));
11179 }
11180 body
11181}
11182
11183fn pr_preview(
11185 state: &RunState,
11186 winner: char,
11187 facts: Option<&BranchFacts>,
11188 head: crate::pr_outcome::FinalHead<'_>,
11189) -> PrMessage {
11190 let summary = state
11191 .candidates
11192 .iter()
11193 .find(|c| c.label == winner)
11194 .map(|c| c.summary.as_str())
11195 .unwrap_or_default();
11196 let review = is_review_run(state);
11199 let title = if review {
11200 facts
11201 .and_then(|f| {
11202 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11203 })
11204 .or_else(|| review_title(state))
11205 .or_else(|| {
11206 state
11207 .candidates
11208 .iter()
11209 .find(|c| c.label == winner)
11210 .filter(|c| !c.branch.starts_with("magi/"))
11211 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11212 })
11213 .unwrap_or_else(|| neutral_title(state, winner))
11214 } else {
11215 summary_title(summary).unwrap_or_else(|| {
11216 english_title(
11217 &state.instruction,
11218 PR_TITLE_MAX,
11219 state.config.graph.github_english(),
11220 )
11221 .unwrap_or_else(|| neutral_title(state, winner))
11222 })
11223 };
11224
11225 let mut body = String::new();
11226 let what = summary_without_title(summary);
11227 if !what.is_empty() {
11228 body.push_str("## Summary\n\n");
11229 body.push_str(&what);
11230 body.push_str("\n\n");
11231 }
11232
11233 body.push_str(&crate::pr_outcome::review_rounds(state));
11235 body.push_str(&crate::pr_outcome::final_verification(state, head));
11236 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11237
11238 let open = state.open_findings();
11239 if !open.is_empty() {
11240 body.push_str("## Open review findings\n\n");
11241 for f in &open {
11242 body.push_str(&format!(
11243 "- `{}` [{:?}] {}\n",
11244 f.id, f.severity, "Context retained in local review records."
11245 ));
11246 }
11247 body.push('\n');
11248 }
11249
11250 if let Some(fix) = fix
11251 && !fix.rejected.is_empty()
11252 {
11253 body.push_str("## Declined by the fixer\n\n");
11254 for r in &fix.rejected {
11255 body.push_str(&format!(
11256 "- `{}`: {}\n",
11257 r.id, "Reason retained in local review records."
11258 ));
11259 }
11260 body.push('\n');
11261 }
11262
11263 if review {
11264 body.push_str("## Commits under review\n\n");
11266 if let Some(facts) = facts {
11267 let mut left = COMMITS_MAX_CHARS;
11268 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11269 let mut entry = format!("- {}\n", markdown_safe(subject));
11270 for l in markdown_safe(text).lines() {
11271 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11272 }
11273 if left == 0 {
11274 body.push_str(&format!(
11275 "- ... {} more commit(s)\n",
11276 facts.commits.len() - i
11277 ));
11278 break;
11279 }
11280 if entry.len() > left {
11281 let mut end = left;
11284 while !entry.is_char_boundary(end) {
11285 end -= 1;
11286 }
11287 entry.truncate(end);
11288 entry.push_str("\n ... (truncated)\n");
11289 left = 0;
11290 } else {
11291 left -= entry.len();
11292 }
11293 body.push_str(&entry);
11294 }
11295 if !facts.stat.trim().is_empty() {
11296 body.push_str(&format!(
11297 "\n## Diff stat\n\n```\n{}\n```\n",
11298 markdown_safe(facts.stat.trim())
11299 ));
11300 }
11301 } else {
11302 match &state.reviewed_commits {
11303 Some(subjects) => {
11304 for s in subjects {
11305 body.push_str(&format!("- {}\n", s.trim()));
11306 }
11307 }
11308 None => {
11309 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11312 body.push_str(rest.trim());
11313 body.push('\n');
11314 }
11315 }
11316 }
11317 } else {
11318 let task = state.instruction.trim();
11319 let task = if task.is_empty() {
11320 "(empty task)"
11321 } else {
11322 task
11323 };
11324 body.push_str(&format!(
11325 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11326 task.replace("</details>", "</details>")
11327 ));
11328 }
11329
11330 body.push_str(&format!(
11331 "\n---\nmagi:run/{} magi:candidate-{}\n",
11332 state.id,
11333 winner.to_ascii_lowercase()
11334 ));
11335
11336 PrMessage { title, body }
11337}
11338
11339fn seeded_instruction(state: &RunState) -> String {
11342 let mut inputs = Vec::new();
11343 if let Some(start) = &state.sources.start_from {
11344 inputs.push(format!(
11345 "Start from `{}` at `{}` (including ancestry).",
11346 start.token, start.sha
11347 ));
11348 }
11349 for pick in &state.sources.pick_commits {
11350 inputs.push(format!(
11351 "Pick `{}` at `{}` (single commit).",
11352 pick.token, pick.sha
11353 ));
11354 }
11355 if inputs.is_empty() {
11356 state.instruction.clone()
11357 } else {
11358 format!(
11359 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11360 state.instruction,
11361 inputs.join("\n")
11362 )
11363 }
11364}
11365
11366async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11370 let base = &state.base_branch;
11371 let mut against = base.clone();
11372 if mode == MergeMode::Pr {
11373 let remote = &state.config.merge.remote;
11377 let tracking = format!("{remote}/{base}");
11378 let fetched = git::fetch(repo, remote, base).await;
11379 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11380 against = tracking;
11381 } else {
11382 return false;
11383 }
11384 }
11385 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11386}
11387
11388fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11391 let mut detail = format!(
11392 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11393 nothing to open a pull request for"
11394 );
11395 detail.push_str(
11396 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11397 );
11398 detail
11399}
11400
11401#[derive(Debug, PartialEq, Eq)]
11404enum PrPlan {
11405 Create,
11406 Adopt { url: String, title: String },
11407 Stop(String),
11408}
11409
11410fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11413 match found {
11414 Ok(land::OpenPr::None) => PrPlan::Create,
11415 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11416 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11417 "several open pull requests exist for this branch, not picking one: {}",
11418 urls.join(" ")
11419 )),
11420 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11421 }
11422}
11423
11424pub(crate) async fn gh_pr_create(
11426 cwd: &Path,
11427 base: &str,
11428 head: &str,
11429 title: &str,
11430 body: &str,
11431) -> Result<String> {
11432 let out = tokio::process::Command::new("gh")
11433 .args([
11434 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11435 ])
11436 .current_dir(cwd)
11437 .quiet()
11438 .stdin(std::process::Stdio::null())
11439 .output()
11440 .await
11441 .context("spawn gh")?;
11442 if out.status.success() {
11443 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11444 } else {
11445 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11446 }
11447}
11448
11449async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11451 match std::fs::symlink_metadata(path) {
11452 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11453 git::worktree_remove_clean(repo, path).await?;
11456 return Ok(());
11457 }
11458 Err(e) => return Err(e.into()),
11459 Ok(_) => {}
11460 }
11461 if !git::is_clean(path).await? {
11462 bail!(
11463 "worktree has uncommitted work; refusing removal: {}",
11464 path.display()
11465 );
11466 }
11467 if !git::worktree_remove_clean(repo, path).await? {
11468 bail!("worktree removal refused: {}", path.display());
11469 }
11470 Ok(())
11471}
11472
11473pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11482 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11483 bail!("refusing fold: {reason}");
11484 }
11485 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11486 bail!("candidate preservation incomplete; refusing cleanup");
11487 }
11488 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11492 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11493 let _driver = crate::driver::maintenance(home, &state.id)?;
11494 let mut output_guards = Vec::new();
11495 for entry in std::fs::read_dir(state.worktree_root())
11496 .into_iter()
11497 .flatten()
11498 .flatten()
11499 {
11500 if entry.path().join(".git").is_file() {
11501 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11502 }
11503 }
11504 let repo = state.repo.clone();
11505 let wait = repository_wait(state, None);
11508 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11509 Ok(lease) => lease,
11510 Err(e) => {
11511 state.save_under(home)?;
11512 return Err(e);
11513 }
11514 };
11515 let _source_outputs = if state.status == RunStatus::Merged
11518 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11519 {
11520 verify_merged_fold_sources(state, drop_winner, home).await?
11521 } else {
11522 Vec::new()
11523 };
11524 let root = state.worktree_root();
11525 let winner = state.tally.as_ref().map(|t| t.winner);
11526 let mut removed = Vec::new();
11527
11528 for i in 0..state.candidates.len() {
11529 let c = state.candidates[i].clone();
11530 let is_winner = Some(c.label) == winner;
11531 if is_winner && !drop_winner {
11532 continue;
11533 }
11534 let existed = c.worktree.exists();
11535 remove_preserved_worktree(&repo, &c.worktree).await?;
11538 if existed {
11539 removed.push(c.worktree.to_string_lossy().into_owned());
11540 }
11541 let handed_over = state.released_branches.contains(&c.branch);
11544 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11545 if !git::branch_delete(&repo, &c.branch).await? {
11546 bail!("candidate branch removal refused");
11547 }
11548 removed.push(c.branch.clone());
11549 }
11550 state.candidates[i].folded = true;
11551 }
11552
11553 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11554 let path = name.path();
11555 let keep = !drop_winner
11556 && winner.is_some_and(|w| {
11557 path.file_name()
11558 .is_some_and(|n| n == format!("cand-{w}").as_str())
11559 });
11560 if keep {
11561 continue;
11562 }
11563 remove_preserved_worktree(&repo, &path).await?;
11564 removed.push(path.to_string_lossy().into_owned());
11565 }
11566
11567 remove_if_empty(&root);
11576
11577 if state.enabled_worktree_config && drop_winner {
11578 git::release_worktree_config(&repo).await.ok();
11582 state.enabled_worktree_config = false;
11583 }
11584 state.save_under(home)?;
11585 Ok(removed)
11586}
11587
11588async fn verify_merged_fold_sources(
11589 state: &RunState,
11590 drop_winner: bool,
11591 home: &Path,
11592) -> Result<Vec<std::fs::File>> {
11593 let winner = state.tally.as_ref().map(|t| t.winner);
11594 let mut heads = BTreeSet::new();
11595 for candidate in &state.candidates {
11596 if Some(candidate.label) == winner && !drop_winner {
11597 continue;
11598 }
11599 anyhow::ensure!(
11600 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11601 "candidate recovery source is outside the inventoried run root: {}",
11602 candidate.worktree.display()
11603 );
11604 match std::fs::symlink_metadata(&candidate.worktree) {
11605 Ok(_) => {}
11606 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11607 Err(error) => return Err(error.into()),
11608 }
11609 if !state.released_branches.contains(&candidate.branch)
11610 && git::branch_exists(&state.repo, &candidate.branch).await?
11611 {
11612 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11613 }
11614 }
11615 let mut cargo_guards = Vec::new();
11616 let entries = match std::fs::read_dir(state.worktree_root()) {
11617 Ok(entries) => Some(entries),
11618 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11619 Err(error) => return Err(error.into()),
11620 };
11621 for entry in entries.into_iter().flatten() {
11622 let path = entry?.path();
11623 let metadata = std::fs::symlink_metadata(&path)?;
11624 anyhow::ensure!(
11625 metadata.is_dir() && !metadata.file_type().is_symlink(),
11626 "non-directory or symlink recovery source retained: {}",
11627 path.display()
11628 );
11629 if !drop_winner
11630 && winner.is_some_and(|w| {
11631 path.file_name()
11632 .is_some_and(|name| name == format!("cand-{w}").as_str())
11633 })
11634 {
11635 continue;
11636 }
11637 anyhow::ensure!(
11638 path.join(".git").is_file(),
11639 "unregistered recovery source retained: {}",
11640 path.display()
11641 );
11642 anyhow::ensure!(
11643 git::git(
11644 &path,
11645 &[
11646 "status",
11647 "--porcelain",
11648 "--untracked-files=all",
11649 "--ignored",
11650 "--",
11651 ".",
11652 ":(exclude)target"
11653 ]
11654 )
11655 .await?
11656 .is_empty(),
11657 "source changes or ignored recovery files retained: {}",
11658 path.display()
11659 );
11660 match std::fs::symlink_metadata(path.join("target")) {
11661 Ok(_) => {
11662 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11663 }
11664 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11665 Err(error) => return Err(error.into()),
11666 }
11667 for checkpoint in [
11668 "rebase-merge",
11669 "rebase-apply",
11670 "sequencer",
11671 "CHERRY_PICK_HEAD",
11672 "MERGE_HEAD",
11673 "REBASE_HEAD",
11674 "REVERT_HEAD",
11675 "BISECT_LOG",
11676 "AUTO_MERGE",
11677 "MERGE_MSG",
11678 ] {
11679 let location = git::git(
11680 &path,
11681 &[
11682 "rev-parse",
11683 "--path-format=absolute",
11684 "--git-path",
11685 checkpoint,
11686 ],
11687 )
11688 .await?;
11689 anyhow::ensure!(
11690 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11691 if e.kind() == std::io::ErrorKind::NotFound),
11692 "Git recovery checkpoint retained: {checkpoint}"
11693 );
11694 }
11695 heads.insert(git::rev_parse(&path, "HEAD").await?);
11696 }
11697 crate::fold_evidence::preserve(state, home, &heads).await?;
11698
11699 Ok(cargo_guards)
11700}
11701
11702fn remove_if_empty(dir: &Path) {
11713 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11714 std::fs::remove_dir(dir).ok();
11715 }
11716}
11717
11718pub fn worst_open(state: &RunState) -> Option<Severity> {
11720 state
11721 .reviews
11722 .last()?
11723 .reviews
11724 .iter()
11725 .flat_map(|r| r.findings.iter())
11726 .map(|f| f.severity)
11727 .max()
11728}
11729
11730async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11734 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11735 git::acquire_worktree_config(repo).await?;
11736 state.enabled_worktree_config = true;
11737 }
11738 Ok(())
11739}
11740
11741#[cfg(test)]
11742mod tests {
11743 #[tokio::test]
11744 async fn launcher_base_fetch_requires_remote_success() {
11745 let tmp = tempfile::tempdir().unwrap();
11746 let repo = tmp.path().join("repo");
11747 let remote = tmp.path().join("remote");
11748 std::fs::create_dir(&repo).unwrap();
11749 std::fs::create_dir(&remote).unwrap();
11750 git::git(&remote, &["init", "--bare", "-b", "main"])
11751 .await
11752 .unwrap();
11753 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11754 git::git(
11755 &repo,
11756 &[
11757 "-c",
11758 "user.name=Fixture",
11759 "-c",
11760 "user.email=fixture@example.invalid",
11761 "commit",
11762 "--allow-empty",
11763 "-m",
11764 "fixture",
11765 ],
11766 )
11767 .await
11768 .unwrap();
11769 git::git(
11770 &repo,
11771 &["remote", "add", "origin", remote.to_str().unwrap()],
11772 )
11773 .await
11774 .unwrap();
11775 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11776 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11777 assert_eq!(
11778 resolve_base(&repo, "main", "origin", None)
11779 .await
11780 .unwrap()
11781 .commit,
11782 head
11783 );
11784 #[cfg(unix)]
11785 {
11786 let policy: crate::config::Coordination = toml::from_str(
11787 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11788 ).unwrap();
11789 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11790 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11791 let held = crate::repo_lock::SeatLease::operation_with(
11792 &repo,
11793 Some(&policy),
11794 &crate::repo_lock::WaitPolicy::default(),
11795 )
11796 .await
11797 .unwrap();
11798 assert!(held.is_some());
11799 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11800 tokio::pin!(fetch);
11801 assert!(
11802 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11803 .await
11804 .is_err()
11805 );
11806 assert!(
11807 !repo.join(".git/FETCH_HEAD").exists(),
11808 "no fetch before admission"
11809 );
11810 drop(held);
11811 let fetched = fetch.await.unwrap();
11812 assert_eq!(fetched.commit, head);
11813 assert_eq!(fetched.remote, "origin");
11814 assert_eq!(fetched.branch, "main");
11815 }
11816 git::git(
11818 &repo,
11819 &[
11820 "remote",
11821 "set-url",
11822 "origin",
11823 tmp.path().join("missing").to_str().unwrap(),
11824 ],
11825 )
11826 .await
11827 .unwrap();
11828 let error = resolve_base(&repo, "main", "origin", None)
11829 .await
11830 .unwrap_err()
11831 .to_string();
11832 assert!(error.contains("cannot read origin/main"));
11833 assert!(error.contains("refusing to branch off"));
11834 }
11835
11836 #[test]
11837 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11838 let leaked = vec!["chore(deps): update a crate".to_owned()];
11839 let own = "fix(daemon): apply a chosen action";
11840 assert!(should_retitle("", own, &leaked));
11841 assert!(should_retitle(
11842 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11843 own,
11844 &leaked
11845 ));
11846 assert!(should_retitle(
11847 "chore: land candidate A of run 1",
11848 own,
11849 &leaked
11850 ));
11851 assert!(should_retitle(
11852 "magi: candidate A (uncommitted work)",
11853 own,
11854 &leaked
11855 ));
11856 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11857 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11858 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11859 }
11860
11861 #[test]
11862 fn pr_merge_plan_creates_adopts_or_stops() {
11863 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11864 assert_eq!(
11865 pr_merge_plan(Ok(land::OpenPr::One {
11866 url: "u".into(),
11867 title: "t".into()
11868 })),
11869 PrPlan::Adopt {
11870 url: "u".into(),
11871 title: "t".into()
11872 }
11873 );
11874 let PrPlan::Stop(many) =
11875 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11876 else {
11877 panic!("many must stop");
11878 };
11879 assert!(many.contains('a') && many.contains('b'));
11880 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11881 panic!("a failed lookup must stop");
11882 };
11883 assert!(err.contains("bad token"));
11884 }
11885
11886 use super::*;
11887 use crate::run::GateStatus;
11888 use std::collections::BTreeMap;
11889 use std::time::Duration;
11890
11891 fn conductor() -> AgentSpec {
11892 AgentSpec {
11893 id: "conductor".to_owned(),
11894 kind: crate::config::AgentKind::Command,
11895 model: None,
11896 command: vec!["true".to_owned()],
11897 extra_args: Vec::new(),
11898 env: BTreeMap::new(),
11899 prompt_delivery: None,
11900 }
11901 }
11902
11903 fn spec(id: &str) -> AgentSpec {
11904 AgentSpec {
11905 id: id.to_owned(),
11906 kind: crate::config::AgentKind::Command,
11907 model: None,
11908 command: vec!["true".to_owned()],
11909 extra_args: Vec::new(),
11910 env: BTreeMap::new(),
11911 prompt_delivery: None,
11912 }
11913 }
11914
11915 fn ids(xs: &[&str]) -> BTreeSet<String> {
11916 xs.iter().map(|s| (*s).to_owned()).collect()
11917 }
11918
11919 #[test]
11920 fn pick_successor_skips_an_agent_another_seat_holds() {
11921 let roster = [spec("a"), spec("b"), spec("c")];
11923 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11924 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11925 }
11926
11927 #[test]
11928 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11929 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11931 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11932 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11933 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11934 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11935 }
11936
11937 #[test]
11938 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11939 let roster = [spec("a"), spec("b")];
11940 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11941 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11942 let carried = ids(&["b"]);
11943 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11944 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11945 }
11946
11947 #[test]
11948 fn pick_successor_returns_none_without_any_untried_successor() {
11949 let roster = [spec("a"), spec("b")];
11950 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11951 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11952 let carried = ids(&["a"]);
11953 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11954 }
11955
11956 #[test]
11957 fn pick_successor_rescue_avoids_another_seats_occupant() {
11958 let roster = [spec("a"), spec("b"), spec("c")];
11960 let carried = ids(&["b", "c"]);
11961 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11962 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11963 }
11964
11965 #[test]
11966 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
11967 let roster = [spec("a"), spec("b"), spec("c")];
11968 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
11969 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11970 }
11971
11972 #[test]
11973 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
11974 let roster = [spec("a"), spec("b"), spec("c")];
11975 let failed = ids(&["a", "b", "c"]);
11976 let mut tried = ids(&["b"]);
11978 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11979 assert_eq!(first.id, "a");
11980 tried.insert(first.id.clone());
11981 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
11982 assert_eq!(second.id, "c");
11983 tried.insert(second.id.clone());
11984 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
11985 }
11986
11987 #[test]
11988 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
11989 let roster = [spec("a"), spec("b")];
11990 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
11991 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11992 }
11993
11994 #[test]
11995 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
11996 let roster = [spec("a"), spec("b"), spec("c")];
11997 let h = SeatHistory {
11998 failed: ids(&["a"]),
11999 last_ok: Some("b".to_owned()),
12000 last_fail: None,
12001 };
12002 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12003 let gone = SeatHistory {
12005 last_ok: Some("zzz".to_owned()),
12006 ..h.clone()
12007 };
12008 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12009 let failed = SeatHistory {
12010 failed: ids(&["a", "b"]),
12011 last_ok: Some("b".to_owned()),
12012 last_fail: None,
12013 };
12014 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12015 }
12016
12017 #[test]
12018 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12019 let first = SeatState::new("review-1", "alpha", 7);
12020 let next = handover_seat("review-1", "gamma", 7);
12021 assert_ne!(first.claude_session, next.claude_session);
12022 }
12023
12024 #[test]
12025 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12026 let mut state = state_with_summary("x", "y");
12027 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12028 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12029 assert_ne!(a.claude_session, b.claude_session);
12030 }
12031
12032 #[test]
12033 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12034 let roster = [spec("a"), spec("b")];
12035 let h = SeatHistory {
12036 failed: ids(&["a", "b"]),
12037 ..SeatHistory::default()
12038 };
12039 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12040 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12041 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12042 }
12043
12044 #[test]
12051 fn failure_signature_ignores_numbers_and_paths() {
12052 assert_eq!(
12053 failure_signature("exited with Some(2) and no usable output"),
12054 failure_signature("exited with Some(137) and no usable output")
12055 );
12056 assert_eq!(
12057 failure_signature("process exited with no usable output (exit code 2)"),
12058 failure_signature("process exited with no usable output (exit code 137)")
12059 );
12060 for (old, new) in [
12061 (
12062 "exited with Some(2) and no usable output",
12063 "process exited with no usable output (exit code 2)",
12064 ),
12065 (
12066 "exited with None and no usable output",
12067 "process exited with no usable output (exit code unavailable)",
12068 ),
12069 ] {
12070 let previous = FailClass::Other(failure_signature(old));
12071 let current = FailClass::Other(failure_signature(new));
12072 assert_eq!(previous, current, "resumption retains the failure class");
12073 assert!(!should_hand_over(Some(&previous), ¤t));
12074 }
12075 assert_eq!(
12076 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12077 failure_signature("cannot open /var/x.txt: denied")
12078 );
12079 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12080 }
12081
12082 #[test]
12083 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12084 let other = FailClass::Other("x".into());
12085 assert!(should_hand_over(None, &FailClass::Quota));
12086 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12087 assert!(should_hand_over(
12088 Some(&FailClass::Timeout),
12089 &FailClass::Timeout
12090 ));
12091 assert!(should_hand_over(None, &other));
12092 assert!(!should_hand_over(Some(&other), &other));
12093 assert!(should_hand_over(
12094 Some(&other),
12095 &FailClass::Other("y".into())
12096 ));
12097 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12099 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12100 }
12101
12102 #[test]
12103 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12104 let a = SeatState::new("judge-1", "alpha", 7);
12105 let b = handover_seat("judge-1", "beta", 7);
12106 assert_ne!(a.claude_session, b.claude_session);
12107 assert_eq!(b.turns, 0);
12108 }
12109
12110 #[test]
12111 fn a_timeout_is_classified_apart_from_other_failures() {
12112 assert_eq!(
12113 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12114 Some(FailClass::Timeout)
12115 );
12116 assert!(matches!(
12117 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12118 Some(FailClass::Other(_))
12119 ));
12120 }
12121
12122 #[test]
12123 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12124 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12125 let tried = BTreeSet::from(["beta".to_owned()]);
12126 let next = next_untried_in_roster(&roster, 1, &tried);
12129 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12130 }
12131
12132 #[test]
12133 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12134 let roster = vec![spec("alpha"), spec("beta")];
12135 let tried = BTreeSet::from(["beta".to_owned()]);
12136 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12140 }
12141
12142 #[test]
12143 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12144 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12145 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12146 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12150 }
12151
12152 #[test]
12153 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12154 let roster = vec![spec("a"), spec("a"), spec("b")];
12155 let tried = BTreeSet::from(["a".to_owned()]);
12156 let next = next_untried_in_roster(&roster, 0, &tried);
12157 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12158 }
12159
12160 #[test]
12161 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12162 let roster = vec![spec("a"), spec("b")];
12163 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12164 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12165 }
12166
12167 #[test]
12168 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12169 let dir = tempfile::tempdir().unwrap();
12170 let bay = dir.path().join("ffff");
12171
12172 remove_if_empty(&bay);
12174 assert!(!bay.exists());
12175
12176 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12179 remove_if_empty(&bay);
12180 assert!(bay.exists(), "non-empty directory must survive");
12181
12182 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12184 remove_if_empty(&bay);
12185 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12186 }
12187
12188 #[test]
12197 fn a_full_panel_that_found_nothing_is_clean() {
12198 assert!(round_is_clean(
12199 0,
12200 true,
12201 2,
12202 2,
12203 0,
12204 IncompleteReviewPolicy::Block
12205 ));
12206 }
12207
12208 #[test]
12209 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12210 assert!(!round_is_clean(
12211 0,
12212 true,
12213 1,
12214 2,
12215 0,
12216 IncompleteReviewPolicy::Block
12217 ));
12218 }
12219
12220 #[test]
12221 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12222 assert!(!round_is_clean(
12223 1,
12224 true,
12225 1,
12226 2,
12227 0,
12228 IncompleteReviewPolicy::Warn
12229 ));
12230 }
12231
12232 #[test]
12233 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12234 assert!(round_is_clean(
12235 0,
12236 true,
12237 1,
12238 2,
12239 0,
12240 IncompleteReviewPolicy::Warn
12241 ));
12242 }
12243
12244 #[test]
12245 fn a_full_panel_with_an_open_finding_is_not_clean() {
12246 assert!(!round_is_clean(
12247 1,
12248 true,
12249 2,
12250 2,
12251 0,
12252 IncompleteReviewPolicy::Block
12253 ));
12254 }
12255
12256 #[test]
12257 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12258 assert!(!round_is_clean(
12259 0,
12260 false,
12261 2,
12262 2,
12263 0,
12264 IncompleteReviewPolicy::Block
12265 ));
12266 }
12267
12268 #[test]
12275 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12276 assert!(round_is_clean(
12279 0,
12280 true,
12281 1,
12282 2,
12283 1,
12284 IncompleteReviewPolicy::Block
12285 ));
12286 }
12287
12288 #[test]
12289 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12290 assert!(!round_is_clean(
12293 0,
12294 true,
12295 1,
12296 2,
12297 0,
12298 IncompleteReviewPolicy::Block
12299 ));
12300 }
12301
12302 #[test]
12303 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12304 assert!(!round_is_clean(
12305 1,
12306 true,
12307 1,
12308 2,
12309 1,
12310 IncompleteReviewPolicy::Block
12311 ));
12312 assert!(!round_is_clean(
12313 0,
12314 false,
12315 1,
12316 2,
12317 1,
12318 IncompleteReviewPolicy::Block
12319 ));
12320 }
12321
12322 #[test]
12323 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12324 assert!(!round_is_clean(
12328 0,
12329 true,
12330 0,
12331 2,
12332 2,
12333 IncompleteReviewPolicy::Block
12334 ));
12335 }
12336
12337 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12338 CommandOutcome {
12339 command: "test".to_owned(),
12340 code,
12341 output_tail: String::new(),
12342 duration_ms: 0,
12343 resource_blocked,
12344 }
12345 }
12346
12347 #[test]
12348 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12349 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12350 assert!(
12351 !verify_inconclusive(&[outcome(Some(1), false)]),
12352 "an ordinary failure is still evidence about the patch"
12353 );
12354 assert!(verify_inconclusive(&[outcome(None, true)]));
12355 assert!(
12356 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12357 "one inconclusive outcome taints the whole batch"
12358 );
12359 assert!(!verify_inconclusive(&[]));
12360 }
12361
12362 #[tokio::test]
12363 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12364 let calls = std::sync::atomic::AtomicUsize::new(0);
12368 let started = Instant::now();
12369 wait_for_pids_with(
12370 &[123],
12371 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12372 Duration::from_millis(5),
12373 Duration::from_secs(5),
12374 )
12375 .await;
12376 assert!(
12377 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12378 "must keep checking rather than deciding on the first answer"
12379 );
12380 assert!(
12381 started.elapsed() < Duration::from_secs(1),
12382 "must return the moment it is confirmed dead, not wait out the ceiling"
12383 );
12384 }
12385
12386 #[tokio::test]
12387 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12388 let started = Instant::now();
12389 wait_for_pids_with(
12390 &[123],
12391 |_| true, Duration::from_millis(5),
12393 Duration::from_millis(30),
12394 )
12395 .await;
12396 let elapsed = started.elapsed();
12397 assert!(
12398 elapsed >= Duration::from_millis(30),
12399 "must not give up before its own ceiling: {elapsed:?}"
12400 );
12401 assert!(
12402 elapsed < Duration::from_secs(1),
12403 "must not wait past its own ceiling either: {elapsed:?}"
12404 );
12405 }
12406
12407 #[tokio::test]
12408 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12409 let started = Instant::now();
12410 wait_for_pids_with(
12411 &[],
12412 |_| true,
12413 Duration::from_secs(5),
12414 Duration::from_secs(5),
12415 )
12416 .await;
12417 assert!(
12418 started.elapsed() < Duration::from_millis(200),
12419 "an empty pid list has nothing to confirm"
12420 );
12421 }
12422
12423 fn review_round(
12429 clean: bool,
12430 blocking: usize,
12431 answered: usize,
12432 expected: usize,
12433 progressed: bool,
12434 e2e_ok: bool,
12435 ) -> ReviewRound {
12436 ReviewRound {
12437 round: 1,
12438 head: "h".to_owned(),
12439 verified_head: None,
12440 verified_at: None,
12441 reviews: Vec::new(),
12442 e2e: vec![CommandOutcome {
12443 command: "test".to_owned(),
12444 code: Some(if e2e_ok { 0 } else { 1 }),
12445 output_tail: String::new(),
12446 duration_ms: 0,
12447 resource_blocked: false,
12448 }],
12449 verify_retried: false,
12450 e2e_deferred: false,
12451 e2e_defer_reason: None,
12452 fix: None,
12453 blocking,
12454 answered,
12455 expected,
12456 clean,
12457 progressed,
12458 vote_split: false,
12459 reconsideration: Vec::new(),
12460 verdict: None,
12461 }
12462 }
12463
12464 #[test]
12465 fn review_conclusion_is_none_when_nothing_has_run() {
12466 assert_eq!(review_conclusion(&[], 3), None);
12467 }
12468
12469 #[test]
12470 fn review_conclusion_is_none_while_rounds_remain() {
12471 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12472 assert_eq!(review_conclusion(&rounds, 3), None);
12473 }
12474
12475 #[test]
12476 fn review_conclusion_is_gating_once_a_round_is_clean() {
12477 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12478 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12479 }
12480
12481 #[test]
12482 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12483 let rounds = vec![
12484 review_round(false, 1, 2, 2, true, true),
12485 review_round(false, 1, 2, 2, true, true),
12486 ];
12487 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12488 }
12489
12490 #[test]
12491 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12492 let rounds = vec![
12493 review_round(false, 1, 2, 2, true, true),
12494 review_round(false, 1, 2, 2, true, false),
12495 ];
12496 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12497 }
12498
12499 #[test]
12500 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12501 let mut blocked = review_round(false, 1, 2, 2, true, false);
12508 blocked.e2e[0].resource_blocked = true;
12509 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12510 assert_eq!(review_conclusion(&rounds, 2), None);
12511 }
12512
12513 #[test]
12514 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12515 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12517 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12518 }
12519
12520 #[test]
12521 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12522 let rounds = vec![
12523 review_round(false, 1, 2, 2, false, true),
12524 review_round(false, 1, 2, 2, false, true),
12525 ];
12526 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12527 }
12528
12529 fn secs(n: u64) -> Duration {
12530 Duration::from_secs(n)
12531 }
12532
12533 #[tokio::test]
12536 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12537 ask_test_home();
12538 let tmp = tempfile::tempdir().unwrap();
12539 let repo = tmp.path().join("repo");
12540 std::fs::create_dir_all(&repo).unwrap();
12541 init_repo(&repo);
12542 let g = |args: &[&str]| {
12543 let out = std::process::Command::new("git")
12544 .args(args)
12545 .current_dir(&repo)
12546 .quiet()
12547 .output()
12548 .unwrap();
12549 assert!(
12550 out.status.success(),
12551 "{}",
12552 String::from_utf8_lossy(&out.stderr)
12553 );
12554 };
12555 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12556 g(&["checkout", "-b", "historical"]);
12557 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12558 g(&["add", "."]);
12559 g(&["commit", "-m", "old alternative"]);
12560 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12561 g(&["checkout", "-b", "magi/task-selected", &base]);
12562 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12563 g(&["add", "."]);
12564 g(&["commit", "-m", "measurement infrastructure"]);
12565 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12566 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12567 g(&["add", "."]);
12568 g(&["commit", "-m", "dependent implementation"]);
12569 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12570 g(&["checkout", "main"]);
12571 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12572 g(&["add", "."]);
12573 g(&["commit", "-m", "unrelated base change"]);
12574 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12575 let origin = tmp.path().join("origin.git");
12576 g(&[
12577 "clone",
12578 "--bare",
12579 repo.to_str().unwrap(),
12580 origin.to_str().unwrap(),
12581 ]);
12582 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12583 g(&["fetch", "origin"]);
12584 let selection = crate::pr_recovery::Selection {
12585 url: "https://example.test/project/pull/1".into(),
12586 branch: "magi/task-selected".into(),
12587 head: head.clone(),
12588 source_head: None,
12589 source_run: None,
12590 source_proof: None,
12591 disposition: Some(crate::pr_recovery::Disposition::Review),
12592 instruction: "preserve measured behavior".into(),
12593 approved_at: jiff::Timestamp::now(),
12594 };
12595 let mut template = runner_at(RunStatus::Reviewing);
12596 template.state.repo = repo.clone();
12597 template.state.base_commit = current.clone();
12598 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12599 template.state.origin = Some(crate::run::Origin::queue("selected"));
12600 template.state.pr_head = Some(selection.branch.clone());
12601 template.state.pr_recovery = Some(selection.clone());
12602 let mut runner =
12603 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12604 .await
12605 .unwrap();
12606 let wt = runner.state.winner().unwrap().worktree.clone();
12607 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12608 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12609 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12610 assert!(runner.state.seeds.is_empty());
12611 assert_eq!(runner.state.instruction, "preserve measured behavior");
12612 runner.sync_to_base().await.unwrap();
12613 assert_eq!(
12614 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12615 "prerequisite"
12616 );
12617 assert_eq!(
12618 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12619 "measured implementation"
12620 );
12621 assert_eq!(
12622 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12623 "current main change"
12624 );
12625 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12626 assert_eq!(
12627 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12628 head
12629 );
12630
12631 let mut partial = runner_at(RunStatus::Reviewing);
12633 partial.state.repo = repo.clone();
12634 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12635 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12636 run: Some("exit 1".into()),
12637 ..Default::default()
12638 }];
12639 let selection = runner.state.pr_recovery.clone().unwrap();
12640 partial.state.pr_recovery = Some(selection.clone());
12641 partial.state.pr_head = Some(selection.branch.clone());
12642 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12643 let id = partial.state.id.clone();
12644 let error = match Runner::open_selected(
12645 &repo,
12646 partial.state,
12647 partial.roles,
12648 1,
12649 runner.state.base_commit.clone(),
12650 selection,
12651 )
12652 .await
12653 {
12654 Ok(_) => panic!("setup must fail"),
12655 Err(e) => e,
12656 };
12657 assert_eq!(
12658 error
12659 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12660 .unwrap()
12661 .run,
12662 id
12663 );
12664 let failed = RunState::load(&id).unwrap();
12665 assert_eq!(failed.status, RunStatus::Blocked);
12666 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12667 assert!(Runner::resume(&id).is_err());
12668 runner.state.status = RunStatus::Ready;
12669 runner.state.driver_exited = true;
12670 runner.state.save().unwrap();
12671 let mut retry = runner_at(RunStatus::Reviewing);
12672 retry.state.repo = repo.clone();
12673 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12674 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12675 reselected.approved_at = jiff::Timestamp::now();
12676 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12677 retry.state.pr_head = Some(reselected.branch.clone());
12678 retry.state.pr_recovery = Some(reselected.clone());
12679 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12680 .await
12681 .unwrap();
12682 let restarted = Runner::open_selected(
12683 &repo,
12684 retry.state,
12685 retry.roles,
12686 1,
12687 runner.state.base_commit.clone(),
12688 reselected,
12689 )
12690 .await
12691 .unwrap();
12692 assert_eq!(
12693 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12694 .await
12695 .unwrap(),
12696 head
12697 );
12698 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12699 assert_eq!(
12700 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12701 head
12702 );
12703 }
12704
12705 #[tokio::test]
12706 #[cfg(unix)]
12707 async fn fallback_and_interrupted_restoration_require_coordination() {
12708 use std::os::fd::AsRawFd as _;
12709 ask_test_home();
12710 let tmp = tempfile::tempdir().unwrap();
12711 let mut runner = registered_runner(tmp.path()).await;
12712 runner.state.config.graph.timeout_resource_wait = 1;
12713 let winner = runner.state.candidates[0].clone();
12714 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12715 let leftover = winner.worktree.join("leftover");
12716 std::fs::write(&leftover, "untrusted edits").unwrap();
12717 let next = tmp.path().join("next-fixer");
12718 let mut failed = spec("first");
12719 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12720 let mut successor = spec("second");
12721 successor.command = vec![
12722 "sh".into(),
12723 "-c".into(),
12724 format!("touch '{}'; echo done", next.display()),
12725 ];
12726 let attempts = vec![
12727 (failed.clone(), "first".into()),
12728 (successor.clone(), "second".into()),
12729 ];
12730 let lock_path = runner.state.repo.join(".git/updater.lock");
12731 let external = std::fs::File::create(&lock_path).unwrap();
12732 assert_eq!(
12733 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12734 0
12735 );
12736 let artifacts = tmp.path().join("artifacts");
12737 let build = |spec, seat| SeatJob {
12738 spec,
12739 seat,
12740 cwd: winner.worktree.clone(),
12741 prompt: "fixture".into(),
12742 timeout: Duration::from_secs(10),
12743 allow_write: true,
12744 sessions: false,
12745 artifacts: artifacts.clone(),
12746 stem: "fix".into(),
12747 handover: None,
12748 };
12749 let (_, _, out) = runner
12750 .ask_fixer_among(
12751 attempts.clone(),
12752 "graft",
12753 None,
12754 Some((&winner.worktree, &head)),
12755 build,
12756 )
12757 .await;
12758 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12759 assert!(leftover.exists(), "contention must prevent clean");
12760 assert!(!next.exists());
12761 assert!(runner.state.handovers.is_empty());
12762 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12764 from_run: "earlier".into(),
12765 generation: 1,
12766 mode: crate::recovery::Mode::VerifyRepair,
12767 reasons: Vec::new(),
12768 reviewed_head: None,
12769 rounds: Vec::new(),
12770 open_findings: Vec::new(),
12771 evidence: None,
12772 criteria_inherited: true,
12773 repair: crate::recovery::Repair::Started {
12774 before: head.clone(),
12775 },
12776 });
12777 assert!(
12778 runner
12779 .recovery_repair(&winner)
12780 .await
12781 .unwrap_err()
12782 .is::<crate::repo_lock::ResourceBlocked>()
12783 );
12784 assert!(matches!(
12785 runner.state.recovery.as_ref().unwrap().repair,
12786 crate::recovery::Repair::Started { .. }
12787 ));
12788 assert!(leftover.exists());
12789 drop(external);
12790 runner.recovery_repair(&winner).await.unwrap();
12791 assert!(!leftover.exists());
12792 assert!(matches!(
12793 runner.state.recovery.as_ref().unwrap().repair,
12794 crate::recovery::Repair::Done { .. }
12795 ));
12796 std::fs::remove_file(&lock_path).unwrap();
12798 std::fs::create_dir(&lock_path).unwrap();
12799 std::fs::write(&leftover, "untrusted edits").unwrap();
12800 let (_, _, out) = runner
12801 .ask_fixer_among(
12802 attempts,
12803 "graft",
12804 None,
12805 Some((&winner.worktree, &head)),
12806 build,
12807 )
12808 .await;
12809 assert!(matches!(out, AgentOutcome::Failed(_)));
12810 assert!(leftover.exists());
12811 assert!(!next.exists());
12812 assert_eq!(
12813 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12814 head
12815 );
12816 }
12817
12818 #[cfg(unix)]
12819 async fn registered_runner(root: &Path) -> Runner {
12820 ask_test_home();
12821 let repo = root.join("repo");
12822 std::fs::create_dir_all(&repo).unwrap();
12823 init_repo(&repo);
12824 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12825 let mut runner = runner_at(RunStatus::Prep);
12826 runner.state.repo = repo.clone();
12827 runner.state.config.coordination = crate::config::test_coordination();
12828 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12829 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12830 runner.state.config.graph.advise = false;
12831 runner.roles.implementers = vec![spec("implement")];
12832 runner.roles.judges = vec![spec("judge")];
12833 runner.prep().await.unwrap();
12834 runner
12835 }
12836
12837 #[tokio::test]
12838 #[cfg(unix)]
12839 async fn parked_implementer_persists_completed_work() {
12840 for registered in [false, true] {
12841 let tmp = tempfile::tempdir().unwrap();
12842 let mut runner = registered_runner(tmp.path()).await;
12843 if !registered {
12844 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12845 }
12846 let marker = tmp.path().join("started");
12847 let release = tmp.path().join("release");
12848 let mut implementer = spec("implement");
12849 implementer.command = vec![
12850 "sh".into(),
12851 "-c".into(),
12852 format!(
12853 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12854 marker.display(),
12855 release.display()
12856 ),
12857 ];
12858 runner.roles.implementers = vec![implementer.clone()];
12859 runner.state.config.agents = vec![implementer];
12860 let pause = runner.pause.clone();
12861 let (result, ()) = tokio::join!(runner.implement(), async {
12862 tokio::time::timeout(Duration::from_secs(10), async {
12863 while !marker.exists() {
12864 tokio::time::sleep(Duration::from_millis(10)).await;
12865 }
12866 })
12867 .await
12868 .unwrap();
12869 pause.park();
12870 std::fs::write(&release, "go").unwrap();
12871 });
12872 result.unwrap();
12873 let candidate = &runner.state.candidates[0];
12874 assert!(candidate.commits > 0, "{candidate:?}");
12875 assert!(candidate.summary.contains("saved-result"));
12876 assert_eq!(candidate.files, 1);
12877 let before = candidate.commits;
12878 runner.implement().await.unwrap();
12879 assert_eq!(runner.state.candidates[0].commits, before);
12880 }
12881 }
12882
12883 #[tokio::test]
12884 #[cfg(unix)]
12885 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12886 let tmp = tempfile::tempdir().unwrap();
12887 let mut runner = registered_runner(tmp.path()).await;
12888 let winner = runner.state.candidates[0].clone();
12889 let cache = tmp.path().join("cache");
12890 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12891 runner.state.config.graph.timeout_resource_wait = 0;
12892 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12893 let crate::cache::AcquireOutcome::Acquired(held) =
12894 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12895 else {
12896 panic!("cache must be free")
12897 };
12898 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12899 from_run: "earlier".into(),
12900 generation: 1,
12901 mode: crate::recovery::Mode::VerifyRepair,
12902 reasons: Vec::new(),
12903 reviewed_head: None,
12904 rounds: Vec::new(),
12905 open_findings: Vec::new(),
12906 evidence: None,
12907 criteria_inherited: true,
12908 repair: crate::recovery::Repair::NotStarted,
12909 });
12910 runner.recovery_repair(&winner).await.unwrap();
12911 assert!(matches!(
12912 runner.state.recovery.as_ref().unwrap().repair,
12913 crate::recovery::Repair::WaitingForRestore { .. }
12914 ));
12915 drop(held);
12916 runner.state.resource_block = None;
12917 runner.state.config.graph.timeout_resource_wait = 1;
12918 runner.recovery_repair(&winner).await.unwrap();
12919 assert!(matches!(
12920 runner.state.recovery.as_ref().unwrap().repair,
12921 crate::recovery::Repair::Done { .. }
12922 ));
12923 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12924 }
12925
12926 #[tokio::test]
12927 #[cfg(unix)]
12928 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12929 use std::os::fd::AsRawFd as _;
12930 for graft in [false, true] {
12931 let tmp = tempfile::tempdir().unwrap();
12932 let mut runner = registered_runner(tmp.path()).await;
12933 runner.state.config.graph.timeout_resource_wait = 1;
12934 let winner = runner.state.candidates[0].clone();
12935 let marker = tmp.path().join("fixer-started");
12936 let mut fixer = spec("fixer");
12937 fixer.command = vec![
12938 "sh".into(),
12939 "-c".into(),
12940 format!("echo started >> '{}'; echo done", marker.display()),
12941 ];
12942 runner.roles.fixer = Some(vec![fixer]);
12943 if graft {
12944 let mut loser = winner.clone();
12945 loser.label = 'B';
12946 loser.index = 1;
12947 runner.state.candidates.push(loser);
12948 runner.state.config.graph.graft_runner_up = true;
12949 runner.state.tally = Some(Tally {
12950 first_choice: BTreeMap::new(),
12951 borda: BTreeMap::new(),
12952 winner: winner.label,
12953 rankings: 1,
12954 unanimous_initial: true,
12955 deliberated: false,
12956 changed_votes: 0,
12957 unanimous_final: true,
12958 tie_break: None,
12959 judges: 1,
12960 present: 1,
12961 quorum: 1,
12962 met_quorum: true,
12963 uncontested: None,
12964 });
12965 runner.state.judgements.push(crate::run::Judgement {
12966 judge: 1,
12967 seat: "judge-1".into(),
12968 agent: "judge".into(),
12969 ranking: vec![winner.label, 'B'],
12970 reasons: BTreeMap::new(),
12971 confidence: None,
12972 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
12973 order: vec![0, 1],
12974 failed: None,
12975 duration_ms: 0,
12976 });
12977 } else {
12978 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12979 from_run: "earlier".into(),
12980 generation: 1,
12981 mode: crate::recovery::Mode::VerifyRepair,
12982 reasons: Vec::new(),
12983 reviewed_head: None,
12984 rounds: Vec::new(),
12985 open_findings: Vec::new(),
12986 evidence: None,
12987 criteria_inherited: true,
12988 repair: crate::recovery::Repair::NotStarted,
12989 });
12990 }
12991 let reader = crate::repo_lock::SeatLease::for_checkout(
12992 &winner.worktree,
12993 false,
12994 crate::config::test_coordination().as_ref(),
12995 )
12996 .await
12997 .unwrap();
12998 let pause = runner.pause.clone();
12999 let (result, ()) = tokio::join!(
13000 async {
13001 if graft {
13002 runner.graft().await
13003 } else {
13004 runner.recovery_repair(&winner).await
13005 }
13006 },
13007 async {
13008 tokio::time::sleep(Duration::from_millis(100)).await;
13009 pause.park();
13010 }
13011 );
13012 result.unwrap();
13013 assert!(!marker.exists(), "a blocked seat must not spawn");
13014 runner.state = RunState::load(&runner.state.id).unwrap();
13016 if graft {
13017 assert_eq!(
13018 runner.state.graft.as_ref().unwrap().outcome,
13019 GraftOutcome::WaitingForRestore
13020 );
13021 } else {
13022 assert!(matches!(
13023 runner.state.recovery.as_ref().unwrap().repair,
13024 crate::recovery::Repair::WaitingForRestore { .. }
13025 ));
13026 }
13027 drop(reader);
13028 runner.pause = Pause::new();
13029 runner.state.resource_block = None;
13030 let external =
13032 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13033 assert_eq!(
13034 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13035 0
13036 );
13037 let leftover = winner.worktree.join("leftover");
13038 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13039 let result = if graft {
13040 runner.graft().await
13041 } else {
13042 runner.recovery_repair(&winner).await
13043 };
13044 assert!(
13045 result
13046 .unwrap_err()
13047 .is::<crate::repo_lock::ResourceBlocked>()
13048 );
13049 assert!(leftover.exists());
13050 assert!(!marker.exists());
13051 drop(external);
13052 runner.state.resource_block = None;
13053 let result = if graft {
13054 runner.graft().await
13055 } else {
13056 runner.recovery_repair(&winner).await
13057 };
13058 result.unwrap();
13059 assert!(!leftover.exists(), "restore must precede retry");
13060 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13061 if graft {
13062 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13063 } else {
13064 assert!(matches!(
13065 runner.state.recovery.as_ref().unwrap().repair,
13066 crate::recovery::Repair::Done { .. }
13067 ));
13068 }
13069 }
13070 }
13071
13072 #[tokio::test]
13073 #[cfg(unix)]
13074 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13075 use std::os::fd::AsRawFd as _;
13076 let tmp = tempfile::tempdir().unwrap();
13077 let mut runner = registered_runner(tmp.path()).await;
13078 runner.state.config.graph.timeout_resource_wait = 1;
13079 let winner = runner.state.candidates[0].clone();
13080 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13081 runner.state.tally = Some(Tally {
13082 first_choice: BTreeMap::new(),
13083 borda: BTreeMap::new(),
13084 winner: winner.label,
13085 rankings: 0,
13086 unanimous_initial: false,
13087 deliberated: false,
13088 changed_votes: 0,
13089 unanimous_final: false,
13090 tie_break: None,
13091 judges: 0,
13092 present: 0,
13093 quorum: 0,
13094 met_quorum: true,
13095 uncontested: Some("fixture".into()),
13096 });
13097 runner.state.graft = Some(GraftRecord {
13098 pre: pre.clone(),
13099 post: None,
13100 lines: None,
13101 limit: None,
13102 agent: None,
13103 outcome: GraftOutcome::Running,
13104 requested: Vec::new(),
13105 adoption: None,
13106 adoption_error: None,
13107 });
13108 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13109 git::commit_all(&winner.worktree, "unfinished graft")
13110 .await
13111 .unwrap();
13112 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13113 let leftover = winner.worktree.join("leftover");
13114 std::fs::write(&leftover, "untrusted edits").unwrap();
13115 let path = runner.state.repo.join(".git/updater.lock");
13116 let external = std::fs::File::create(&path).unwrap();
13117 assert_eq!(
13118 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13119 0
13120 );
13121 assert!(
13122 runner
13123 .graft()
13124 .await
13125 .unwrap_err()
13126 .is::<crate::repo_lock::ResourceBlocked>()
13127 );
13128 assert!(matches!(
13129 runner.state.graft.as_ref().unwrap().outcome,
13130 GraftOutcome::Running
13131 ));
13132 assert_eq!(
13133 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13134 unfinished
13135 );
13136 assert!(leftover.exists());
13137 drop(external);
13138 std::fs::remove_file(&path).unwrap();
13139 std::fs::create_dir(&path).unwrap();
13140 assert!(runner.graft().await.is_err());
13141 assert!(matches!(
13142 runner.state.graft.as_ref().unwrap().outcome,
13143 GraftOutcome::Running
13144 ));
13145 assert_eq!(
13146 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13147 unfinished
13148 );
13149 assert!(leftover.exists());
13150 std::fs::remove_dir(&path).unwrap();
13151 runner.graft().await.unwrap();
13152 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13153 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13154 assert!(!leftover.exists());
13155 }
13156
13157 #[tokio::test]
13158 #[cfg(unix)]
13159 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13160 use std::os::fd::AsRawFd as _;
13161 let tmp = tempfile::tempdir().unwrap();
13162 let mut runner = registered_runner(tmp.path()).await;
13163 runner.state.prep_complete = Some(false);
13164 runner.state.config.graph.timeout_resource_wait = 1;
13165 runner.state.save().unwrap();
13166 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13167 assert_eq!(
13168 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13169 0
13170 );
13171 runner.execute().await.unwrap();
13172 let disk = RunState::load(&runner.state.id).unwrap();
13173 assert!(disk.driver_exited);
13174 assert_eq!(disk.driver_pid, None);
13175 assert_eq!(runner.state.status, RunStatus::Blocked);
13176 assert_eq!(
13177 runner.state.resource_block.as_ref().unwrap().kind,
13178 "repository coordination"
13179 );
13180 assert!(runner.state.prep_failure.is_none());
13181 assert!(runner.state.seats.is_empty());
13182 runner.pause.park();
13183 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13186 .await
13187 .unwrap()
13188 .unwrap();
13189 assert!(runner.state.parked);
13190 assert!(runner.state.prep_failure.is_none());
13191 let disk = RunState::load(&runner.state.id).unwrap();
13192 assert!(disk.driver_exited);
13193 assert_eq!(disk.driver_pid, None);
13194 }
13195
13196 #[tokio::test]
13197 #[cfg(unix)]
13198 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13199 use std::os::fd::AsRawFd as _;
13200 let tmp = tempfile::tempdir().unwrap();
13201 let mut runner = registered_runner(tmp.path()).await;
13202 runner.state.prep_complete = Some(false);
13203 runner.state.config.graph.timeout_resource_wait = 30;
13204 runner.state.save().unwrap();
13205 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13206 assert_eq!(
13207 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13208 0
13209 );
13210 assert!(
13211 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13212 .await
13213 .is_err()
13214 );
13215 let disk = RunState::load(&runner.state.id).unwrap();
13216 assert!(disk.driver_exited);
13217 assert_eq!(disk.driver_pid, None);
13218 assert!(disk.status.resumable());
13219 assert!(runner.state.driver_exited);
13220 assert_eq!(runner.state.driver_pid, None);
13221 runner.state.save().unwrap();
13222 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13223 }
13224
13225 #[tokio::test]
13226 #[cfg(unix)]
13227 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13228 let tmp = tempfile::tempdir().unwrap();
13229 let mut runner = registered_runner(tmp.path()).await;
13230 let winner = runner.state.candidates[0].clone();
13231 let cache = tmp.path().join("cache");
13232 let started = tmp.path().join("pre-gate-started");
13233 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13234 runner.state.config.verify.gate =
13235 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13236 let home = crate::run::home();
13237 let owner =
13238 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13239 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13240 crate::cache::AcquireOutcome::Acquired(g) => g,
13241 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13242 };
13243 let mut gate = Box::pin(runner.run_gate(&winner));
13244 assert!(
13245 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13246 .await
13247 .is_err()
13248 );
13249 assert!(!started.exists(), "pre-gate must wait for the cache too");
13250 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13254 let editor = crate::repo_lock::SeatLease::checkout_with(
13255 &winner.worktree,
13256 true,
13257 crate::config::test_coordination().as_ref(),
13258 &wait,
13259 )
13260 .await
13261 .unwrap();
13262 drop(editor);
13263 guard.release();
13264 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13265 .await
13266 .unwrap()
13267 .unwrap();
13268 drop(gate);
13269 assert!(outcomes.iter().all(CommandOutcome::ok));
13270 assert!(started.exists());
13271 }
13272
13273 #[tokio::test]
13274 #[cfg(unix)]
13275 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13276 use std::os::fd::AsRawFd as _;
13277
13278 ask_test_home();
13279 let tmp = tempfile::tempdir().unwrap();
13280 let repo = tmp.path().join("repo");
13281 std::fs::create_dir_all(&repo).unwrap();
13282 init_repo(&repo);
13283 let common = repo.join(".git");
13284 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13285 let mut runner = runner_at(RunStatus::Prep);
13286 runner.state.repo = repo.clone();
13287 runner.state.config.coordination = crate::config::test_coordination();
13288 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13289 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13290 runner.state.config.graph.advise = false;
13291 runner.roles.implementers = vec![spec("implement")];
13292 runner.roles.judges = vec![spec("judge")];
13293 runner.prep().await.unwrap();
13294 let winner = runner.state.candidates[0].clone();
13295 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13298 let completed = tmp.path().join("seat-completed");
13299 runner.roles.implementers[0].command = vec![
13300 "sh".into(),
13301 "-c".into(),
13302 format!("touch '{}'; echo completed", completed.display()),
13303 ];
13304 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13305 assert_eq!(
13306 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13307 0
13308 );
13309 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13310 let mut implement = Box::pin(runner.implement());
13311 tokio::select! {
13312 result = &mut implement => panic!("rescue must wait: {result:?}"),
13313 result = tokio::time::timeout(Duration::from_secs(10), async {
13314 while !completed.exists() {
13315 tokio::time::sleep(Duration::from_millis(20)).await;
13316 }
13317 }) => result.unwrap(),
13318 }
13319 assert!(
13320 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13321 .await
13322 .is_err()
13323 );
13324 assert_eq!(
13325 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13326 before
13327 );
13328 assert_eq!(
13329 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13330 0
13331 );
13332 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13333 .await
13334 .unwrap()
13335 .unwrap();
13336 drop(implement);
13337 assert_ne!(
13338 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13339 before
13340 );
13341 assert!(git::is_clean(&winner.worktree).await.unwrap());
13342
13343 let started = tmp.path().join("pre-gate-started");
13344 let proceed = tmp.path().join("pre-gate-proceed");
13345 let checked = tmp.path().join("gate-ran");
13346 runner.state.config.verify.pre_gate = vec![format!(
13347 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13348 started.display(),
13349 proceed.display()
13350 )];
13351 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13352 assert_eq!(
13353 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13354 0
13355 );
13356 let mut gate = Box::pin(runner.run_gate(&winner));
13357 assert!(
13358 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13359 .await
13360 .is_err()
13361 );
13362 assert!(!started.exists());
13363 assert!(!checked.exists());
13364 assert!(git::is_clean(&winner.worktree).await.unwrap());
13365 assert_eq!(
13366 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13367 0
13368 );
13369 tokio::select! {
13370 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13371 result = tokio::time::timeout(Duration::from_secs(10), async {
13372 while !started.exists() {
13373 tokio::time::sleep(Duration::from_millis(20)).await;
13374 }
13375 }) => result.unwrap(),
13376 }
13377 assert_eq!(
13378 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13379 -1,
13380 "mechanical edits must hold the real lock"
13381 );
13382 std::fs::write(proceed, "ready").unwrap();
13383 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13384 .await
13385 .unwrap()
13386 .unwrap();
13387 drop(gate);
13388 assert!(outcomes.iter().all(CommandOutcome::ok));
13389 assert!(checked.exists());
13390 assert!(runner.state.pre_gate_commit.is_some());
13391 assert!(git::is_clean(&winner.worktree).await.unwrap());
13392 assert_eq!(
13393 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13394 0,
13395 "gate completion must release coordination"
13396 );
13397 drop(outsider);
13398 std::fs::remove_file(common.join("updater.lock")).unwrap();
13399 std::fs::create_dir(common.join("updater.lock")).unwrap();
13400 std::fs::remove_file(&started).unwrap();
13401 std::fs::remove_file(&checked).unwrap();
13402 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13403 let error = runner.run_gate(&winner).await.unwrap_err();
13404 assert!(
13405 error
13406 .to_string()
13407 .contains("acquire mandatory repository lock")
13408 );
13409 assert!(
13410 !started.exists(),
13411 "failed acquisition must prevent pre-gate"
13412 );
13413 assert!(!checked.exists(), "failed acquisition must prevent gate");
13414 assert!(git::is_clean(&winner.worktree).await.unwrap());
13415 assert_eq!(
13416 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13417 head
13418 );
13419 }
13420
13421 #[tokio::test]
13422 #[cfg(unix)]
13423 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13424 use std::os::fd::AsRawFd as _;
13425 use std::time::Duration;
13426
13427 ask_test_home();
13428 let tmp = tempfile::tempdir().unwrap();
13429 let repo = tmp.path().join("repo");
13430 std::fs::create_dir_all(&repo).unwrap();
13431 init_repo(&repo);
13432 let common = repo.join(".git");
13433 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13434 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13435 assert_eq!(
13436 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13437 0
13438 );
13439 let mut runner = runner_at(RunStatus::Prep);
13440 runner.state.repo = repo.clone();
13441 runner.state.config.coordination = crate::config::test_coordination();
13442 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13443 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13444 runner.state.config.graph.advise = true;
13445 runner.state.config.graph.advisors = 1;
13446 runner.roles.implementers = vec![spec("implement")];
13447 runner.roles.judges = vec![spec("judge")];
13448 let root = runner.state.worktree_root();
13449 let started = tmp.path().join("setup-started");
13450 let proceed = tmp.path().join("setup-proceed");
13451 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13452 run: Some(format!(
13453 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13454 started.display(),
13455 proceed.display()
13456 )),
13457 ..Default::default()
13458 }];
13459 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13460 let config = git::git(&repo, &["config", "--local", "--list"])
13461 .await
13462 .unwrap();
13463 let mut prep = Box::pin(runner.prep());
13464 assert!(
13465 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13466 .await
13467 .is_err()
13468 );
13469 assert!(
13470 !root.exists(),
13471 "contention must prevent even checkout creation"
13472 );
13473 assert!(!started.exists(), "contention must prevent setup");
13474 assert_eq!(
13475 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13476 branches
13477 );
13478 assert_eq!(
13479 git::git(&repo, &["config", "--local", "--list"])
13480 .await
13481 .unwrap(),
13482 config
13483 );
13484 drop(lock);
13485 tokio::select! {
13486 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13487 result = tokio::time::timeout(Duration::from_secs(10), async {
13488 while !started.exists() {
13489 tokio::time::sleep(Duration::from_millis(20)).await;
13490 }
13491 }) => result.unwrap(),
13492 }
13493 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13494 assert_eq!(
13495 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13496 -1
13497 );
13498 assert!(
13499 tokio::time::timeout(
13500 Duration::from_millis(100),
13501 crate::repo_lock::SeatLease::for_checkout(
13502 &repo,
13503 true,
13504 crate::config::test_coordination().as_ref()
13505 )
13506 )
13507 .await
13508 .is_err(),
13509 "preparation must also exclude local editors"
13510 );
13511 std::fs::write(proceed, "ready").unwrap();
13512 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13513 .await
13514 .unwrap()
13515 .unwrap();
13516 drop(prep);
13517 assert_eq!(runner.state.prep_complete, Some(true));
13518 assert!(root.join("judge-1").exists());
13519 assert!(root.join("advisor-1").exists());
13520 assert_eq!(
13521 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13522 0
13523 );
13524 }
13525
13526 #[tokio::test]
13527 #[cfg(unix)]
13528 async fn preparation_lock_failure_prevents_repository_changes() {
13529 ask_test_home();
13530 let tmp = tempfile::tempdir().unwrap();
13531 let repo = tmp.path().join("repo");
13532 std::fs::create_dir_all(&repo).unwrap();
13533 init_repo(&repo);
13534 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13535 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13538 let mut runner = runner_at(RunStatus::Prep);
13539 runner.state.repo = repo.clone();
13540 runner.state.config.coordination = crate::config::test_coordination();
13541 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13542 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13543 runner.roles.implementers = vec![spec("implement")];
13544 let error = runner.prep().await.unwrap_err();
13545 assert!(
13546 error
13547 .to_string()
13548 .contains("acquire mandatory repository lock")
13549 );
13550 assert!(!runner.state.worktree_root().exists());
13551 assert!(runner.state.candidates.is_empty());
13552 assert!(!repo.join(".git/config.worktree").exists());
13553 assert_eq!(
13554 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13555 "* main"
13556 );
13557 }
13558
13559 #[tokio::test]
13560 async fn preparation_records_creation_intent_before_git_can_fail() {
13561 ask_test_home();
13562 let tmp = tempfile::tempdir().unwrap();
13563 let repo = tmp.path().join("repo");
13564 std::fs::create_dir_all(&repo).unwrap();
13565 init_repo(&repo);
13566 let mut runner = runner_at(RunStatus::Prep);
13567 runner.state.repo = repo.clone();
13568 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13569 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13570 runner.state.config.blind.commit_msg_hook = false;
13571 runner.roles.implementers = vec![spec("one")];
13572 let label = blind::assign_labels(1, runner.state.seed)[0];
13573 let branch = runner.state.branch_for(label);
13574 git::git(&repo, &["branch", &branch]).await.unwrap();
13577 assert!(runner.prep().await.is_err());
13578 let saved = RunState::load(&runner.state.id).unwrap();
13579 assert!(saved.candidates.is_empty());
13580 let event = saved
13581 .events
13582 .iter()
13583 .find(|e| e.node == "prep-intent")
13584 .unwrap();
13585 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13586 assert_eq!(intent["branch"], branch);
13587 assert_eq!(intent["start"], runner.state.base_commit);
13588 assert_eq!(
13589 intent["worktree"],
13590 runner
13591 .state
13592 .worktree_root()
13593 .join(format!("cand-{label}"))
13594 .to_string_lossy()
13595 .as_ref()
13596 );
13597 assert_eq!(intent["picks"], serde_json::json!([]));
13598 }
13599
13600 #[tokio::test]
13601 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13602 ask_test_home();
13603 let tmp = tempfile::tempdir().unwrap();
13604 let repo = tmp.path().join("repo");
13605 std::fs::create_dir_all(&repo).unwrap();
13606 init_repo(&repo);
13607 let mut runner = runner_at(RunStatus::Prep);
13608 runner.state.repo = repo.clone();
13609 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13610 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13611 runner.state.sources.pick_commits.push(refs::Input {
13612 token: "missing".into(),
13613 sha: "0".repeat(40),
13614 provenance: refs::Provenance::ExplicitCli,
13615 });
13616 assert!(runner.prep().await.is_err());
13617 assert!(runner.state.candidates.is_empty());
13618 assert!(!runner.state.worktree_root().exists());
13619 }
13620
13621 #[tokio::test]
13622 async fn prose_and_legacy_inference_never_import_into_candidates() {
13623 ask_test_home();
13624 let tmp = tempfile::tempdir().unwrap();
13625 let repo = tmp.path().join("repo");
13626 std::fs::create_dir_all(&repo).unwrap();
13627 init_repo(&repo);
13628 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13629 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13630 .await
13631 .unwrap();
13632 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13633 git::git(&repo, &["add", "."]).await.unwrap();
13634 git::git(&repo, &["commit", "-m", "historical implementation"])
13635 .await
13636 .unwrap();
13637 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13638 git::git(&repo, &["checkout", "main"]).await.unwrap();
13639 for instruction in [
13640 format!("Example: magi/history/A at {source}"),
13641 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13642 format!("Never import magi/history/A or {source}"),
13643 format!("Conversation answer: I mentioned {source} only as background"),
13644 ] {
13645 let mut runner = runner_at(RunStatus::Prep);
13646 runner.state.repo = repo.clone();
13647 runner.state.base_commit = base.clone();
13648 runner.state.instruction = instruction.clone();
13649 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13650 runner.state.config.blind.commit_msg_hook = false;
13651 runner.state.seeds = vec![refs::Seed {
13652 token: "magi/history/A".into(),
13653 sha: source.clone(),
13654 kind: refs::SeedKind::Unmerged,
13655 branch: true,
13656 detail: String::new(),
13657 }];
13658 runner.roles.implementers = vec![spec("one")];
13659 assert_eq!(seeded_instruction(&runner.state), instruction);
13660 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13662 legacy.as_object_mut().unwrap().remove("sources");
13663 runner.state = serde_json::from_value(legacy).unwrap();
13664 assert_eq!(runner.state.sources, refs::Sources::default());
13665 runner.state.save().unwrap();
13666 runner.state = RunState::load(&runner.state.id).unwrap();
13667 runner.prep().await.unwrap();
13668 for candidate in &runner.state.candidates {
13669 assert_eq!(
13670 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13671 base
13672 );
13673 assert!(!candidate.worktree.join("unwanted").exists());
13674 }
13675 }
13676 }
13677
13678 #[tokio::test]
13679 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13680 ask_test_home();
13681 let tmp = tempfile::tempdir().unwrap();
13682 let repo = tmp.path().join("repo");
13683 std::fs::create_dir_all(&repo).unwrap();
13684 init_repo(&repo);
13685 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13686 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13687 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13688 git::git(&repo, &["add", "."]).await.unwrap();
13689 git::git(&repo, &["commit", "-m", "required seed"])
13690 .await
13691 .unwrap();
13692 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13693 git::git(&repo, &["checkout", "main"]).await.unwrap();
13694 let mut runner = runner_at(RunStatus::Prep);
13695 runner.state.repo = repo.clone();
13696 runner.state.base_commit = base.clone();
13697 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13698 runner.state.config.graph.advise = false;
13699 runner.state.config.blind.commit_msg_hook = false;
13700 runner.state.sources =
13701 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13702 .await
13703 .unwrap();
13704 runner.roles.implementers = vec![spec("one"), spec("two")];
13705 runner.roles.judges = vec![spec("judge")];
13706 let label = blind::assign_labels(2, runner.state.seed)[0];
13707 let branch = runner.state.branch_for(label);
13708 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13709 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13710 .await
13711 .unwrap();
13712 runner.state.candidates.push(Candidate {
13713 index: 0,
13714 label,
13715 agent: "one".into(),
13716 branch,
13717 worktree,
13718 summary: String::new(),
13719 stat: String::new(),
13720 files: 0,
13721 commits: 0,
13722 empty: false,
13723 failed: None,
13724 verified_noop: None,
13725 duration_ms: 0,
13726 folded: false,
13727 });
13728 runner.state.save().unwrap();
13729 runner.state = RunState::load(&runner.state.id).unwrap();
13731 assert_eq!(runner.state.prep_complete, Some(false));
13732 runner.prep().await.unwrap();
13733 assert_eq!(runner.state.prep_complete, Some(true));
13734 assert_eq!(runner.state.candidates.len(), 2);
13735 for candidate in &runner.state.candidates {
13736 assert_eq!(
13737 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13738 git::tree_of(&repo, &seed).await.unwrap()
13739 );
13740 assert_eq!(
13741 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13742 "required commit"
13743 );
13744 }
13745 assert!(runner.state.worktree_root().join("judge-1").exists());
13746 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13747 runner.prep().await.unwrap();
13748 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13749
13750 runner.state.prep_complete = Some(false);
13751 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13752 run: Some(format!(
13753 "test -f '{}'",
13754 tmp.path().join("dependency-ready").display()
13755 )),
13756 ..Default::default()
13757 }];
13758 assert!(runner.prep().await.is_err());
13759 assert!(runner.state.prep_failure.is_some());
13760 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13761 runner.state = RunState::load(&runner.state.id).unwrap();
13762 assert!(
13763 runner.prep().await.is_err(),
13764 "correcting the cause alone must not auto-retry"
13765 );
13766 let failed_at = runner.state.events.last().unwrap().at;
13767 assert!(
13768 runner.retry_preparation(failed_at).is_err(),
13769 "old approvals cannot unlock the hold"
13770 );
13771 let mut task = crate::queue::Task::new(
13772 "retry".into(),
13773 "retry setup".into(),
13774 repo.clone(),
13775 crate::queue::Source::Human,
13776 );
13777 task.start(runner.state.id.clone());
13778 task.hold_machine(Some("setup failed".into()));
13779 task.release_operator("cli", None);
13780 runner
13781 .retry_preparation(task.resumption.as_ref().unwrap().at)
13782 .unwrap();
13783 runner.prep().await.unwrap();
13784 assert!(runner.state.prep_failure.is_none());
13785 assert_eq!(runner.state.prep_complete, Some(true));
13786 assert_eq!(runner.state.candidates.len(), 2);
13787 assert!(runner.state.worktree_root().join("judge-1").exists());
13788 for candidate in &runner.state.candidates {
13789 assert_eq!(
13790 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13791 git::tree_of(&repo, &seed).await.unwrap()
13792 );
13793 }
13794 }
13795
13796 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13797 let mut heads = Vec::new();
13798 for candidate in candidates {
13799 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13800 }
13801 heads
13802 }
13803
13804 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13805 ask_test_home();
13806 let tmp = tempfile::tempdir().unwrap();
13807 let repo = tmp.path().join("repo");
13808 std::fs::create_dir(&repo).unwrap();
13809 init_repo(&repo);
13810 let mut runner = runner_at(RunStatus::Reviewing);
13811 runner.state.repo = repo.clone();
13812 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13813 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13814 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13815 let branch = format!("candidate-{label}");
13816 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13817 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13818 .await
13819 .unwrap();
13820 if label == 'B' {
13821 std::fs::write(worktree.join("change"), "winner").unwrap();
13822 git::rescue_commit(&worktree, "candidate change")
13823 .await
13824 .unwrap();
13825 }
13826 runner.state.candidates.push(Candidate {
13827 index,
13828 label,
13829 agent: "alpha".into(),
13830 branch,
13831 worktree,
13832 summary: "completed reply".into(),
13833 stat: String::new(),
13834 files: usize::from(label == 'B'),
13835 commits: usize::from(label == 'B'),
13836 empty: label == 'A',
13837 failed: None,
13838 verified_noop: None,
13839 duration_ms: 1,
13840 folded: false,
13841 });
13842 }
13843 runner.state.judge_skipped = true;
13844 runner.state.tally = Some(Tally {
13845 first_choice: BTreeMap::new(),
13846 borda: BTreeMap::new(),
13847 winner: 'B',
13848 rankings: 0,
13849 unanimous_initial: false,
13850 deliberated: false,
13851 changed_votes: 0,
13852 unanimous_final: false,
13853 tie_break: None,
13854 judges: 0,
13855 present: 0,
13856 quorum: 0,
13857 met_quorum: true,
13858 uncontested: Some("only B produced a change".into()),
13859 });
13860 runner.state.save().unwrap();
13861 (tmp, runner)
13862 }
13863
13864 #[tokio::test]
13865 #[cfg(unix)]
13866 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13867 for (solo, failed_checkpoint) in
13868 [(true, false), (false, false), (true, true), (false, true)]
13869 {
13870 let (_tmp, mut runner) = preservation_resume_fixture().await;
13871 if solo {
13872 runner.state.candidates.retain(|c| c.label == 'A');
13873 }
13874 runner.state.candidates[0].empty = false;
13875 runner.state.candidates[0].summary.clear();
13876 runner.state.tally = None;
13877 runner.state.judge_skipped = false;
13878 runner.state.status = RunStatus::Implementing;
13879 runner.state.config.disk.min_free_bytes = u64::MAX;
13880 let mut implementer = spec("alpha");
13881 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13882 runner.roles.implementers = vec![implementer];
13883 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13884 if failed_checkpoint {
13885 std::fs::create_dir_all(&patch).unwrap();
13886 }
13887 runner.state.save().unwrap();
13888 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13889 runner.state = RunState::load(&runner.state.id).unwrap();
13890 assert_eq!(runner.state.status, RunStatus::Blocked);
13891 assert!(runner.state.interrupted_implementations.contains(&'A'));
13892 assert_eq!(runner.state.candidates[0].commits, 0);
13893 assert!(!runner.state.candidates[0].empty);
13894 assert!(runner.state.candidates[0].failed.is_none());
13895 if failed_checkpoint {
13896 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13897 std::fs::remove_dir(patch).unwrap();
13898 }
13899 runner.state.resource_block = None;
13900 runner.state.config.disk.min_free_bytes = 0;
13901 runner.roles.implementers[0].command = vec![
13902 "sh".into(),
13903 "-c".into(),
13904 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13905 ];
13906 runner.implement().await.unwrap();
13907 assert!(runner.state.interrupted_implementations.is_empty());
13908 assert!(runner.state.candidate_preservation.is_empty());
13909 assert_eq!(runner.state.candidates[0].commits, 1);
13910 assert!(!runner.state.candidates[0].empty);
13911 assert!(
13912 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13913 .unwrap()
13914 .contains("retried empty interrupted seat")
13915 );
13916 if !solo {
13917 assert_eq!(runner.state.candidates[1].commits, 1);
13918 }
13919 }
13920 }
13921
13922 #[tokio::test]
13923 #[cfg(unix)]
13924 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13925 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13926 let (_tmp, mut runner) = preservation_resume_fixture().await;
13927 runner.state.candidates.retain(|c| c.label == 'A');
13928 runner.state.candidates[0].empty = false;
13929 runner.state.candidates[0].summary.clear();
13930 runner.state.tally = None;
13931 runner.state.judge_skipped = false;
13932 runner.state.status = RunStatus::Implementing;
13933 runner.state.config.disk.min_free_bytes = u64::MAX;
13936 let wt = runner.state.candidates[0].worktree.clone();
13937 std::fs::write(
13938 wt.join("interrupted-source"),
13939 "retain implementation work\n",
13940 )
13941 .unwrap();
13942 if committed {
13943 git::rescue_commit(&wt, "implementation checkpoint")
13944 .await
13945 .unwrap();
13946 }
13947 let lock = git::git(
13948 &wt,
13949 &[
13950 "rev-parse",
13951 "--path-format=absolute",
13952 "--git-path",
13953 "index.lock",
13954 ],
13955 )
13956 .await
13957 .unwrap();
13958 if block_rescue {
13959 std::fs::write(&lock, "fixture-owned lock").unwrap();
13960 }
13961 let mut implementer = spec("alpha");
13962 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13963 runner.roles.implementers = vec![implementer];
13964 runner.state.save().unwrap();
13965 let result = runner.implement().await;
13966 assert_eq!(result.is_err(), block_rescue);
13967 assert_eq!(
13968 runner.state.resource_block.as_ref().unwrap().kind,
13969 "disk pressure"
13970 );
13971 assert!(
13972 runner
13973 .state
13974 .events
13975 .iter()
13976 .any(|e| e.message.contains("agent execution interrupted"))
13977 );
13978 assert!(
13979 runner
13980 .state
13981 .dir()
13982 .join("artifacts/candidate-A-recovery/staged.patch")
13983 .exists()
13984 );
13985 if block_rescue {
13986 let saved = RunState::load(&runner.state.id).unwrap();
13987 assert!(saved.candidate_preservation.contains_key(&'A'));
13988 assert!(!saved.candidates[0].empty);
13989 assert_eq!(
13990 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
13991 "retain implementation work\n"
13992 );
13993 std::fs::remove_file(lock).unwrap();
13994 runner.state = saved;
13995 } else {
13996 assert_eq!(runner.state.status, RunStatus::Blocked);
13997 assert_eq!(runner.state.candidates[0].commits, 1);
13998 }
13999 runner.roles.implementers.clear();
14002 runner.state.config.disk.min_free_bytes = 0;
14003 runner.state.resource_block = None;
14004 runner.implement().await.unwrap();
14005 assert!(runner.state.candidate_preservation.is_empty());
14006 assert_eq!(runner.state.candidates[0].commits, 1);
14007 assert!(!runner.state.candidates[0].empty);
14008 assert!(git::is_clean(&wt).await.unwrap());
14009 assert!(
14010 crate::run::read_artifact(&runner.state, "cand-A.patch")
14011 .unwrap()
14012 .contains("retain implementation work")
14013 );
14014 }
14015 }
14016
14017 #[tokio::test]
14018 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14019 let (_tmp, mut runner) = preservation_resume_fixture().await;
14020 runner.implement().await.unwrap();
14022 assert!(runner.state.candidate_preservation.is_empty());
14023 assert!(
14024 !runner
14025 .state
14026 .dir()
14027 .join("artifacts/candidate-A-recovery")
14028 .exists()
14029 );
14030 runner.fold_losers().await.unwrap();
14031 assert!(!runner.state.candidates[0].worktree.exists());
14032 for schema in [14, crate::run::SCHEMA] {
14033 runner.state.schema = schema;
14034 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14035 runner.state.save().unwrap();
14036 runner.state = RunState::load(&runner.state.id).unwrap();
14037 runner.implement().await.unwrap();
14038 assert!(runner.state.candidate_preservation.is_empty());
14039 assert!(runner.state.candidates[0].folded);
14040 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14041 }
14042 }
14043
14044 #[tokio::test]
14045 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14046 let (_tmp, mut runner) = preservation_resume_fixture().await;
14047 let wt = runner.state.candidates[0].worktree.clone();
14048 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14049 runner.state.schema = 14;
14050 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14051 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14052 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14053 for _ in 0..2 {
14054 assert!(runner.implement().await.is_err());
14055 assert_eq!(runner.state.status, RunStatus::Blocked);
14056 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14057 assert!(
14058 runner.state.candidates[0].empty,
14059 "old verdict must remain unusable"
14060 );
14061 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14062 assert!(runner.state.judge_skipped);
14063 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14064 assert!(
14065 crate::run::read_artifact(&runner.state, "cand-A.patch")
14066 .unwrap()
14067 .contains("owned edits")
14068 );
14069 assert!(wt.exists());
14070 assert_eq!(
14071 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14072 .await
14073 .unwrap(),
14074 1
14075 );
14076 runner.state = RunState::load(&runner.state.id).unwrap();
14077 }
14078 }
14079
14080 #[tokio::test]
14081 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14082 let (_tmp, mut runner) = preservation_resume_fixture().await;
14083 let wt = runner.state.candidates[0].worktree.clone();
14084 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14085 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14086 let index_lock = PathBuf::from(
14087 git::git(
14088 &wt,
14089 &[
14090 "rev-parse",
14091 "--path-format=absolute",
14092 "--git-path",
14093 "index.lock",
14094 ],
14095 )
14096 .await
14097 .unwrap(),
14098 );
14099 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14100 runner.state.schema = 14;
14101 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14102 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14103 assert!(runner.implement().await.is_err());
14104 assert!(runner.after_implement().is_err());
14105 assert!(runner.fold_losers().await.is_err());
14106 assert_eq!(
14107 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14108 "failed seat edits"
14109 );
14110 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14111 runner.state = RunState::load(&runner.state.id).unwrap();
14112 std::fs::remove_file(index_lock).unwrap();
14113 assert!(runner.implement().await.is_err());
14115 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14116 assert!(runner.state.candidates[0].empty);
14117 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14118 assert!(
14119 crate::run::read_artifact(&runner.state, "cand-A.patch")
14120 .unwrap()
14121 .contains("failed seat edits")
14122 );
14123 assert!(wt.exists());
14124 }
14125
14126 #[tokio::test]
14127 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14128 let (_tmp, mut runner) = preservation_resume_fixture().await;
14129 let repo = runner.state.repo.clone();
14130 let common = git::git(
14131 &repo,
14132 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14133 )
14134 .await
14135 .unwrap();
14136 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14137 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14138 assert!(runner.fold_losers().await.is_err());
14139 assert!(!runner.state.candidates[0].worktree.exists());
14140 assert!(!runner.state.candidates[0].folded);
14141 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14142 runner.state = RunState::load(&runner.state.id).unwrap();
14143 std::fs::remove_file(branch_lock).unwrap();
14144 let judge = runner.state.worktree_root().join("judge-1");
14145 git::worktree_add_detached(&repo, &judge, "main")
14146 .await
14147 .unwrap();
14148 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14149 runner.roles.judges = vec![spec("judge")];
14150 assert!(runner.fold_losers().await.is_err());
14151 runner.state = RunState::load(&runner.state.id).unwrap();
14152 assert!(
14153 runner.state.candidates[0].folded,
14154 "candidate progress saved before judge failure"
14155 );
14156 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14157 assert!(judge.join("uncommitted").exists());
14158 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14159 runner.fold_losers().await.unwrap();
14160 assert!(!judge.exists());
14161 runner.fold_losers().await.unwrap();
14162 }
14163
14164 #[tokio::test]
14165 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14166 let (_tmp, mut runner) = preservation_resume_fixture().await;
14167 runner.state.candidates.retain(|c| c.label == 'B');
14168 runner.state.tally = None;
14169 runner.state.judge_skipped = false;
14170 runner.state.prep_complete = Some(true);
14171 runner.state.config.graph.advise = false;
14172 runner.state.config.graph.selected_implementer = Some("alpha".into());
14173 let wt = runner.state.candidates[0].worktree.clone();
14174 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14175 runner
14176 .state
14177 .candidate_preservation
14178 .insert('B', "awaiting persistence".into());
14179 let index_lock = git::git(
14180 &wt,
14181 &[
14182 "rev-parse",
14183 "--path-format=absolute",
14184 "--git-path",
14185 "index.lock",
14186 ],
14187 )
14188 .await
14189 .unwrap();
14190 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14191 runner.state.save().unwrap();
14192 assert!(runner.execute_graph().await.is_err());
14193 assert_eq!(runner.state.status, RunStatus::Blocked);
14194 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14195 std::fs::remove_file(index_lock).unwrap();
14196 for _ in 0..2 {
14199 runner.state = RunState::load(&runner.state.id).unwrap();
14200 runner.pause.park();
14201 runner.execute_graph().await.unwrap();
14202 assert_eq!(runner.state.status, RunStatus::Implementing);
14203 assert!(runner.state.candidate_preservation.is_empty());
14204 assert_eq!(runner.state.candidates[0].commits, 2);
14205 assert!(runner.state.tally.is_none());
14206 }
14207 runner.implement().await.unwrap();
14208 assert_eq!(runner.state.status, RunStatus::Judging);
14209 runner.state.status = RunStatus::Blocked;
14211 runner.state.candidates[0].failed = Some("implementation failed".into());
14212 runner.state.save().unwrap();
14213 let error = runner.execute_graph().await.unwrap_err();
14214 assert!(
14215 error
14216 .to_string()
14217 .contains("selected implementation is held")
14218 );
14219 }
14220
14221 #[tokio::test]
14222 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14223 let (_tmp, mut runner) = preservation_resume_fixture().await;
14224 let repo = runner.state.repo.clone();
14225 let candidate = runner.state.candidates[0].clone();
14226 let registered = git::worktree_holding(&repo, &candidate.branch)
14229 .await
14230 .unwrap()
14231 .unwrap();
14232 assert_eq!(
14233 registered.canonicalize().unwrap(),
14234 candidate.worktree.canonicalize().unwrap()
14235 );
14236 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14237 assert_eq!(
14238 git::worktree_holding(&repo, &candidate.branch)
14239 .await
14240 .unwrap(),
14241 Some(registered)
14242 );
14243 assert!(
14244 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14245 "the stale registration initially keeps the branch checked out"
14246 );
14247 runner.fold_losers().await.unwrap();
14248 assert!(runner.state.candidates[0].folded);
14249 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14250 assert_eq!(
14251 git::worktree_holding(&repo, &candidate.branch)
14252 .await
14253 .unwrap(),
14254 None
14255 );
14256 runner.state = RunState::load(&runner.state.id).unwrap();
14257 runner.fold_losers().await.unwrap();
14258 }
14259
14260 #[tokio::test]
14261 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14262 let (_tmp, mut runner) = preservation_resume_fixture().await;
14263 let repo = runner.state.repo.clone();
14264 let candidate = runner.state.candidates[0].clone();
14265 let winner = runner.state.candidates[1].clone();
14266 let registered = git::worktree_holding(&repo, &candidate.branch)
14267 .await
14268 .unwrap()
14269 .unwrap();
14270 assert_eq!(
14271 registered.canonicalize().unwrap(),
14272 candidate.worktree.canonicalize().unwrap()
14273 );
14274 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14275 assert_eq!(
14276 git::worktree_holding(&repo, &candidate.branch)
14277 .await
14278 .unwrap(),
14279 Some(registered)
14280 );
14281 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14282
14283 for _ in 0..2 {
14284 fold_run(&mut runner.state, false, &crate::run::home())
14285 .await
14286 .unwrap();
14287 runner.state = RunState::load(&runner.state.id).unwrap();
14288 assert!(runner.state.candidates[0].folded);
14289 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14290 assert_eq!(
14291 git::worktree_holding(&repo, &candidate.branch)
14292 .await
14293 .unwrap(),
14294 None
14295 );
14296 assert!(!runner.state.candidates[1].folded);
14297 assert!(winner.worktree.exists());
14298 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14299 }
14300 }
14301
14302 #[tokio::test]
14303 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14304 ask_test_home();
14305 let tmp = tempfile::tempdir().unwrap();
14306 let repo = tmp.path().join("repo");
14307 std::fs::create_dir(&repo).unwrap();
14308 init_repo(&repo);
14309 let mut runner = runner_at(RunStatus::Implementing);
14310 runner.state.repo = repo.clone();
14311 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14312 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14313 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14314 let branch = format!("candidate-{label}");
14315 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14316 git::worktree_add_branch(&repo, &wt, &branch, "main")
14317 .await
14318 .unwrap();
14319 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14320 git::git(&wt, &["add", "README.md"]).await.unwrap();
14321 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14322 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14323 runner.state.candidates.push(Candidate {
14324 index,
14325 label,
14326 agent: "alpha".to_owned(),
14327 branch,
14328 worktree: wt,
14329 summary: "completed implementation".to_owned(),
14330 stat: String::new(),
14331 files: 0,
14332 commits: 0,
14333 empty: false,
14334 failed: None,
14335 verified_noop: None,
14336 duration_ms: 42,
14337 folded: false,
14338 });
14339 runner
14340 .state
14341 .candidate_preservation
14342 .insert(label, "awaiting persistence".to_owned());
14343 }
14344 let wt = runner.state.candidates[0].worktree.clone();
14345 let index_lock = git::git(
14346 &wt,
14347 &[
14348 "rev-parse",
14349 "--path-format=absolute",
14350 "--git-path",
14351 "index.lock",
14352 ],
14353 )
14354 .await
14355 .unwrap();
14356 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14357 runner.state.save().unwrap();
14358 assert!(runner.persist_candidates().await.is_err());
14359 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14360 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14361 assert_eq!(
14362 runner.state.candidates[1].commits, 1,
14363 "other results are still collected"
14364 );
14365 assert!(
14366 !runner.state.candidates[0].empty,
14367 "failed rescue is not no change"
14368 );
14369 assert_eq!(
14370 runner.state.candidates[0].summary,
14371 "completed implementation"
14372 );
14373 assert!(runner.after_implement().is_err());
14374 let home = tmp.path().join("fold-home");
14375 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14376 assert!(wt.join("new-file").exists());
14377 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14378 assert!(recovery.join("staged.patch").exists());
14379 assert!(recovery.join("unstaged.patch").exists());
14380 assert_eq!(
14381 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14382 "untracked edits\n"
14383 );
14384 runner.state.begin_pass();
14385 runner.state.end_pass_parked(ParkCause::Upgrade {
14386 from: "1.0".into(),
14387 to: Some("1.1".into()),
14388 });
14389 runner.state.save().unwrap();
14390 runner.state = RunState::load(&runner.state.id).unwrap();
14391 assert!(matches!(
14392 runner.state.passes.last().unwrap().end,
14393 Some(crate::run::PassEnd::Parked {
14394 cause: ParkCause::Upgrade { .. }
14395 })
14396 ));
14397 std::fs::remove_file(index_lock).unwrap();
14398 runner.implement().await.unwrap();
14400 assert!(runner.state.candidate_preservation.is_empty());
14401 assert!(
14402 runner
14403 .state
14404 .candidates
14405 .iter()
14406 .all(|c| c.commits == 1 && !c.empty)
14407 );
14408 assert!(git::is_clean(&wt).await.unwrap());
14409 assert!(
14410 crate::run::read_artifact(&runner.state, "cand-A.patch")
14411 .unwrap()
14412 .contains("later tracked edits")
14413 );
14414 }
14415
14416 #[tokio::test]
14417 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14418 ask_test_home();
14419 let tmp = tempfile::tempdir().unwrap();
14420 let repo = tmp.path().join("repo");
14421 std::fs::create_dir(&repo).unwrap();
14422 init_repo(&repo);
14423 let wt = tmp.path().join("candidate");
14424 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14425 .await
14426 .unwrap();
14427 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14428 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14429 assert!(wt.join("uncommitted").exists());
14430 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14431 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14432 let mut runner = runner_at(RunStatus::Implementing);
14433 runner.state.repo = repo.clone();
14434 runner.state.base_commit = "missing-base".to_owned();
14435 runner.state.candidates.push(Candidate {
14436 index: 0,
14437 label: 'A',
14438 agent: "alpha".into(),
14439 branch: "candidate".into(),
14440 worktree: wt.clone(),
14441 summary: "completed".into(),
14442 stat: String::new(),
14443 files: 0,
14444 commits: 0,
14445 empty: false,
14446 failed: None,
14447 verified_noop: None,
14448 duration_ms: 1,
14449 folded: false,
14450 });
14451 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14452 assert!(
14453 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14454 .await
14455 .is_err()
14456 );
14457 assert!(!runner.state.candidates[0].folded);
14458 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14459 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14460 runner
14461 .state
14462 .candidate_preservation
14463 .insert('A', "pending".into());
14464 assert!(runner.persist_candidates().await.is_err());
14465 assert!(!runner.state.candidates[0].empty);
14466 assert!(!runner.state.candidates[0].folded);
14467 assert!(wt.exists());
14468 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14469 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14470 std::fs::create_dir_all(&patch_path).unwrap();
14471 assert!(
14472 runner.persist_candidates().await.is_err(),
14473 "artifact failure stays pending"
14474 );
14475 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14476 assert!(!runner.state.candidates[0].empty);
14477 std::fs::remove_dir(patch_path).unwrap();
14478 runner.persist_candidates().await.unwrap();
14479 assert!(
14480 runner.state.candidates[0].empty,
14481 "only successful inspection confirms no change"
14482 );
14483 }
14484
14485 fn init_repo(dir: &Path) {
14486 let run = |args: &[&str]| {
14487 let out = std::process::Command::new("git")
14488 .args(args)
14489 .current_dir(dir)
14490 .quiet()
14491 .output()
14492 .expect("spawn git");
14493 assert!(
14494 out.status.success(),
14495 "git {args:?} failed: {}",
14496 String::from_utf8_lossy(&out.stderr)
14497 );
14498 };
14499 run(&["init", "-b", "main"]);
14500 run(&["config", "user.name", "magi test"]);
14501 run(&["config", "user.email", "magi@example.com"]);
14502 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14503 run(&["add", "-A"]);
14504 run(&["commit", "-m", "init"]);
14505 }
14506
14507 fn ask_test_home() {
14515 crate::run::pin_test_home();
14516 }
14517
14518 fn runner_at(status: RunStatus) -> Runner {
14521 let mut state = RunState::new(
14522 PathBuf::from("/nonexistent/repo"),
14523 "main".to_owned(),
14524 "deadbeef".to_owned(),
14525 "task".to_owned(),
14526 Config::default(),
14527 );
14528 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14531 state.id = format!(
14532 "20000101-000000-{:04x}",
14533 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14534 );
14535 state.status = status;
14536 Runner {
14537 state,
14538 roles: ResolvedRoles {
14539 implementers: Vec::new(),
14540 judges: Vec::new(),
14541 reviewers: Vec::new(),
14542 fixer: None,
14543 conductor: conductor(),
14544 implementer_roster: Vec::new(),
14545 judge_roster: Vec::new(),
14546 reviewer_roster: Vec::new(),
14547 },
14548 sem: Arc::new(Semaphore::new(1)),
14549 pause: Pause::new(),
14550 interrupt: Pause::new(),
14551 }
14552 }
14553
14554 #[test]
14555 fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14556 crate::run::pin_test_home();
14557 let mut runner = runner_at(RunStatus::Gating);
14558 runner.state.save().unwrap();
14559 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14562 assert!(runner.claim_driver().is_err());
14563 assert!(runner.state.driver_exited);
14564 assert_eq!(runner.state.driver_pid, None);
14565 let disk = RunState::load(&runner.state.id).unwrap();
14566 assert!(disk.driver_exited);
14567 assert_eq!(disk.driver_pid, None);
14568 assert_eq!(disk.status, RunStatus::Gating);
14569 }
14570
14571 #[tokio::test]
14572 async fn an_early_graph_error_releases_its_driver() {
14573 crate::run::pin_test_home();
14574 let mut runner = runner_at(RunStatus::Blocked);
14575 runner.state.config.graph.selected_implementer = Some("selected".into());
14576 runner.state.save().unwrap();
14577 assert!(runner.execute().await.is_err());
14578 let disk = RunState::load(&runner.state.id).unwrap();
14579 assert!(disk.driver_exited);
14580 assert_eq!(disk.driver_pid, None);
14581 assert_eq!(disk.status, RunStatus::Blocked);
14582 }
14583
14584 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14585 crate::run::MergeRepair {
14586 question: "repair-question".into(),
14587 head: "old-head".into(),
14588 pr: "https://github.com/o/r/pull/42".into(),
14589 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14590 gate: vec![outcome(Some(0), false)],
14591 gate_head: Some("old-head".into()),
14592 gate_fixes: Vec::new(),
14593 pre_gate: Vec::new(),
14594 phase,
14595 failure: None,
14596 }
14597 }
14598
14599 #[tokio::test]
14600 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14601 crate::run::pin_test_home();
14602 let mut runner = runner_at(RunStatus::Landing);
14603 runner.state.merge_repairs.push(merge_repair_request(
14604 crate::run::MergeRepairPhase::Requested,
14605 ));
14606 runner.state.save().unwrap();
14607 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14608 assert!(
14609 runner
14610 .execute()
14611 .await
14612 .unwrap_err()
14613 .to_string()
14614 .contains("already running")
14615 );
14616 assert_eq!(
14617 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14618 crate::run::MergeRepairPhase::Requested
14619 );
14620 drop(claim);
14621 let mut disk = runner.state.clone();
14622 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14623 disk.save().unwrap();
14624 assert!(
14625 runner
14626 .execute()
14627 .await
14628 .unwrap_err()
14629 .to_string()
14630 .contains("decision changed")
14631 );
14632 assert_eq!(
14633 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14634 crate::run::MergeRepairPhase::Complete
14635 );
14636 }
14637
14638 #[test]
14639 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14640 crate::run::pin_test_home();
14641 let mut runner = runner_at(RunStatus::Landing);
14642 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14643 runner.state.reviews = request.reviews.clone();
14644 runner.state.gate = request.gate.clone();
14645 runner.state.gate_ran = true;
14646 runner.state.land_approval = Some(crate::run::LandApproval {
14647 question: "old".into(),
14648 head: "old-head".into(),
14649 });
14650 runner.state.merge_repairs.push(request);
14651 runner.begin_merge_repair_review(0).unwrap();
14652 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14653 assert!(!runner.state.gate_status().ok());
14654 assert!(runner.state.land_approval.is_none());
14655 let disk = RunState::load(&runner.state.id).unwrap();
14656 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14657 assert!(disk.merge_repairs[0].gate[0].ok());
14658 assert_eq!(
14659 disk.merge_repairs[0].phase,
14660 crate::run::MergeRepairPhase::Reviewing
14661 );
14662 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14663 assert!(disk.reviews.is_empty());
14664 }
14665
14666 #[tokio::test]
14667 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14668 crate::run::pin_test_home();
14669 for phase in [
14670 crate::run::MergeRepairPhase::Fixing,
14671 crate::run::MergeRepairPhase::Requested,
14672 ] {
14673 let mut runner = runner_at(RunStatus::Landing);
14674 runner.state.config.graph.land_rounds = 0;
14675 runner.state.merge_repairs.push(merge_repair_request(phase));
14676 runner.state.save().unwrap();
14677 assert!(!runner.repair_merge_review().await.unwrap());
14678 assert_eq!(runner.state.status, RunStatus::Blocked);
14679 assert_eq!(
14680 runner.state.merge_repairs[0].phase,
14681 crate::run::MergeRepairPhase::Failed
14682 );
14683 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14684 assert!(
14685 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14686 "interrupted"
14687 } else {
14688 "budget exhausted"
14689 })
14690 );
14691 assert!(runner.state.merge.is_none());
14692 }
14693 }
14694
14695 #[test]
14699 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14700 crate::run::pin_test_home();
14701 let mut runner = runner_at(RunStatus::Implementing);
14702 let interrupt = Pause::new();
14703 runner.watch_interrupt(interrupt.clone());
14704
14705 interrupt.park_because("task a1b2 asked to run first");
14706
14707 assert!(runner.park_here().expect("park_here"));
14708 assert!(runner.state.parked);
14709 let last = runner.state.events.last().expect("a park event");
14710 assert_eq!(last.node, "park");
14711 assert!(
14712 last.message.contains("task a1b2 asked to run first"),
14713 "expected the interrupt reason in {:?}",
14714 last.message
14715 );
14716 }
14717
14718 #[test]
14721 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14722 crate::run::pin_test_home();
14723 let mut runner = runner_at(RunStatus::Implementing);
14724 runner.state.begin_pass();
14725 let shutdown = Pause::new();
14726 runner.on_pause(shutdown.clone());
14727 let upgrade = ParkCause::Upgrade {
14728 from: "0.121.0".to_owned(),
14729 to: Some("0.121.1".to_owned()),
14730 };
14731 shutdown.park_for(upgrade.clone());
14732 shutdown.park_for(ParkCause::Operator);
14733
14734 assert!(runner.park_here().expect("park_here"));
14735 assert_eq!(
14736 runner.state.passes[0].end,
14737 Some(crate::run::PassEnd::Parked { cause: upgrade })
14738 );
14739 let last = runner.state.events.last().expect("a park event");
14740 assert!(last.message.contains("0.121.0"), "{}", last.message);
14741 }
14742
14743 #[test]
14751 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14752 crate::run::pin_test_home();
14753 let mut runner = runner_at(RunStatus::Implementing);
14754 let shutdown = Pause::new();
14755 runner.on_pause(shutdown.clone());
14756 let interrupt = Pause::new();
14757 runner.watch_interrupt(interrupt.clone());
14758
14759 assert!(!runner.park_here().expect("park_here"));
14761 assert!(!runner.state.parked);
14762
14763 interrupt.park_because("test");
14765 assert!(!shutdown.parked());
14766 assert!(runner.park_here().expect("park_here"));
14767 }
14768
14769 #[tokio::test]
14783 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14784 crate::run::pin_test_home();
14785 let mut runner = runner_at(RunStatus::Implementing);
14786 let interrupt = Pause::new();
14787 runner.watch_interrupt(interrupt.clone());
14788
14789 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14790 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14791
14792 let node = async move {
14796 started_tx.send(()).expect("send started");
14797 finish_rx.await.expect("recv finish");
14798 "node finished"
14799 };
14800
14801 let interrupter = async move {
14802 started_rx.await.expect("recv started");
14803 interrupt.park_because("higher-priority task waiting");
14805 tokio::task::yield_now().await;
14809 finish_tx.send(()).expect("send finish");
14810 };
14811
14812 let (node_result, ()) = tokio::join!(node, interrupter);
14813 assert_eq!(
14814 node_result, "node finished",
14815 "the in-flight call ran to completion"
14816 );
14817
14818 assert!(runner.park_here().expect("park_here"));
14821 assert!(runner.state.parked);
14822 }
14823
14824 #[test]
14830 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14831 crate::run::pin_test_home();
14832 let mut runner = runner_at(RunStatus::Judging);
14833 runner.state.config.agents = vec![conductor()];
14837 runner.state.candidates = vec![Candidate {
14838 index: 0,
14839 label: 'A',
14840 agent: "alpha".to_owned(),
14841 branch: "magi/x/A".to_owned(),
14842 worktree: PathBuf::from("/nonexistent/worktree"),
14843 summary: "did the thing".to_owned(),
14844 stat: "1 file changed".to_owned(),
14845 files: 1,
14846 commits: 1,
14847 empty: false,
14848 failed: None,
14849 verified_noop: None,
14850 duration_ms: 1234,
14851 folded: false,
14852 }];
14853 let run_id = runner.state.id.clone();
14854
14855 let interrupt = Pause::new();
14856 runner.watch_interrupt(interrupt.clone());
14857 interrupt.park_because("task c3d4 asked to run first");
14858 assert!(runner.park_here().expect("park_here"));
14859
14860 let resumed = Runner::resume(&run_id).expect("resume");
14861 assert_eq!(resumed.state.candidates.len(), 1);
14862 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14863 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14864 assert_eq!(resumed.state.status, runner.state.status);
14865 assert!(
14866 resumed.state.parked,
14867 "still parked until `execute` actually walks the graph again"
14868 );
14869 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14870 }
14871
14872 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14874 let mut q = ask::Question::new(
14875 run.to_owned(),
14876 "implement".to_owned(),
14877 "impl-A".to_owned(),
14878 "Which storage backend should the cache use?".to_owned(),
14879 String::new(),
14880 vec!["SQLite".to_owned(), "Redis".to_owned()],
14881 );
14882 store.put(&mut q).unwrap();
14883 q
14884 }
14885
14886 #[test]
14887 fn a_failed_runs_open_question_is_abandoned() {
14888 ask_test_home();
14889 let store = ask::Questions::open();
14890 let mut runner = runner_at(RunStatus::Failed);
14891 let run = runner.state.id.clone();
14892 let q = ask_open_question(&store, &run);
14893
14894 runner.settle_questions();
14895
14896 let back = store.get(&q.id).unwrap();
14897 assert!(
14898 !back.status.open(),
14899 "the seat that asked died with the run; nobody is left to read an answer"
14900 );
14901 assert!(
14902 back.detail.contains(&run) && back.detail.contains("failed"),
14903 "the reason names what the run became, not just that it is gone: {}",
14904 back.detail
14905 );
14906 }
14907
14908 #[test]
14909 fn a_merged_runs_open_question_is_abandoned_too() {
14910 ask_test_home();
14911 let store = ask::Questions::open();
14912 for status in [RunStatus::Merged, RunStatus::Ready] {
14915 let mut runner = runner_at(status);
14916 let run = runner.state.id.clone();
14917 let q = ask_open_question(&store, &run);
14918
14919 runner.settle_questions();
14920
14921 let back = store.get(&q.id).unwrap();
14922 assert!(
14923 !back.status.open(),
14924 "{status:?} run's question must not outlive the run"
14925 );
14926 }
14927 }
14928
14929 #[test]
14930 fn a_still_resumable_runs_open_question_is_left_alone() {
14931 ask_test_home();
14932 let store = ask::Questions::open();
14933 for status in [RunStatus::Blocked, RunStatus::Stalled] {
14939 let mut runner = runner_at(status);
14940 let run = runner.state.id.clone();
14941 let q = ask_open_question(&store, &run);
14942
14943 runner.settle_questions();
14944
14945 let back = store.get(&q.id).unwrap();
14946 assert!(
14947 back.status.open(),
14948 "{status:?} is still alive; the question must still be waiting"
14949 );
14950 }
14951 }
14952
14953 #[test]
14954 fn settle_questions_never_touches_an_already_answered_question() {
14955 ask_test_home();
14956 let store = ask::Questions::open();
14957 let mut runner = runner_at(RunStatus::Failed);
14958 let run = runner.state.id.clone();
14959 let mut q = ask_open_question(&store, &run);
14960 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
14961 .unwrap();
14962 store.put(&mut q).unwrap();
14963
14964 runner.settle_questions();
14969 runner.settle_questions();
14970
14971 let back = store.get(&q.id).unwrap();
14972 assert_eq!(
14973 back.status,
14974 ask::QuestionStatus::Answered,
14975 "a real answer is a decision on record, never overwritten by a sweep"
14976 );
14977 }
14978
14979 #[tokio::test]
14990 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
14991 crate::run::pin_test_home();
14992 let tmp = tempfile::tempdir().expect("tempdir");
14993 let repo = tmp.path().join("repo");
14994 std::fs::create_dir_all(&repo).unwrap();
14995 init_repo(&repo);
14996
14997 let mut config = Config::default();
14998 config.graph.worktree_root = Some(tmp.path().join("wt"));
14999
15000 let mut state = RunState::new(
15001 repo.clone(),
15002 "main".to_owned(),
15003 "deadbeef".to_owned(),
15004 "task".to_owned(),
15005 config,
15006 );
15007 let root = state.worktree_root();
15008 let wt_a = root.join("cand-A");
15009 let wt_b = root.join("cand-B");
15010 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15011 .await
15012 .expect("worktree A");
15013 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15014 .await
15015 .expect("worktree B");
15016
15017 state.candidates = vec![
15018 Candidate {
15019 index: 0,
15020 label: 'A',
15021 agent: "alpha".to_owned(),
15022 branch: "magi/x/A".to_owned(),
15023 worktree: wt_a.clone(),
15024 summary: String::new(),
15025 stat: String::new(),
15026 files: 0,
15027 commits: 0,
15028 empty: false,
15029 failed: None,
15030 verified_noop: None,
15031 duration_ms: 0,
15032 folded: false,
15033 },
15034 Candidate {
15035 index: 1,
15036 label: 'B',
15037 agent: "beta".to_owned(),
15038 branch: "magi/x/B".to_owned(),
15039 worktree: wt_b.clone(),
15040 summary: String::new(),
15041 stat: String::new(),
15042 files: 0,
15043 commits: 0,
15044 empty: false,
15045 failed: None,
15046 verified_noop: None,
15047 duration_ms: 0,
15048 folded: false,
15049 },
15050 ];
15051 state.tally = Some(Tally {
15052 first_choice: BTreeMap::from([('A', 1)]),
15053 borda: BTreeMap::new(),
15054 winner: 'A',
15055 rankings: 1,
15056 unanimous_initial: true,
15057 deliberated: false,
15058 changed_votes: 0,
15059 unanimous_final: true,
15060 tie_break: None,
15061 judges: 1,
15062 present: 1,
15063 quorum: 1,
15064 met_quorum: true,
15065 uncontested: None,
15066 });
15067 state.status = RunStatus::Ready;
15068
15069 fold_run(&mut state, false, &crate::run::home())
15070 .await
15071 .expect("fold_run");
15072
15073 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15074 assert!(
15075 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15076 "the unmerged winner's branch survives"
15077 );
15078 assert!(
15079 !state.candidates[0].folded,
15080 "the winner is not marked folded"
15081 );
15082
15083 assert!(!wt_b.exists(), "the loser's worktree is removed");
15084 assert!(
15085 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15086 "the loser's branch is removed"
15087 );
15088 assert!(state.candidates[1].folded, "the loser is marked folded");
15089 }
15090
15091 #[tokio::test]
15092 #[cfg(unix)]
15093 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15094 use std::os::fd::AsRawFd as _;
15095 use std::time::Duration;
15096
15097 let tmp = tempfile::tempdir().unwrap();
15098 let repo = tmp.path().join("repo");
15099 std::fs::create_dir_all(&repo).unwrap();
15100 init_repo(&repo);
15101 let common = repo.join(".git");
15102 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15103 let home = tmp.path().join("home");
15104 let mut config = Config::default();
15105 config.coordination = crate::config::test_coordination();
15106 config.graph.worktree_root = Some(tmp.path().join("wt"));
15107 config.graph.timeout_resource_wait = 1;
15108 let mut state = RunState::new(
15109 repo.clone(),
15110 "main".into(),
15111 "deadbeef".into(),
15112 "task".into(),
15113 config,
15114 );
15115 let root = state.worktree_root();
15116 let candidate = root.join("cand-A");
15117 let reviewer = root.join("review-1");
15118 let branch = state.branch_for('A');
15119 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15120 .await
15121 .unwrap();
15122 git::worktree_add_detached(&repo, &reviewer, "main")
15123 .await
15124 .unwrap();
15125 git::acquire_worktree_config(&repo).await.unwrap();
15126 state.enabled_worktree_config = true;
15127 state.status = RunStatus::Ready;
15128 state.candidates.push(Candidate {
15129 index: 0,
15130 label: 'A',
15131 agent: "implement".into(),
15132 branch: branch.clone(),
15133 worktree: candidate.clone(),
15134 summary: String::new(),
15135 stat: String::new(),
15136 files: 0,
15137 commits: 0,
15138 empty: false,
15139 failed: None,
15140 verified_noop: None,
15141 duration_ms: 0,
15142 folded: false,
15143 });
15144 state.save_under(&home).unwrap();
15145 let config_before = std::fs::read(common.join("config")).unwrap();
15146 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15147 .await
15148 .unwrap();
15149 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15152 assert_eq!(
15153 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15154 0
15155 );
15156 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15157 assert!(
15158 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15159 .await
15160 .is_err()
15161 );
15162 assert!(candidate.exists());
15163 assert!(reviewer.exists());
15164 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15165 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15166 assert_eq!(
15167 git::git(&repo, &["worktree", "list", "--porcelain"])
15168 .await
15169 .unwrap(),
15170 worktrees_before
15171 );
15172 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15173 .await
15174 .unwrap()
15175 .unwrap_err();
15176 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15177 drop(fold);
15178 assert!(candidate.exists() && reviewer.exists());
15179 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15180 assert_eq!(
15181 state.resource_block.as_ref().unwrap().kind,
15182 "repository coordination"
15183 );
15184 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15185 drop(external);
15186 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15187 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15188 .await
15189 .unwrap()
15190 .unwrap();
15191 drop(fold);
15192 assert!(removed.contains(&branch));
15193 assert!(!root.exists());
15194 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15195 assert!(state.candidates[0].folded);
15196 assert!(!state.enabled_worktree_config);
15197 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15198 let enabled = git::git_raw(
15199 &repo,
15200 &["config", "--local", "--get", "extensions.worktreeConfig"],
15201 )
15202 .await
15203 .unwrap();
15204 assert!(!enabled.ok(), "the last config reference was released");
15205 }
15206
15207 #[tokio::test]
15210 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15211 let tmp = tempfile::tempdir().expect("tempdir");
15212 let repo = tmp.path().join("repo");
15213 std::fs::create_dir_all(&repo).unwrap();
15214 init_repo(&repo);
15215 let home = tmp.path().join("home");
15216
15217 let mut config = Config::default();
15218 config.graph.worktree_root = Some(tmp.path().join("wt"));
15219 let mut state = RunState::new(
15220 repo.clone(),
15221 "main".to_owned(),
15222 "deadbeef".to_owned(),
15223 "task".to_owned(),
15224 config,
15225 );
15226 git::git(&repo, &["branch", "magi/x/A", "main"])
15228 .await
15229 .expect("branch");
15230 state.candidates = vec![Candidate {
15231 index: 0,
15232 label: 'A',
15233 agent: "alpha".to_owned(),
15234 branch: "magi/x/A".to_owned(),
15235 worktree: state.worktree_root().join("cand-A"),
15236 summary: String::new(),
15237 stat: String::new(),
15238 files: 0,
15239 commits: 0,
15240 empty: false,
15241 failed: None,
15242 verified_noop: None,
15243 duration_ms: 0,
15244 folded: true,
15245 }];
15246 state.released_to = Some("20260901-000000-new1".to_owned());
15247 state.released_branches = vec!["magi/x/A".to_owned()];
15248
15249 fold_run(&mut state, true, &home).await.expect("fold_run");
15250 state.status = RunStatus::Merged;
15251 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15252 state.merge = Some(crate::run::MergeOutcome {
15253 mode: MergeMode::Pr,
15254 ok: true,
15255 detail: "confirmed merged".into(),
15256 empty: false,
15257 });
15258 fold_run(&mut state, true, &home)
15259 .await
15260 .expect("merged fold");
15261
15262 assert!(
15263 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15264 "the handed-over branch survives a fold"
15265 );
15266 }
15267
15268 #[tokio::test]
15272 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15273 let tmp = tempfile::tempdir().expect("tempdir");
15274 let repo = tmp.path().join("repo");
15275 std::fs::create_dir_all(&repo).unwrap();
15276 init_repo(&repo);
15277 let run = |args: &[&str]| {
15278 let out = std::process::Command::new("git")
15279 .quiet()
15280 .args(args)
15281 .current_dir(&repo)
15282 .output()
15283 .expect("spawn git");
15284 assert!(out.status.success(), "git {args:?}");
15285 };
15286 run(&["branch", "magi/x/A"]);
15287 run(&["checkout", "-q", "-b", "magi/x/B"]);
15288 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15289 run(&["add", "-A"]);
15290 run(&["commit", "-q", "-m", "work"]);
15291 run(&["checkout", "-q", "main"]);
15292
15293 let probe = RunState::new(
15297 repo.clone(),
15298 "main".to_owned(),
15299 "deadbeef".to_owned(),
15300 "task".to_owned(),
15301 Config::default(),
15302 );
15303 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15304 let bare = tmp.path().join("origin.git");
15305 let out = std::process::Command::new("git")
15306 .quiet()
15307 .args(["init", "-q", "--bare"])
15308 .arg(&bare)
15309 .output()
15310 .expect("spawn git");
15311 assert!(out.status.success(), "git init --bare");
15312 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15313 run(&["push", "-q", "origin", "main"]);
15314
15315 let mut state = RunState::new(
15316 repo.clone(),
15317 "main".to_owned(),
15318 "deadbeef".to_owned(),
15319 "task".to_owned(),
15320 Config::default(),
15321 );
15322 state.seeds = vec![refs::Seed {
15323 token: "magi/27b2/A".to_owned(),
15324 kind: refs::SeedKind::Unresolved,
15325 sha: String::new(),
15326 branch: true,
15327 detail: "no branch or commit named magi/27b2/A".to_owned(),
15328 }];
15329
15330 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15331 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15332 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15333 let detail = empty_candidate_detail(&state, "main");
15334 assert!(detail.starts_with("empty candidate"), "{detail}");
15335 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15336 assert!(detail.contains("--start-from"), "{detail}");
15337 }
15338
15339 #[tokio::test]
15348 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15349 let tmp = tempfile::tempdir().expect("tempdir");
15350 let repo = tmp.path().join("repo");
15351 std::fs::create_dir_all(&repo).unwrap();
15352 init_repo(&repo);
15353
15354 let mut config = Config::default();
15355 config.merge.mode = MergeMode::Local;
15356
15357 let mut state = RunState::new(
15358 repo.clone(),
15359 "main".to_owned(),
15360 "deadbeef".to_owned(),
15361 "task".to_owned(),
15362 config,
15363 );
15364 state.candidates = vec![Candidate {
15365 index: 0,
15366 label: 'A',
15367 agent: "alpha".to_owned(),
15368 branch: "does-not-exist".to_owned(),
15369 worktree: repo.clone(),
15370 summary: String::new(),
15371 stat: String::new(),
15372 files: 0,
15373 commits: 0,
15374 empty: false,
15375 failed: None,
15376 verified_noop: None,
15377 duration_ms: 0,
15378 folded: false,
15379 }];
15380 state.tally = Some(Tally {
15381 first_choice: BTreeMap::from([('A', 1)]),
15382 borda: BTreeMap::new(),
15383 winner: 'A',
15384 rankings: 1,
15385 unanimous_initial: true,
15386 deliberated: false,
15387 changed_votes: 0,
15388 unanimous_final: true,
15389 tie_break: None,
15390 judges: 0,
15391 present: 0,
15392 quorum: 0,
15393 met_quorum: true,
15394 uncontested: Some("only candidate A produced a change".to_owned()),
15395 });
15396 state.reviews = vec![ReviewRound {
15397 round: 1,
15398 head: "deadbeef".to_owned(),
15399 verified_head: None,
15400 verified_at: None,
15401 reviews: Vec::new(),
15402 e2e: Vec::new(),
15403 fix: None,
15404 blocking: 0,
15405 answered: 0,
15406 expected: 0,
15407 clean: true,
15408 verify_retried: false,
15409 e2e_deferred: false,
15410 e2e_defer_reason: None,
15411 progressed: false,
15412 vote_split: false,
15413 reconsideration: Vec::new(),
15414 verdict: None,
15415 }];
15416 state.gate = vec![CommandOutcome {
15417 command: "test".to_owned(),
15418 code: Some(0),
15419 output_tail: String::new(),
15420 duration_ms: 0,
15421 resource_blocked: false,
15422 }];
15423 state.gate_ran = true;
15424 state.status = RunStatus::Ready;
15429 state.merge = Some(MergeOutcome {
15430 mode: MergeMode::Local,
15431 ok: false,
15432 detail: "already concluded".to_owned(),
15433 empty: false,
15434 });
15435
15436 let mut runner = Runner {
15437 state,
15438 roles: ResolvedRoles {
15439 implementers: Vec::new(),
15440 judges: Vec::new(),
15441 reviewers: Vec::new(),
15442 fixer: None,
15443 conductor: conductor(),
15444 implementer_roster: Vec::new(),
15445 judge_roster: Vec::new(),
15446 reviewer_roster: Vec::new(),
15447 },
15448 sem: Arc::new(Semaphore::new(1)),
15449 pause: Pause::new(),
15450 interrupt: Pause::new(),
15451 };
15452
15453 runner.merge().await.expect("merge");
15454
15455 assert_eq!(
15456 runner.state.status,
15457 RunStatus::Ready,
15458 "a concluded run's status must not change on reentry"
15459 );
15460 assert_eq!(
15461 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15462 Some("already concluded"),
15463 "merge must not run again once the node already recorded an outcome"
15464 );
15465 }
15466
15467 #[tokio::test]
15476 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15477 let tmp = tempfile::tempdir().expect("tempdir");
15478 let repo = tmp.path().join("repo");
15479 std::fs::create_dir_all(&repo).unwrap();
15480 init_repo(&repo);
15481
15482 let mut config = Config::default();
15483 config.merge.mode = MergeMode::Local;
15484
15485 let mut state = RunState::new(
15486 repo.clone(),
15487 "main".to_owned(),
15488 "deadbeef".to_owned(),
15489 "task".to_owned(),
15490 config,
15491 );
15492 state.candidates = vec![Candidate {
15493 index: 0,
15494 label: 'A',
15495 agent: "alpha".to_owned(),
15496 branch: "does-not-exist".to_owned(),
15497 worktree: repo.clone(),
15498 summary: String::new(),
15499 stat: String::new(),
15500 files: 0,
15501 commits: 0,
15502 empty: false,
15503 failed: None,
15504 verified_noop: None,
15505 duration_ms: 0,
15506 folded: false,
15507 }];
15508 state.tally = Some(Tally {
15509 first_choice: BTreeMap::from([('A', 1)]),
15510 borda: BTreeMap::new(),
15511 winner: 'A',
15512 rankings: 1,
15513 unanimous_initial: true,
15514 deliberated: false,
15515 changed_votes: 0,
15516 unanimous_final: true,
15517 tie_break: None,
15518 judges: 0,
15519 present: 0,
15520 quorum: 0,
15521 met_quorum: true,
15522 uncontested: Some("only candidate A produced a change".to_owned()),
15523 });
15524 state.reviews = vec![ReviewRound {
15525 round: 1,
15526 head: "deadbeef".to_owned(),
15527 verified_head: None,
15528 verified_at: None,
15529 reviews: Vec::new(),
15530 e2e: Vec::new(),
15531 fix: None,
15532 blocking: 0,
15533 answered: 0,
15534 expected: 0,
15535 clean: true,
15536 verify_retried: false,
15537 e2e_deferred: false,
15538 e2e_defer_reason: None,
15539 progressed: false,
15540 vote_split: false,
15541 reconsideration: Vec::new(),
15542 verdict: None,
15543 }];
15544 state.gate = Vec::new();
15546 state.gate_ran = false;
15547 state.status = RunStatus::Gating;
15548
15549 let mut runner = Runner {
15550 state,
15551 roles: ResolvedRoles {
15552 implementers: Vec::new(),
15553 judges: Vec::new(),
15554 reviewers: Vec::new(),
15555 fixer: None,
15556 conductor: conductor(),
15557 implementer_roster: Vec::new(),
15558 judge_roster: Vec::new(),
15559 reviewer_roster: Vec::new(),
15560 },
15561 sem: Arc::new(Semaphore::new(1)),
15562 pause: Pause::new(),
15563 interrupt: Pause::new(),
15564 };
15565
15566 runner.merge().await.expect("merge");
15567
15568 assert!(
15569 runner.state.merge.is_none(),
15570 "an empty gate must never be read as a passing one: {:?}",
15571 runner.state.merge
15572 );
15573 }
15574
15575 #[tokio::test]
15582 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15583 let tmp = tempfile::tempdir().expect("tempdir");
15584 let repo = tmp.path().join("repo");
15585 std::fs::create_dir_all(&repo).unwrap();
15586 init_repo(&repo);
15587
15588 let config = Config::default();
15590
15591 let mut state = RunState::new(
15592 repo.clone(),
15593 "main".to_owned(),
15594 "deadbeef".to_owned(),
15595 "task".to_owned(),
15596 config,
15597 );
15598 state.candidates = vec![Candidate {
15599 index: 0,
15600 label: 'A',
15601 agent: "alpha".to_owned(),
15602 branch: "does-not-exist".to_owned(),
15603 worktree: repo.clone(),
15604 summary: String::new(),
15605 stat: String::new(),
15606 files: 0,
15607 commits: 0,
15608 empty: false,
15609 failed: None,
15610 verified_noop: None,
15611 duration_ms: 0,
15612 folded: false,
15613 }];
15614 state.tally = Some(Tally {
15615 first_choice: BTreeMap::from([('A', 1)]),
15616 borda: BTreeMap::new(),
15617 winner: 'A',
15618 rankings: 1,
15619 unanimous_initial: true,
15620 deliberated: false,
15621 changed_votes: 0,
15622 unanimous_final: true,
15623 tie_break: None,
15624 judges: 0,
15625 present: 0,
15626 quorum: 0,
15627 met_quorum: true,
15628 uncontested: Some("only candidate A produced a change".to_owned()),
15629 });
15630 state.reviews = vec![ReviewRound {
15631 round: 1,
15632 head: "deadbeef".to_owned(),
15633 verified_head: None,
15634 verified_at: None,
15635 reviews: Vec::new(),
15636 e2e: Vec::new(),
15637 fix: None,
15638 blocking: 0,
15639 answered: 0,
15640 expected: 0,
15641 clean: true,
15642 verify_retried: false,
15643 e2e_deferred: false,
15644 e2e_defer_reason: None,
15645 progressed: false,
15646 vote_split: false,
15647 reconsideration: Vec::new(),
15648 verdict: None,
15649 }];
15650
15651 let mut runner = Runner {
15652 state,
15653 roles: ResolvedRoles {
15654 implementers: Vec::new(),
15655 judges: Vec::new(),
15656 reviewers: Vec::new(),
15657 fixer: None,
15658 conductor: conductor(),
15659 implementer_roster: Vec::new(),
15660 judge_roster: Vec::new(),
15661 reviewer_roster: Vec::new(),
15662 },
15663 sem: Arc::new(Semaphore::new(1)),
15664 pause: Pause::new(),
15665 interrupt: Pause::new(),
15666 };
15667
15668 runner.gate().await.expect("gate");
15669 assert!(
15670 runner.state.gate_ran,
15671 "zero configured commands is still a real attempt, not an unrun gate"
15672 );
15673 assert!(runner.state.gate.is_empty());
15674 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15675 assert_ne!(
15676 runner.state.status,
15677 RunStatus::Blocked,
15678 "a gate with nothing to check must not read as failed"
15679 );
15680
15681 runner.merge().await.expect("merge");
15682 assert_eq!(
15683 runner.state.status,
15684 RunStatus::Ready,
15685 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15686 );
15687 }
15688
15689 #[tokio::test]
15700 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15701 crate::run::pin_test_home();
15702 let home = crate::run::home();
15703
15704 let tmp = tempfile::tempdir().expect("tempdir");
15705 let repo = tmp.path().join("repo");
15706 std::fs::create_dir_all(&repo).unwrap();
15707 init_repo(&repo);
15708 let cache_dir = tmp.path().join("target");
15711
15712 let mut config = Config::default();
15713 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15714 config.graph.timeout_verify = Some(2);
15717 config.graph.timeout_resource_wait = 2;
15718
15719 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15720 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15721 .expect("no io error acquiring directly")
15722 {
15723 crate::cache::AcquireOutcome::Acquired(g) => g,
15724 crate::cache::AcquireOutcome::Busy(b) => {
15725 panic!("expected the direct acquire to win the lease first: {b:?}")
15726 }
15727 };
15728
15729 let mut state = RunState::new(
15730 repo.clone(),
15731 "main".to_owned(),
15732 "deadbeef".to_owned(),
15733 "task".to_owned(),
15734 config,
15735 );
15736 state.candidates = vec![Candidate {
15737 index: 0,
15738 label: 'A',
15739 agent: "alpha".to_owned(),
15740 branch: "does-not-exist".to_owned(),
15741 worktree: repo.clone(),
15742 summary: String::new(),
15743 stat: String::new(),
15744 files: 0,
15745 commits: 0,
15746 empty: false,
15747 failed: None,
15748 verified_noop: None,
15749 duration_ms: 0,
15750 folded: false,
15751 }];
15752 state.tally = Some(Tally {
15753 first_choice: BTreeMap::from([('A', 1)]),
15754 borda: BTreeMap::new(),
15755 winner: 'A',
15756 rankings: 1,
15757 unanimous_initial: true,
15758 deliberated: false,
15759 changed_votes: 0,
15760 unanimous_final: true,
15761 tie_break: None,
15762 judges: 0,
15763 present: 0,
15764 quorum: 0,
15765 met_quorum: true,
15766 uncontested: Some("only candidate A produced a change".to_owned()),
15767 });
15768 state.reviews = vec![ReviewRound {
15769 round: 1,
15770 head: "deadbeef".to_owned(),
15771 verified_head: None,
15772 verified_at: None,
15773 reviews: Vec::new(),
15774 e2e: Vec::new(),
15775 fix: None,
15776 blocking: 0,
15777 answered: 0,
15778 expected: 0,
15779 clean: true,
15780 verify_retried: false,
15781 e2e_deferred: false,
15782 e2e_defer_reason: None,
15783 progressed: false,
15784 vote_split: false,
15785 reconsideration: Vec::new(),
15786 verdict: None,
15787 }];
15788
15789 let mut runner = Runner {
15790 state,
15791 roles: ResolvedRoles {
15792 implementers: Vec::new(),
15793 judges: Vec::new(),
15794 reviewers: Vec::new(),
15795 fixer: None,
15796 conductor: conductor(),
15797 implementer_roster: Vec::new(),
15798 judge_roster: Vec::new(),
15799 reviewer_roster: Vec::new(),
15800 },
15801 sem: Arc::new(Semaphore::new(1)),
15802 pause: Pause::new(),
15803 interrupt: Pause::new(),
15804 };
15805
15806 let started = std::time::Instant::now();
15807 runner.gate().await.expect("gate");
15808 assert!(
15809 started.elapsed() < Duration::from_secs(1),
15810 "a gate with nothing to run must never wait on a lease it never needed"
15811 );
15812 assert!(
15813 runner.state.gate_ran,
15814 "zero commands is still a real, immediate attempt"
15815 );
15816 assert!(runner.state.gate.is_empty());
15817 assert_ne!(
15818 runner.state.status,
15819 RunStatus::Blocked,
15820 "must not read as resource-blocked on a lease it never asked for"
15821 );
15822 }
15823
15824 #[cfg(unix)]
15825 #[tokio::test]
15826 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15827 crate::run::pin_test_home();
15828 let tmp = tempfile::tempdir().unwrap();
15829 let mut config = Config::default();
15830 config.disk.min_free_bytes = 0;
15831 let mut state = RunState::new(
15832 tmp.path().to_owned(),
15833 "main".into(),
15834 "base".into(),
15835 "verification evidence".into(),
15836 config,
15837 );
15838 let (outcomes, pids) = run_commands(
15839 &mut state,
15840 "gate",
15841 "gate",
15842 1,
15843 &["sh".into(), "-c".into()],
15844 &[
15845 "printf before-timeout; exec sleep 30".into(),
15846 "kill -TERM $$".into(),
15847 "exit 7".into(),
15848 "printf success".into(),
15849 ],
15850 tmp.path(),
15851 Duration::from_secs(1),
15852 )
15853 .await;
15854 wait_for_timed_out_children_to_die(&pids).await;
15855 assert_eq!(
15856 pids.len(),
15857 1,
15858 "unleased timeout retains its death-check PID"
15859 );
15860 assert_eq!(outcomes.len(), 4);
15861 assert_eq!(outcomes[0].code, None);
15862 assert!(outcomes[0].output_tail.contains("before-timeout"));
15863 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15864 assert_eq!(outcomes[1].code, None);
15865 assert!(outcomes[1].output_tail.contains("command terminated:"));
15866 assert!(!outcomes[1].output_tail.contains("timed out"));
15867 assert_eq!(outcomes[2].code, Some(7));
15868 assert!(!outcomes[2].output_tail.contains("timed out"));
15869 assert_eq!(outcomes[3].code, Some(0));
15870 assert_eq!(outcomes[3].output_tail, "success");
15871 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15872 }
15873
15874 #[tokio::test]
15884 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15885 crate::run::pin_test_home();
15886
15887 let tmp = tempfile::tempdir().expect("tempdir");
15888 let repo = tmp.path().join("repo");
15889 std::fs::create_dir_all(&repo).unwrap();
15890 init_repo(&repo);
15891
15892 let mut config = Config::default();
15893 config.verify.gate = vec![
15894 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15895 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15896 .to_owned(),
15897 ];
15898
15899 let mut state = RunState::new(
15900 repo.clone(),
15901 "main".to_owned(),
15902 "deadbeef".to_owned(),
15903 "task".to_owned(),
15904 config,
15905 );
15906 let run_id = state.id.clone();
15907 state.candidates = vec![Candidate {
15908 index: 0,
15909 label: 'A',
15910 agent: "alpha".to_owned(),
15911 branch: "does-not-exist".to_owned(),
15912 worktree: repo.clone(),
15913 summary: String::new(),
15914 stat: String::new(),
15915 files: 0,
15916 commits: 0,
15917 empty: false,
15918 failed: None,
15919 verified_noop: None,
15920 duration_ms: 0,
15921 folded: false,
15922 }];
15923 state.tally = Some(Tally {
15924 first_choice: BTreeMap::from([('A', 1)]),
15925 borda: BTreeMap::new(),
15926 winner: 'A',
15927 rankings: 1,
15928 unanimous_initial: true,
15929 deliberated: false,
15930 changed_votes: 0,
15931 unanimous_final: true,
15932 tie_break: None,
15933 judges: 0,
15934 present: 0,
15935 quorum: 0,
15936 met_quorum: true,
15937 uncontested: Some("only candidate A produced a change".to_owned()),
15938 });
15939 state.reviews = vec![ReviewRound {
15940 round: 1,
15941 head: "deadbeef".to_owned(),
15942 verified_head: None,
15943 verified_at: None,
15944 reviews: Vec::new(),
15945 e2e: Vec::new(),
15946 fix: None,
15947 blocking: 0,
15948 answered: 0,
15949 expected: 0,
15950 clean: true,
15951 verify_retried: false,
15952 e2e_deferred: false,
15953 e2e_defer_reason: None,
15954 progressed: false,
15955 vote_split: false,
15956 reconsideration: Vec::new(),
15957 verdict: None,
15958 }];
15959
15960 let mut runner = Runner {
15961 state,
15962 roles: ResolvedRoles {
15963 implementers: Vec::new(),
15964 judges: Vec::new(),
15965 reviewers: Vec::new(),
15966 fixer: None,
15967 conductor: conductor(),
15968 implementer_roster: Vec::new(),
15969 judge_roster: Vec::new(),
15970 reviewer_roster: Vec::new(),
15971 },
15972 sem: Arc::new(Semaphore::new(1)),
15973 pause: Pause::new(),
15974 interrupt: Pause::new(),
15975 };
15976
15977 let started_marker = repo.join("started.marker");
15978 let release_marker = repo.join("release.marker");
15979 let poller = tokio::spawn(async move {
15980 for _ in 0..100 {
15985 if started_marker.exists()
15986 && let Ok(s) = crate::run::RunState::load(&run_id)
15987 && let Some(a) = s.active.get("gate")
15988 {
15989 std::fs::write(&release_marker, b"go").expect("release marker");
15990 return Some(a.clone());
15991 }
15992 tokio::time::sleep(Duration::from_millis(50)).await;
15993 }
15994 None
15995 });
15996
15997 runner.gate().await.expect("gate");
15998 let captured = poller.await.expect("poller task");
15999 let captured = captured.expect(
16000 "the poller never saw a `gate` task entry in run.json while the command was \
16001 still blocked on its own release marker",
16002 );
16003
16004 assert_eq!(captured.task.as_deref(), Some("gate"));
16005 assert_eq!(captured.node, "gate");
16006 assert_eq!(captured.index, Some(1));
16007 assert_eq!(captured.total, Some(1));
16008 assert!(
16009 captured
16010 .command
16011 .as_deref()
16012 .is_some_and(|c| c.contains("started.marker")),
16013 "{captured:?}"
16014 );
16015
16016 assert!(
16017 runner.state.active.is_empty(),
16018 "the entry must be cleared once the command actually finished: {:?}",
16019 runner.state.active
16020 );
16021 assert!(runner.state.gate_ran);
16022 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16023 }
16024
16025 #[tokio::test]
16028 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16029 use crate::verdict::{Finding, ReviewVote, Severity};
16030 crate::run::pin_test_home();
16031 let tmp = tempfile::tempdir().expect("tempdir");
16032 let repo = tmp.path().join("repo");
16033 std::fs::create_dir_all(&repo).unwrap();
16034 init_repo(&repo);
16035
16036 for (severity, vote, expect) in [
16037 (Severity::Major, ReviewVote::Reject, true),
16038 (Severity::Minor, ReviewVote::Reject, false),
16039 (Severity::Major, ReviewVote::Approve, false),
16040 ] {
16041 let mut round = review_round(false, 1, 1, 1, false, true);
16042 round.reviews = vec![ReviewRecord {
16043 reviewer: 1,
16044 agent: "alpha".to_owned(),
16045 summary: String::new(),
16046 findings: vec![Finding {
16047 id: "R1-1-1".to_owned(),
16048 severity,
16049 file: None,
16050 line: None,
16051 title: "t".to_owned(),
16052 detail: String::new(),
16053 }],
16054 vote: Some(vote),
16055 failed: None,
16056 duration_ms: 0,
16057 attempts: 0,
16058 }];
16059 let mut state = RunState::new(
16060 repo.clone(),
16061 "main".to_owned(),
16062 "deadbeef".to_owned(),
16063 "task".to_owned(),
16064 Config::default(),
16065 );
16066 state.reviews = vec![round];
16067 let mut runner = Runner {
16068 state,
16069 roles: ResolvedRoles {
16070 implementers: Vec::new(),
16071 judges: Vec::new(),
16072 reviewers: Vec::new(),
16073 fixer: None,
16074 conductor: conductor(),
16075 implementer_roster: Vec::new(),
16076 judge_roster: Vec::new(),
16077 reviewer_roster: Vec::new(),
16078 },
16079 sem: Arc::new(Semaphore::new(1)),
16080 pause: Pause::new(),
16081 interrupt: Pause::new(),
16082 };
16083 let shell = runner.state.config.shell();
16084 runner
16085 .stop_reviewing("round budget spent", &shell, &repo)
16086 .await
16087 .expect("stop_reviewing");
16088 assert_eq!(runner.state.status, RunStatus::Gating);
16089 assert_eq!(
16090 runner.state.contested_handoff.is_some(),
16091 expect,
16092 "{severity:?} + {vote:?}"
16093 );
16094 }
16095 }
16096
16097 #[tokio::test]
16110 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16111 crate::run::pin_test_home();
16112 let home = crate::run::home();
16113
16114 let tmp = tempfile::tempdir().expect("tempdir");
16115 let repo = tmp.path().join("repo");
16116 std::fs::create_dir_all(&repo).unwrap();
16117 init_repo(&repo);
16118 let head = crate::git::rev_parse(&repo, "HEAD")
16119 .await
16120 .expect("rev-parse");
16121 let cache_dir = tmp.path().join("target");
16124
16125 let mut config = Config::default();
16126 config.verify.e2e = vec![format!(
16127 "CARGO_TARGET_DIR='{}' test -f README.md",
16128 cache_dir.display()
16129 )];
16130 config.graph.review_rounds = 1;
16131 config.graph.timeout_verify = Some(2);
16134 config.graph.timeout_resource_wait = 2;
16135
16136 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16137 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16138 .expect("no io error acquiring directly")
16139 {
16140 crate::cache::AcquireOutcome::Acquired(g) => g,
16141 crate::cache::AcquireOutcome::Busy(b) => {
16142 panic!("expected the direct acquire to win the lease first: {b:?}")
16143 }
16144 };
16145
16146 let mut state = RunState::new(
16147 repo.clone(),
16148 "main".to_owned(),
16149 head.clone(),
16150 "task".to_owned(),
16151 config,
16152 );
16153 state.candidates = vec![Candidate {
16154 index: 0,
16155 label: 'A',
16156 agent: "alpha".to_owned(),
16157 branch: "does-not-exist".to_owned(),
16158 worktree: repo.clone(),
16159 summary: String::new(),
16160 stat: String::new(),
16161 files: 0,
16162 commits: 0,
16163 empty: false,
16164 failed: None,
16165 verified_noop: None,
16166 duration_ms: 0,
16167 folded: false,
16168 }];
16169 state.tally = Some(Tally {
16170 first_choice: BTreeMap::from([('A', 1)]),
16171 borda: BTreeMap::new(),
16172 winner: 'A',
16173 rankings: 1,
16174 unanimous_initial: true,
16175 deliberated: false,
16176 changed_votes: 0,
16177 unanimous_final: true,
16178 tie_break: None,
16179 judges: 0,
16180 present: 0,
16181 quorum: 0,
16182 met_quorum: true,
16183 uncontested: Some("only candidate A produced a change".to_owned()),
16184 });
16185 state.reviews = vec![ReviewRound {
16189 round: 1,
16190 head: head.clone(),
16191 verified_head: None,
16192 verified_at: None,
16193 reviews: Vec::new(),
16194 e2e: Vec::new(),
16195 fix: None,
16196 blocking: 1,
16197 answered: 1,
16198 expected: 1,
16199 clean: false,
16200 verify_retried: false,
16201 e2e_deferred: true,
16202 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16203 progressed: false,
16204 vote_split: false,
16205 reconsideration: Vec::new(),
16206 verdict: None,
16207 }];
16208
16209 let mut runner = Runner {
16210 state,
16211 roles: ResolvedRoles {
16212 implementers: Vec::new(),
16213 judges: Vec::new(),
16214 reviewers: Vec::new(),
16215 fixer: None,
16216 conductor: conductor(),
16217 implementer_roster: Vec::new(),
16218 judge_roster: Vec::new(),
16219 reviewer_roster: Vec::new(),
16220 },
16221 sem: Arc::new(Semaphore::new(1)),
16222 pause: Pause::new(),
16223 interrupt: Pause::new(),
16224 };
16225
16226 let shell = runner.state.config.shell();
16227 runner
16228 .stop_reviewing("round budget spent", &shell, &repo)
16229 .await
16230 .expect("stop_reviewing");
16231
16232 let last = runner.state.reviews.last().expect("round record");
16233 assert_eq!(
16234 last.e2e_status(),
16235 E2eStatus::ResourceBlocked,
16236 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16237 failed: {last:?}"
16238 );
16239 assert_eq!(
16240 last.verified_head.as_deref(),
16241 Some(head.as_str()),
16242 "which commit this attempt targeted is known even though nothing finished checking \
16243 it"
16244 );
16245 let first_attempt_at = last
16246 .verified_at
16247 .expect("when this attempt ran is known too");
16248 assert_ne!(
16249 runner.state.status,
16250 RunStatus::Blocked,
16251 "contention is evidence about the machine, not the patch — it must not settle the \
16252 run as blocked: {:?}",
16253 runner.state.status
16254 );
16255 assert!(
16256 !runner
16257 .state
16258 .events
16259 .iter()
16260 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16261 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16262 runner.state.events
16263 );
16264
16265 runner
16270 .stop_reviewing("round budget spent", &shell, &repo)
16271 .await
16272 .expect("stop_reviewing retry");
16273 assert_eq!(
16274 runner.state.reviews.len(),
16275 1,
16276 "no new round was started: {:?}",
16277 runner.state.reviews
16278 );
16279 let last = runner.state.reviews.last().expect("round record");
16280 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16281 assert!(
16282 last.verified_at.expect("still known") > first_attempt_at,
16283 "a second reentry must be a fresh attempt, not a stale copy of the first"
16284 );
16285 assert_ne!(runner.state.status, RunStatus::Blocked);
16286
16287 held.release();
16288 }
16289
16290 #[tokio::test]
16302 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16303 crate::run::pin_test_home();
16304 let home = crate::run::home();
16305
16306 let tmp = tempfile::tempdir().expect("tempdir");
16307 let repo = tmp.path().join("repo");
16308 std::fs::create_dir_all(&repo).unwrap();
16309 init_repo(&repo);
16310 let head = crate::git::rev_parse(&repo, "HEAD")
16311 .await
16312 .expect("rev-parse");
16313 let cache_dir = tmp.path().join("target");
16314
16315 let mut config = Config::default();
16316 config.verify.e2e = vec![format!(
16317 "CARGO_TARGET_DIR='{}' test -f README.md",
16318 cache_dir.display()
16319 )];
16320 config.graph.review_rounds = 1;
16321 config.graph.timeout_verify = Some(2);
16322 config.graph.timeout_resource_wait = 2;
16323
16324 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16325 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16326 .expect("no io error acquiring directly")
16327 {
16328 crate::cache::AcquireOutcome::Acquired(g) => g,
16329 crate::cache::AcquireOutcome::Busy(b) => {
16330 panic!("expected the direct acquire to win the lease first: {b:?}")
16331 }
16332 };
16333
16334 let mut state = RunState::new(
16335 repo.clone(),
16336 "main".to_owned(),
16337 head.clone(),
16338 "task".to_owned(),
16339 config,
16340 );
16341 state.candidates = vec![Candidate {
16342 index: 0,
16343 label: 'A',
16344 agent: "alpha".to_owned(),
16345 branch: "does-not-exist".to_owned(),
16346 worktree: repo.clone(),
16347 summary: String::new(),
16348 stat: String::new(),
16349 files: 0,
16350 commits: 0,
16351 empty: false,
16352 failed: None,
16353 verified_noop: None,
16354 duration_ms: 0,
16355 folded: false,
16356 }];
16357 state.tally = Some(Tally {
16358 first_choice: BTreeMap::from([('A', 1)]),
16359 borda: BTreeMap::new(),
16360 winner: 'A',
16361 rankings: 1,
16362 unanimous_initial: true,
16363 deliberated: false,
16364 changed_votes: 0,
16365 unanimous_final: true,
16366 tie_break: None,
16367 judges: 0,
16368 present: 0,
16369 quorum: 0,
16370 met_quorum: true,
16371 uncontested: Some("only candidate A produced a change".to_owned()),
16372 });
16373 state.reviews = vec![ReviewRound {
16377 round: 1,
16378 head: head.clone(),
16379 verified_head: Some(head.clone()),
16380 verified_at: Some(jiff::Timestamp::now()),
16381 reviews: Vec::new(),
16382 e2e: vec![CommandOutcome {
16383 command: format!(
16384 "CARGO_TARGET_DIR='{}' test -f README.md",
16385 cache_dir.display()
16386 ),
16387 code: None,
16388 output_tail: "waiting for the shared build cache".to_owned(),
16389 duration_ms: 0,
16390 resource_blocked: true,
16391 }],
16392 fix: None,
16393 blocking: 1,
16394 answered: 1,
16395 expected: 1,
16396 clean: false,
16397 verify_retried: false,
16398 e2e_deferred: false,
16399 e2e_defer_reason: None,
16400 progressed: false,
16401 vote_split: false,
16402 reconsideration: Vec::new(),
16403 verdict: None,
16404 }];
16405
16406 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16407 let mut runner = Runner {
16408 state,
16409 roles: ResolvedRoles {
16410 implementers: Vec::new(),
16411 judges: Vec::new(),
16412 reviewers: Vec::new(),
16413 fixer: None,
16414 conductor: conductor(),
16415 implementer_roster: Vec::new(),
16416 judge_roster: Vec::new(),
16417 reviewer_roster: Vec::new(),
16418 },
16419 sem: Arc::new(Semaphore::new(1)),
16420 pause: Pause::new(),
16421 interrupt: Pause::new(),
16422 };
16423
16424 runner.review_loop().await.expect("review_loop");
16429
16430 assert_eq!(
16431 runner.state.reviews.len(),
16432 1,
16433 "no new round was started on top of the unresolved one: {:?}",
16434 runner.state.reviews
16435 );
16436 let last = &runner.state.reviews[0];
16437 assert_eq!(
16438 last.e2e_status(),
16439 E2eStatus::ResourceBlocked,
16440 "still contended: {last:?}"
16441 );
16442 assert!(
16443 last.verified_at.expect("still known") > first_attempt_at,
16444 "review_loop must have actually retried the check, not left it exactly as found"
16445 );
16446 assert_ne!(
16447 runner.state.status,
16448 RunStatus::Blocked,
16449 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16450 runner.state.status
16451 );
16452
16453 held.release();
16454 }
16455
16456 #[tokio::test]
16457 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16458 crate::run::pin_test_home();
16459 for status in [RunStatus::Blocked, RunStatus::Gating] {
16460 let mut runner = runner_at(status);
16461 runner.state.config.graph.land = true;
16462 runner.state.gate_ran = true;
16463 runner.state.merge = Some(MergeOutcome {
16464 mode: MergeMode::Pr,
16465 ok: false,
16466 detail: "the fixer produced no commit".into(),
16467 empty: false,
16468 });
16469 let url = "https://example.invalid/o/r/pull/1";
16470 runner.state.pr = Some(crate::run::PrRecord {
16471 url: url.into(),
16472 number: 1,
16473 state: "open".into(),
16474 checks: "red".into(),
16475 round: 1,
16476 rounds: 1,
16477 red_at_merge: Vec::new(),
16478 });
16479 runner.state.save().unwrap();
16480 runner.execute_graph().await.unwrap();
16484 assert_eq!(runner.state.status, RunStatus::Blocked);
16485 assert!(runner.state.land_sync_pending);
16486 assert!(runner.state.events.iter().any(|event|
16487 event.node == "land" && event.message == format!("watching {url}")));
16488 assert_eq!(
16489 runner.state.merge.as_ref().unwrap().detail,
16490 "the fixer produced no commit"
16491 );
16492 }
16493 }
16494
16495 #[tokio::test]
16496 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16497 crate::run::pin_test_home();
16498 let tmp = tempfile::tempdir().expect("tempdir");
16499 let repo = tmp.path().join("repo");
16500 std::fs::create_dir_all(&repo).unwrap();
16501 init_repo(&repo);
16502
16503 let mut config = Config::default();
16504 config.merge.mode = MergeMode::Pr;
16505 config.graph.land = true;
16506 config.graph.land_approval = false;
16507
16508 let mut state = RunState::new(
16509 repo.clone(),
16510 "main".to_owned(),
16511 "deadbeef".to_owned(),
16512 "task".to_owned(),
16513 config,
16514 );
16515 state.candidates = vec![Candidate {
16516 index: 0,
16517 label: 'A',
16518 agent: "alpha".to_owned(),
16519 branch: "does-not-exist".to_owned(),
16520 worktree: repo.clone(),
16521 summary: String::new(),
16522 stat: String::new(),
16523 files: 0,
16524 commits: 0,
16525 empty: false,
16526 failed: None,
16527 verified_noop: None,
16528 duration_ms: 0,
16529 folded: false,
16530 }];
16531 state.tally = Some(Tally {
16532 first_choice: BTreeMap::from([('A', 1)]),
16533 borda: BTreeMap::new(),
16534 winner: 'A',
16535 rankings: 1,
16536 unanimous_initial: true,
16537 deliberated: false,
16538 changed_votes: 0,
16539 unanimous_final: true,
16540 tie_break: None,
16541 judges: 0,
16542 present: 0,
16543 quorum: 0,
16544 met_quorum: true,
16545 uncontested: Some("only candidate A produced a change".to_owned()),
16546 });
16547 state.reviews = vec![ReviewRound {
16548 round: 1,
16549 head: "deadbeef".to_owned(),
16550 verified_head: None,
16551 verified_at: None,
16552 reviews: Vec::new(),
16553 e2e: Vec::new(),
16554 fix: None,
16555 blocking: 0,
16556 answered: 0,
16557 expected: 0,
16558 clean: true,
16559 verify_retried: false,
16560 e2e_deferred: false,
16561 e2e_defer_reason: None,
16562 progressed: false,
16563 vote_split: false,
16564 reconsideration: Vec::new(),
16565 verdict: None,
16566 }];
16567 state.gate = vec![CommandOutcome {
16568 command: "test".to_owned(),
16569 code: Some(0),
16570 output_tail: String::new(),
16571 duration_ms: 0,
16572 resource_blocked: false,
16573 }];
16574 state.gate_ran = true;
16575 state.status = RunStatus::Landing;
16579 state.merge = Some(MergeOutcome {
16580 mode: MergeMode::Pr,
16581 ok: true,
16582 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16583 empty: false,
16584 });
16585
16586 ask_test_home();
16590 state.save().expect("save landing run");
16593 let store = ask::Questions::open();
16594 let q = ask_open_question(&store, &state.id);
16595
16596 let mut runner = Runner {
16597 state,
16598 roles: ResolvedRoles {
16599 implementers: Vec::new(),
16600 judges: Vec::new(),
16601 reviewers: Vec::new(),
16602 fixer: None,
16603 conductor: conductor(),
16604 implementer_roster: Vec::new(),
16605 judge_roster: Vec::new(),
16606 reviewer_roster: Vec::new(),
16607 },
16608 sem: Arc::new(Semaphore::new(1)),
16609 pause: Pause::new(),
16610 interrupt: Pause::new(),
16611 };
16612
16613 runner.execute().await.expect("execute");
16618
16619 assert_eq!(
16620 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16621 Some("https://example.invalid/x/y/pull/1"),
16622 "reentry must not push again or open a second pull request over the \
16623 one `land` is already watching"
16624 );
16625 assert_ne!(
16626 runner.state.status,
16627 RunStatus::Landing,
16628 "land could not actually reach the fake pull request, so it must \
16629 have given up rather than left the run silently parked forever"
16630 );
16631 assert_eq!(runner.state.status, RunStatus::Blocked);
16635 assert!(
16636 store.get(&q.id).unwrap().status.open(),
16637 "Blocked is still alive; settle_questions must have been a no-op here"
16638 );
16639 }
16640
16641 fn state_with_round(round: ReviewRound) -> RunState {
16642 let mut s = RunState::new(
16643 PathBuf::from("/repo"),
16644 "main".to_owned(),
16645 "abc1234".to_owned(),
16646 "add retries".to_owned(),
16647 Config::default(),
16648 );
16649 s.reviews = vec![round];
16650 s
16651 }
16652
16653 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16654 crate::verdict::Finding {
16655 id: id.to_owned(),
16656 severity,
16657 file: None,
16658 line: None,
16659 title: title.to_owned(),
16660 detail: String::new(),
16661 }
16662 }
16663
16664 #[test]
16665 fn pr_body_names_open_findings_and_declined_ones() {
16666 let round = ReviewRound {
16667 round: 2,
16668 head: "deadbee".to_owned(),
16669 verified_head: None,
16670 verified_at: None,
16671 reviews: vec![ReviewRecord {
16672 attempts: 0,
16673 reviewer: 1,
16674 agent: "alpha".to_owned(),
16675 summary: String::new(),
16676 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16677 vote: None,
16678 failed: None,
16679 duration_ms: 0,
16680 }],
16681 e2e: vec![CommandOutcome {
16682 command: "cargo test".to_owned(),
16683 code: Some(0),
16684 output_tail: String::new(),
16685 duration_ms: 0,
16686 resource_blocked: false,
16687 }],
16688 verify_retried: false,
16689 e2e_deferred: false,
16690 e2e_defer_reason: None,
16691 fix: Some(FixRecord {
16692 agent: "alpha".to_owned(),
16693 addressed: Vec::new(),
16694 rejected: vec![crate::verdict::Rejection {
16695 id: "R1-1-1".to_owned(),
16696 why: "not reachable from any caller".to_owned(),
16697 }],
16698 notes: String::new(),
16699 committed: true,
16700 failed: None,
16701 duration_ms: 0,
16702 continuation: None,
16703 }),
16704 blocking: 0,
16705 answered: 1,
16706 expected: 1,
16707 clean: false,
16708 progressed: true,
16709 vote_split: false,
16710 reconsideration: Vec::new(),
16711 verdict: None,
16712 };
16713 let state = state_with_round(round);
16714 let body = pr_message(&state, 'A').body;
16715
16716 assert!(body.contains("add retries"), "the task must still be there");
16717 assert!(body.contains("R2-1-1"), "{body}");
16718 assert!(!body.contains("unused import"), "{body}");
16719 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16720 assert!(
16721 !body.contains("not reachable from any caller"),
16722 "the reason it was declined: {body}"
16723 );
16724 }
16725
16726 #[test]
16727 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16728 let round = ReviewRound {
16729 round: 1,
16730 head: "deadbee".to_owned(),
16731 verified_head: None,
16732 verified_at: None,
16733 reviews: vec![ReviewRecord {
16734 attempts: 0,
16735 reviewer: 1,
16736 agent: "alpha".to_owned(),
16737 summary: String::new(),
16738 findings: Vec::new(),
16739 vote: None,
16740 failed: None,
16741 duration_ms: 0,
16742 }],
16743 e2e: Vec::new(),
16744 verify_retried: false,
16745 e2e_deferred: false,
16746 e2e_defer_reason: None,
16747 fix: None,
16748 blocking: 0,
16749 answered: 1,
16750 expected: 1,
16751 clean: true,
16752 progressed: false,
16753 vote_split: false,
16754 reconsideration: Vec::new(),
16755 verdict: None,
16756 };
16757 let state = state_with_round(round);
16758 let body = pr_message(&state, 'A').body;
16759 assert!(!body.contains("Open review findings"), "{body}");
16760 assert!(!body.contains("Declined"), "{body}");
16761 }
16762
16763 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16764 let mut state = RunState::new(
16765 PathBuf::from("/repo"),
16766 "main".to_owned(),
16767 "abc1234".to_owned(),
16768 instruction.to_owned(),
16769 Config::default(),
16770 );
16771 state.candidates.push(Candidate {
16772 index: 0,
16773 label: 'A',
16774 agent: "alpha".to_owned(),
16775 branch: "magi/x/A".to_owned(),
16776 worktree: PathBuf::from("/wt"),
16777 summary: summary.to_owned(),
16778 stat: String::new(),
16779 files: 1,
16780 commits: 1,
16781 empty: false,
16782 failed: None,
16783 verified_noop: None,
16784 folded: false,
16785 duration_ms: 0,
16786 });
16787 state
16788 }
16789
16790 fn review_state(subjects: &[&str]) -> RunState {
16791 let mut state = state_with_summary(
16792 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16793 "",
16794 );
16795 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16796 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16797 state
16798 }
16799
16800 #[tokio::test]
16803 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16804 ask_test_home();
16805 let tmp = tempfile::tempdir().unwrap();
16806 let repo = tmp.path().join("repo");
16807 std::fs::create_dir_all(&repo).unwrap();
16808 init_repo(&repo);
16809 let origin = tmp.path().join("origin.git");
16810 let g = |dir: &Path, args: &[&str]| {
16811 let out = std::process::Command::new("git")
16812 .args(args)
16813 .current_dir(dir)
16814 .quiet()
16815 .output()
16816 .expect("spawn git");
16817 assert!(
16818 out.status.success(),
16819 "git {args:?}: {}",
16820 String::from_utf8_lossy(&out.stderr)
16821 );
16822 };
16823 g(
16824 tmp.path(),
16825 &[
16826 "clone",
16827 "--bare",
16828 "-q",
16829 repo.to_str().unwrap(),
16830 origin.to_str().unwrap(),
16831 ],
16832 );
16833 g(
16834 &repo,
16835 &["remote", "add", "origin", origin.to_str().unwrap()],
16836 );
16837 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16838
16839 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16841 g(&repo, &["add", "-A"]);
16842 g(
16843 &repo,
16844 &["commit", "-q", "-m", "chore(deps): update a crate"],
16845 );
16846 g(&repo, &["push", "-q", "origin", "main"]);
16847 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16848 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16849 g(&repo, &["add", "-A"]);
16850 g(
16851 &repo,
16852 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16853 );
16854 g(&repo, &["checkout", "-q", "main"]);
16855
16856 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16857 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16858 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16860 assert_eq!(fallback, c1);
16861
16862 let mut state = review_state(&[
16863 "chore(deps): update a crate",
16864 "fix(daemon): apply a chosen action",
16865 ]);
16866 state.repo = repo.clone();
16867 state.base_branch = "main".to_owned();
16868 state.base_commit = c1;
16869 refresh_reviewed_commits(&mut state, "feat/own").await;
16870 assert_eq!(
16871 state.reviewed_commits,
16872 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16873 );
16874 assert_eq!(
16875 review_title(&state).as_deref(),
16876 Some("fix(daemon): apply a chosen action")
16877 );
16878 assert_eq!(
16879 leaked_subjects(&state, "feat/own").await,
16880 Some(vec!["chore(deps): update a crate".to_owned()])
16881 );
16882 state.config.merge.remote = "nowhere".to_owned();
16885 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16886 }
16887
16888 struct SummaryFixture {
16892 _tmp: tempfile::TempDir,
16893 repo: PathBuf,
16894 reply: PathBuf,
16895 runner: Runner,
16896 }
16897
16898 fn summary_fixture() -> SummaryFixture {
16899 ask_test_home();
16900 let tmp = tempfile::tempdir().unwrap();
16901 let repo = tmp.path().join("repo");
16902 std::fs::create_dir_all(&repo).unwrap();
16903 init_repo(&repo);
16904 let origin = tmp.path().join("origin.git");
16905 let g = |args: &[&str]| {
16906 let out = std::process::Command::new("git")
16907 .args(args)
16908 .current_dir(&repo)
16909 .quiet()
16910 .output()
16911 .expect("spawn git");
16912 assert!(
16913 out.status.success(),
16914 "git {args:?}: {}",
16915 String::from_utf8_lossy(&out.stderr)
16916 );
16917 };
16918 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16919 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16920 g(&["push", "-q", "origin", "main"]);
16921 g(&["checkout", "-q", "-b", "feat/own"]);
16922 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16923 g(&["add", "-A"]);
16924 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16925 let reply = tmp.path().join("reply.txt");
16926 std::fs::write(
16927 &reply,
16928 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16929 )
16930 .unwrap();
16931 let mut runner = runner_at(RunStatus::Gating);
16932 let mut state = review_state(&["feat(own): add the own file"]);
16933 state.repo = repo.clone();
16934 state.base_branch = "main".to_owned();
16935 state.candidates[0].branch = "feat/own".to_owned();
16936 state.candidates[0].worktree = repo.clone();
16937 let mut author = spec("alpha");
16938 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
16939 author
16940 .env
16941 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
16942 state.config.agents = vec![author];
16943 runner.state = state;
16944 SummaryFixture {
16945 _tmp: tmp,
16946 repo,
16947 reply,
16948 runner,
16949 }
16950 }
16951
16952 async fn posted(f: &mut SummaryFixture) -> PrMessage {
16953 let winner = f.runner.state.candidates[0].clone();
16954 f.runner
16955 .guarded_pr_message(&winner, None, true)
16956 .await
16957 .unwrap()
16958 .expect("an English description never parks")
16959 }
16960
16961 fn clean_round_with_minor_findings() -> ReviewRound {
16962 let mut round = passing_round(2, None);
16963 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
16964 round.reviews = vec![ReviewRecord {
16965 attempts: 0,
16966 reviewer: 1,
16967 agent: "alpha".into(),
16968 summary: "内部レビュー".into(),
16969 findings: vec![
16970 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
16971 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
16972 ],
16973 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
16974 failed: None,
16975 duration_ms: 0,
16976 }];
16977 round
16978 }
16979
16980 #[tokio::test]
16981 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
16982 for review_only in [false, true] {
16983 let mut f = summary_fixture();
16984 if !review_only {
16985 f.runner.state.candidates[0].agent = "alpha".into();
16986 }
16987 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
16988 assert!(f.runner.state.reviews[0].clean);
16989 assert!(f.runner.state.reviews[0].fix.is_none());
16990 assert!(f.runner.state.open_findings().is_empty());
16991 assert_eq!(
16992 crate::pr_outcome::required_finding_ids(&f.runner.state),
16993 ["R2-1-1", "R2-1-2"]
16994 );
16995 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
16996 let evidence = crate::pr_outcome::summary_evidence(
16997 &f.runner.state,
16998 crate::pr_outcome::FinalHead::At(&head),
16999 );
17000 assert!(
17001 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17002 );
17003 let omitted = posted(&mut f).await;
17005 assert!(omitted.body.contains("No description"));
17006 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17007 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17008 assert!(f.runner.state.change_summary.is_none());
17009 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17010 let described = posted(&mut f).await;
17011 assert!(described.body.contains("improve documentation for callers"));
17012 assert!(described.body.contains("remaining edge case"));
17013 assert!(described.body.contains("## Open review findings"));
17014 assert!(!described.body.contains("未修正"));
17015 }
17016 }
17017
17018 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17019 let mut f = summary_fixture();
17020 if !review_only {
17021 f.runner.state.candidates[0].agent = "alpha".into();
17022 }
17023 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17024 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17025 fixed
17026 .fix
17027 .as_mut()
17028 .unwrap()
17029 .rejected
17030 .push(crate::verdict::Rejection {
17031 id: "R1-1-1".into(),
17032 why: "呼び出されないコードのため辞退".into(),
17033 });
17034 let mut clean = clean_round_with_minor_findings();
17035 clean.verified_head = Some(head.clone());
17036 f.runner.state.reviews = vec![fixed, clean];
17037 ok_gate(&mut f.runner.state, Some(&head));
17038 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17039 f.runner.state.config.agents.push(scripted("beta", reply));
17040 f.runner.state.config.roles.language_judge =
17041 Some(crate::config::AgentChoice::One("beta".into()));
17042 f
17043 }
17044
17045 #[tokio::test]
17046 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17047 for review_only in [false, true] {
17048 let mut f = rewrite_fixture_with_findings(review_only,
17049 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17050 let m = posted(&mut f).await;
17051 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17052 let expected =
17053 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17054 assert!(m.body.contains(&expected));
17055 assert!(m.body.contains("## Review rounds"));
17056 assert!(
17057 m.body
17058 .contains("Gate (`verify.gate`): passed on the final commit")
17059 );
17060 assert!(
17061 m.body
17062 .contains("Tests (`verify.e2e`): passed on the final commit")
17063 );
17064 let c = f.runner.state.change_summary.as_ref().unwrap();
17065 assert!(
17066 m.body
17067 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17068 );
17069 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17070 assert!(crate::pr_outcome::covers_required_findings(
17071 &f.runner.state,
17072 &m.body
17073 ));
17074 assert!(m.body.contains("documentation remains incomplete"));
17075 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17076 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17077 assert!(f.runner.state.github_text.is_none());
17078 }
17079 }
17080
17081 #[tokio::test]
17082 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17083 for review_only in [false, true] {
17084 for reply in [
17085 "TITLE: fix: public description\nA short English description.",
17086 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17087 ] {
17088 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17089 let winner = f.runner.state.candidates[0].clone();
17090 let posted = f
17091 .runner
17092 .guarded_pr_message(&winner, None, true)
17093 .await
17094 .unwrap();
17095 assert!(posted.is_none());
17096 assert!(f.runner.state.parked);
17097 let withheld = f.runner.state.github_text.as_ref().unwrap();
17098 assert!(
17099 withheld.rewrite.is_none(),
17100 "an incomplete rewrite must not be selectable"
17101 );
17102 let q = ask::Questions::open()
17103 .get(withheld.question.as_ref().unwrap())
17104 .unwrap();
17105 assert!(
17106 !q.choices
17107 .contains(&crate::github_text::USE_REWRITE.to_owned())
17108 );
17109 assert!(
17110 q.detail.contains("R1-1-1")
17111 && q.detail.contains("R2-1-1")
17112 && q.detail.contains("R2-1-2")
17113 );
17114 }
17115 }
17116 }
17117
17118 #[tokio::test]
17119 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17120 for review_only in [false, true] {
17121 let mut f = summary_fixture();
17122 if !review_only {
17123 f.runner.state.candidates[0].agent = "alpha".into();
17124 }
17125 f.runner.state.instruction = "内部タスクの説明".into();
17126 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17127 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17128 f.runner.state.reviews = vec![
17129 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17130 round_with_notes(2, Some("二回目の内部修正報告です。")),
17131 passing_round(3, Some(&head)),
17132 ];
17133 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17134 ok_gate(&mut f.runner.state, Some(&head));
17135 let captured = f.repo.join("public-prompt.txt");
17136 let author = &mut f.runner.state.config.agents[0];
17137 author.command = vec![
17138 "sh".into(),
17139 "-c".into(),
17140 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17141 ];
17142 author.env.insert(
17143 "PROMPT_FILE".into(),
17144 captured.to_string_lossy().into_owned(),
17145 );
17146 let m = posted(&mut f).await;
17147 assert!(m.body.contains("Adds `own.txt`"));
17148 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17149 assert!(m.body.contains("R1-1-1"));
17150 assert!(
17151 m.body
17152 .contains("Gate (`verify.gate`): passed on the final commit")
17153 );
17154 assert!(
17155 m.body
17156 .contains("Tests (`verify.e2e`): passed on the final commit")
17157 );
17158 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17159 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17160 let prompt = std::fs::read_to_string(captured).unwrap();
17161 assert!(prompt.contains(&format!(
17162 "git diff {} {head}",
17163 f.runner.state.change_summary.as_ref().unwrap().base
17164 )));
17165 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17166 assert_eq!(
17167 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17168 "二回目の内部修正報告です。"
17169 );
17170 }
17171 }
17172
17173 #[tokio::test]
17174 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17175 for review_only in [false, true] {
17176 let mut f = summary_fixture();
17177 if !review_only {
17178 f.runner.state.candidates[0].agent = "alpha".into();
17179 }
17180 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17181 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17182 std::fs::write(&f.reply, "no usable description").unwrap();
17183 let failed = posted(&mut f).await;
17184 assert!(
17185 failed
17186 .body
17187 .contains("No description of what this change does could be generated")
17188 );
17189 assert!(failed.body.is_ascii());
17190 assert!(f.runner.state.github_text.is_none());
17191 assert!(f.runner.state.change_summary.is_none());
17192 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17193 let evidence = crate::pr_outcome::summary_evidence(
17194 &f.runner.state,
17195 crate::pr_outcome::FinalHead::At(&head),
17196 );
17197 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17198 assert!(
17199 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17200 .exists()
17201 );
17202 f.runner.state =
17203 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17204 std::fs::write(
17205 &f.reply,
17206 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17207 )
17208 .unwrap();
17209 let retried = posted(&mut f).await;
17210 assert!(retried.body.contains("Adds the committed own file."));
17211 assert_eq!(
17212 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17213 "日本語の修正報告"
17214 );
17215 }
17216 }
17217
17218 #[tokio::test]
17219 async fn same_head_verification_and_findings_invalidate_public_prose() {
17220 let mut f = summary_fixture();
17221 posted(&mut f).await;
17222 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17223 let original = f
17224 .runner
17225 .state
17226 .change_summary
17227 .as_ref()
17228 .unwrap()
17229 .evidence
17230 .clone();
17231 ok_gate(&mut f.runner.state, Some(&head));
17232 std::fs::write(
17233 &f.reply,
17234 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17235 )
17236 .unwrap();
17237 let verified = posted(&mut f).await;
17238 assert!(verified.body.contains("now verified"));
17239 assert_ne!(
17240 f.runner.state.change_summary.as_ref().unwrap().evidence,
17241 original
17242 );
17243 let mut round = round_with_notes(1, Some("内部報告"));
17244 round.reviews.push(ReviewRecord {
17245 attempts: 0,
17246 reviewer: 1,
17247 agent: "alpha".into(),
17248 summary: "内部レビュー".into(),
17249 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17250 vote: None,
17251 failed: None,
17252 duration_ms: 0,
17253 });
17254 round
17255 .fix
17256 .as_mut()
17257 .unwrap()
17258 .rejected
17259 .push(crate::verdict::Rejection {
17260 id: "R1-1-1".into(),
17261 why: "呼び出し元から到達できない".into(),
17262 });
17263 f.runner.state.reviews = vec![round];
17264 let omitted = posted(&mut f).await;
17266 assert!(omitted.body.contains("No description"));
17267 assert!(omitted.body.contains("R1-1-1"));
17268 assert!(!omitted.body.contains("到達"));
17269 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17270 let described = posted(&mut f).await;
17271 assert!(described.body.contains("unreachable from callers"));
17272 assert!(described.body.contains("R1-1-1"));
17273 assert!(!described.body.contains("到達"));
17274 let evidence = f
17275 .runner
17276 .state
17277 .change_summary
17278 .as_ref()
17279 .unwrap()
17280 .evidence
17281 .clone();
17282 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17283 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17284 assert!(posted(&mut f).await.body.contains("rationale updated"));
17285 assert_ne!(
17286 f.runner.state.change_summary.as_ref().unwrap().evidence,
17287 evidence
17288 );
17289 f.runner
17291 .state
17292 .change_summary
17293 .as_mut()
17294 .unwrap()
17295 .evidence
17296 .clear();
17297 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17298 assert!(posted(&mut f).await.body.contains("current specification"));
17299 }
17300
17301 #[tokio::test]
17302 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17303 let mut f = summary_fixture();
17304 let m = posted(&mut f).await;
17305 assert_eq!(m.title, "feat(own): add the own file");
17306 assert!(
17307 m.body
17308 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17309 "{}",
17310 m.body
17311 );
17312 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17313 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17314 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17315 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17316 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17317 let kept = f.runner.state.change_summary.as_ref().unwrap();
17318 assert_eq!(
17319 (kept.head.as_str(), kept.base.as_str()),
17320 (head.as_str(), base.as_str())
17321 );
17322 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17323 }
17324
17325 #[tokio::test]
17326 async fn review_only_summary_receives_the_configured_github_language() {
17327 let mut f = summary_fixture();
17328 f.runner.state.config.graph.github_language = "ja".into();
17329 let captured = f.repo.join("summary-prompt.txt");
17330 let author = &mut f.runner.state.config.agents[0];
17331 author.command = vec![
17332 "sh".into(),
17333 "-c".into(),
17334 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17335 ];
17336 author.env.insert(
17337 "PROMPT_FILE".into(),
17338 captured.to_string_lossy().into_owned(),
17339 );
17340 std::fs::write(
17341 &f.reply,
17342 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17343 )
17344 .unwrap();
17345 let m = posted(&mut f).await;
17346 let prompt = std::fs::read_to_string(captured).unwrap();
17347 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17348 assert!(!prompt.contains("in English"));
17349 assert_eq!(m.title, "feat(own): ファイルを追加");
17350 assert!(m.body.contains("own.txt を追加します。"));
17351 }
17352
17353 #[tokio::test]
17354 async fn review_fixes_come_after_the_whole_change_summary() {
17355 let mut f = summary_fixture();
17356 f.runner.state.reviews = vec![
17357 round_with_notes(1, Some("handled the PATH case")),
17358 round_with_notes(2, None),
17359 ];
17360 let m = posted(&mut f).await;
17361 let summary = m.body.find("## Summary").expect("summary");
17362 let fixes = m.body.find("## Review rounds").expect("review rounds");
17363 assert!(summary < fixes, "{}", m.body);
17364 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17365 assert!(m.body.contains("## Final verification"), "{}", m.body);
17366 }
17367
17368 #[tokio::test]
17369 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17370 let mut f = summary_fixture();
17371 let first = posted(&mut f).await;
17372 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17373 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17375 let again = posted(&mut f).await;
17376 assert_eq!((again.title, again.body), (first.title, first.body));
17377
17378 let g = |args: &[&str]| {
17379 let out = std::process::Command::new("git")
17380 .args(args)
17381 .current_dir(&f.repo)
17382 .quiet()
17383 .output()
17384 .expect("spawn git");
17385 assert!(out.status.success(), "git {args:?}");
17386 };
17387 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17388 g(&["add", "-A"]);
17389 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17390 let second = posted(&mut f).await;
17391 let kept = f.runner.state.change_summary.clone().unwrap();
17392 assert_ne!(kept.head, first_head);
17393 assert!(second.body.contains("Not read yet."), "{}", second.body);
17394 assert!(!second.body.contains("so the first summary is read"));
17395 assert_eq!(second.title, "feat(own): other");
17396 }
17397
17398 #[tokio::test]
17399 async fn an_unavailable_summary_is_stated_and_never_invented() {
17400 let mut f = summary_fixture();
17401 std::fs::write(&f.reply, "no title line here").unwrap();
17402 let m = posted(&mut f).await;
17403 assert!(
17404 m.body
17405 .contains("No description of what this change does could be generated"),
17406 "{}",
17407 m.body
17408 );
17409 assert!(m.body.contains("internal reports remain in local records"));
17410 assert!(f.runner.state.change_summary.is_none());
17411 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17412 let winner = f.runner.state.candidates[0].clone();
17414 let preview = f
17415 .runner
17416 .guarded_pr_message(&winner, None, false)
17417 .await
17418 .unwrap()
17419 .unwrap();
17420 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17421 }
17422
17423 #[test]
17424 fn pr_message_review_single_commit_uses_its_subject() {
17425 let state = review_state(&["feat(nats): per-role user"]);
17426 let m = pr_message(&state, 'A');
17427 assert_eq!(m.title, "feat(nats): per-role user");
17428 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17429 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17430 }
17431
17432 #[test]
17433 fn pr_message_review_multi_commit_takes_the_oldest() {
17434 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17435 let m = pr_message(&state, 'A');
17436 assert_eq!(m.title, "feat: the change");
17437 for s in ["feat: the change", "fix: typo", "fix: again"] {
17438 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17439 }
17440 }
17441
17442 #[test]
17443 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17444 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17445 let mut state = review_state(&[first, "fix: later fixup"]);
17446 state.candidates[0].branch = "機能/ログイン".to_owned();
17447 let m = pr_message(&state, 'A');
17448 assert!(
17449 m.title.starts_with("chore: land candidate A of run"),
17450 "{}",
17451 m.title
17452 );
17453 }
17454 }
17455
17456 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17457 BranchFacts {
17458 commits: commits
17459 .iter()
17460 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17461 .collect(),
17462 stat: stat.to_owned(),
17463 }
17464 }
17465
17466 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17467 let mut r = review_round(true, 0, 1, 1, true, true);
17468 r.round = round;
17469 r.fix = notes.map(|n| FixRecord {
17470 agent: "fixer".to_owned(),
17471 addressed: Vec::new(),
17472 rejected: Vec::new(),
17473 notes: n.to_owned(),
17474 committed: true,
17475 failed: None,
17476 duration_ms: 0,
17477 continuation: None,
17478 });
17479 r
17480 }
17481
17482 #[test]
17483 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17484 let state = review_state(&["ignored"]);
17485 let f = facts(
17486 &[
17487 (
17488 "fix(login): resolve PATH on macOS",
17489 "Login shells skip rc files.",
17490 ),
17491 ("fix: address review", ""),
17492 ],
17493 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17494 );
17495 let m = pr_message_with(&state, 'A', Some(&f));
17496 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17497 assert!(
17498 m.body
17499 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17500 "{}",
17501 m.body
17502 );
17503 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17504 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17505 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17506 for banned in [
17507 "Review the work already",
17508 "no task statement",
17509 "Original task",
17510 ] {
17511 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17512 }
17513 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17514 }
17515
17516 #[test]
17517 fn pr_message_review_truncates_a_huge_first_commit_body() {
17518 let state = review_state(&["ignored"]);
17519 let f = facts(
17520 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17521 "s",
17522 );
17523 let m = pr_message_with(&state, 'A', Some(&f));
17524 assert!(m.body.len() < 30_000, "{}", m.body.len());
17525 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17526 assert!(
17527 m.body.contains("- ... 1 more commit(s)"),
17528 "{}",
17529 m.body.len()
17530 );
17531 assert!(m.body.ends_with("magi:candidate-a\n"));
17532 }
17533
17534 #[test]
17535 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17536 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17537 assert_eq!(m.title, "feat: x");
17538 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17539 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17540 assert!(!m.body.contains("no task statement"), "{}", m.body);
17541 }
17542
17543 #[test]
17544 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17545 let mut state = review_state(&["日本語の件名"]);
17546 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17547 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17548 state.candidates[0].branch = "機能/ログイン".to_owned();
17549 assert!(
17550 pr_message(&state, 'A')
17551 .title
17552 .starts_with("chore: land candidate A")
17553 );
17554 }
17555
17556 #[test]
17557 fn pr_message_review_fixes_survive_a_clean_final_round() {
17558 let mut state = review_state(&["feat: x"]);
17559 state.reviews = vec![
17560 round_with_notes(1, Some("handled the PATH case")),
17561 round_with_notes(2, None),
17562 ];
17563 let body = pr_message(&state, 'A').body;
17564 assert!(
17565 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17566 "{body}"
17567 );
17568 assert!(body.contains("### Round 1"), "{body}");
17569
17570 state.reviews = vec![
17571 round_with_notes(1, Some("first fix")),
17572 round_with_notes(2, Some("")),
17573 round_with_notes(3, Some("second fix")),
17574 round_with_notes(4, None),
17575 ];
17576 let body = pr_message(&state, 'A').body;
17577 assert!(
17578 body.contains("### Round 1") && !body.contains("first fix"),
17579 "{body}"
17580 );
17581 assert!(
17582 body.contains("### Round 3") && !body.contains("second fix"),
17583 "{body}"
17584 );
17585 assert!(body.contains("### Round 2"), "{body}");
17586 }
17587
17588 const FINAL: &str = "abcdef1234567890";
17589
17590 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17591 let mut r = round_with_notes(round, None);
17592 r.verified_head = verified.map(str::to_owned);
17593 r
17594 }
17595
17596 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17597 state.gate = vec![CommandOutcome {
17598 command: "gate".to_owned(),
17599 code: Some(0),
17600 output_tail: String::new(),
17601 duration_ms: 0,
17602 resource_blocked: false,
17603 }];
17604 state.gate_ran = true;
17605 state.gate_head = head.map(str::to_owned);
17606 }
17607
17608 fn body_at(state: &RunState) -> String {
17609 pr_message_raw(
17610 state,
17611 'A',
17612 None,
17613 crate::pr_outcome::FinalHead::At(FINAL),
17614 ChangeNote::NotAsked,
17615 )
17616 .body
17617 }
17618
17619 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17620
17621 #[test]
17622 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17623 let mut state = review_state(&["feat: x"]);
17624 state.reviews = vec![
17625 round_with_notes(1, Some(SANDBOXED)),
17626 passing_round(2, Some(FINAL)),
17627 ];
17628 ok_gate(&mut state, Some(FINAL));
17629 let body = body_at(&state);
17630 assert!(!body.contains("Handled the PATH case."), "{body}");
17631 assert!(!body.contains("could not commit"), "{body}");
17632 assert!(!body.contains("could not run"), "{body}");
17633 assert!(
17634 body.contains("The fix is committed on the branch"),
17635 "{body}"
17636 );
17637 assert!(!body.contains("sandbox"), "{body}");
17638 assert!(
17639 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17640 "{body}"
17641 );
17642 assert!(
17643 body.contains("Gate (`verify.gate`): passed on the final commit"),
17644 "{body}"
17645 );
17646 assert!(
17648 state.reviews[0]
17649 .fix
17650 .as_ref()
17651 .unwrap()
17652 .notes
17653 .contains("could not commit")
17654 );
17655 }
17656
17657 #[test]
17658 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17659 let mut state = review_state(&["feat: x"]);
17660 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17661 r1.fix.as_mut().unwrap().committed = false;
17662 let mut r2 = passing_round(2, Some(FINAL));
17663 r2.e2e[0].code = Some(1);
17664 state.reviews = vec![r1, r2];
17665 let body = body_at(&state);
17666 assert!(!body.contains("could not commit"), "{body}");
17667 assert!(!body.contains("could not run"), "{body}");
17668 assert!(body.contains("This round produced no commit"), "{body}");
17669 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17670 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17671 assert!(!body.contains("passed"), "{body}");
17672 }
17673
17674 #[test]
17675 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17676 let mut state = review_state(&["feat: x"]);
17677 state.reviews = vec![
17678 round_with_notes(1, Some(SANDBOXED)),
17679 passing_round(2, Some("0123456789abcdef")),
17680 ];
17681 ok_gate(&mut state, Some("0123456789abcdef"));
17682 let body = body_at(&state);
17683 assert!(!body.contains("HTTP tests could not run"), "{body}");
17684 assert!(!body.contains("could not commit"), "{body}");
17686 assert!(
17687 body.contains("passed on an earlier commit (`0123456`)"),
17688 "{body}"
17689 );
17690 assert!(!body.contains("passed on the final commit"), "{body}");
17691
17692 ok_gate(&mut state, None);
17694 let body = body_at(&state);
17695 assert!(
17696 body.contains("passed on a commit that was not recorded"),
17697 "{body}"
17698 );
17699 }
17700
17701 #[test]
17702 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17703 let state = state_with_summary(
17704 "add retries to the client",
17705 "TITLE: feat: retries\n\nDid it.",
17706 );
17707 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17708 assert_eq!(m.title, "feat: retries");
17709 assert!(
17710 m.body.contains("<summary>Original task</summary>"),
17711 "{}",
17712 m.body
17713 );
17714 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17715 assert!(
17716 m.body
17717 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17718 "{}",
17719 m.body
17720 );
17721 }
17722
17723 #[test]
17724 fn pr_message_review_bounds_a_long_english_subject() {
17725 let long = format!("feat: {}", "word ".repeat(100));
17726 let m = pr_message(&review_state(&[&long]), 'A');
17727 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17728 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17729 }
17730
17731 #[test]
17732 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17733 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17734 let m = pr_message(&state, 'A');
17735 assert_eq!(m.title, "add retries");
17736 assert!(m.body.contains("<summary>Original task</summary>"));
17737 assert!(!m.body.contains("Commits under review"));
17738 assert_eq!(landing_subject_source(&state), state.instruction);
17739 }
17740
17741 #[test]
17742 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17743 let state = review_state(&["feat: the change", "fix: typo"]);
17744 let source = landing_subject_source(&state);
17745 assert_eq!(land::merge_subject("", &source), "feat: the change");
17746 assert_eq!(
17747 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17748 "feat: the change"
17749 );
17750 assert_eq!(
17752 land::merge_subject("feat: renamed by hand", &source),
17753 "feat: renamed by hand"
17754 );
17755 let blank = review_state(&["日本語"]);
17756 assert!(
17757 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17758 );
17759 }
17760
17761 #[test]
17762 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17763 let state = review_state(&["feat: the change"]);
17764 let source = landing_subject_source(&state);
17765 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17766 assert_eq!(
17767 land::merge_subject(landing_title(&state, old), &source),
17768 "feat: the change"
17769 );
17770 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17771 let task = state_with_summary("add retries", "");
17772 assert_eq!(landing_title(&task, old), old);
17773 }
17774
17775 #[test]
17776 fn pr_message_describes_the_change_not_the_task() {
17777 let state = state_with_summary(
17778 "今回やってほしいこと: results projector を直す",
17779 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17780 );
17781 let m = pr_message(&state, 'A');
17782 assert_eq!(m.title, "fix(web): batch the runs list reads");
17783 assert!(
17784 m.body.starts_with("## Summary\n\n- reads run.json once"),
17785 "{}",
17786 m.body
17787 );
17788 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17789 let task_at = m.body.find("今回やってほしいこと").unwrap();
17790 let details_at = m.body.find("<details>").unwrap();
17791 assert!(
17792 details_at < task_at,
17793 "the task lives inside <details>: {}",
17794 m.body
17795 );
17796 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17797 assert!(m.body.contains("magi:candidate-a"));
17798 }
17799
17800 #[test]
17801 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17802 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17803 let m = pr_message(&state, 'A');
17804 assert_eq!(m.title, "add retries");
17805 assert!(
17806 m.body.contains("## Summary\n\n- did some things"),
17807 "{}",
17808 m.body
17809 );
17810
17811 let none = RunState::new(
17812 PathBuf::from("/repo"),
17813 "main".to_owned(),
17814 "abc1234".to_owned(),
17815 "add retries".to_owned(),
17816 Config::default(),
17817 );
17818 let m = pr_message(&none, 'A');
17819 assert_eq!(m.title, "add retries");
17820 assert!(!m.body.contains("## Summary"), "{}", m.body);
17821 }
17822
17823 #[test]
17824 fn pr_message_refuses_the_candidate_commit_subject() {
17825 for bad in [
17826 "TITLE: magi: candidate A (uncommitted work)",
17827 "TITLE: chore: stuff (uncommitted work)",
17828 "TITLE: ",
17829 ] {
17830 let state = state_with_summary("add retries", bad);
17831 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17832 }
17833 }
17834
17835 #[test]
17836 fn pr_message_bounds_a_very_long_task_and_title() {
17837 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17838 let state = state_with_summary(&long, "- nothing");
17839 let m = pr_message(&state, 'A');
17840 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17841 assert!(!m.title.contains('\n'));
17842
17843 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17844 let m = pr_message(&state, 'A');
17845 assert!(m.title.starts_with("feat: "));
17846 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17847 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17848 }
17849
17850 fn long_title_of(instruction: &str) -> String {
17851 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17852 }
17853
17854 #[test]
17855 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17856 let first = "Make the landing path keep a readable title for long tasks";
17857 let line = format!(
17858 "{first}. {}",
17859 "And then keep going with more words ".repeat(20)
17860 );
17861 let t = long_title_of(&line);
17862 assert_eq!(t, first);
17863 assert!(!t.starts_with("chore: land"));
17864 }
17865
17866 #[test]
17867 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17868 let line = "word ".repeat(200);
17869 let t = long_title_of(&line);
17870 assert!(t.ends_with("word..."), "{t}");
17871 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17872 }
17873
17874 #[test]
17875 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17876 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17877 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17878 }
17879 }
17880
17881 #[test]
17882 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17883 let mut state = state_with_summary("再試行を追加する", "- nothing");
17884 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17885 state.config.graph.github_language = "ja".into();
17886 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17887 }
17888
17889 #[test]
17890 fn pr_message_title_limit_is_exact() {
17891 let at = "a".repeat(PR_TITLE_MAX);
17892 assert_eq!(long_title_of(&at), at);
17893 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17894 assert!(over.ends_with("..."), "{over}");
17895 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17896 }
17897
17898 #[test]
17899 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17900 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17901 let t = long_title_of(&line);
17902 assert!(t.ends_with("..."), "{t}");
17903 assert!(t.is_ascii(), "{t}");
17904 }
17905
17906 #[test]
17907 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17908 let line = format!(
17909 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17910 "plus more filler words ".repeat(20)
17911 );
17912 let t = long_title_of(&line);
17913 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17914 }
17915
17916 #[test]
17917 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17918 let line = format!("\n\n# {}", "title words ".repeat(40));
17919 let state = state_with_summary(&line, "- nothing");
17920 let m = pr_message(&state, 'A');
17921 assert!(m.title.starts_with("title words"), "{}", m.title);
17922 assert_eq!(
17923 land::merge_subject(&m.title, &landing_subject_source(&state)),
17924 m.title
17925 );
17926 assert_eq!(
17928 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17929 "feat: renamed by hand"
17930 );
17931 }
17932
17933 #[tokio::test]
17934 async fn github_text_rewrite_is_bounded_and_falls_back() {
17935 crate::run::pin_test_home();
17936 for (reply, accepted) in [
17937 (
17938 "TITLE: fix: retries\nAdd retries for failed requests.",
17939 true,
17940 ),
17941 (
17942 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
17943 false,
17944 ),
17945 (
17946 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
17947 true,
17948 ),
17949 (
17950 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
17951 true,
17952 ),
17953 (
17954 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
17955 true,
17956 ),
17957 (
17958 "TITLE: fix: retries\n日本語の説明です。 token=secret",
17959 false,
17960 ),
17961 ("TITLE: fix: retries\n", false),
17962 ] {
17963 let dir = tempfile::tempdir().unwrap();
17964 let mut runner = runner_at(RunStatus::Gating);
17965 runner.state = state_with_summary(
17966 "add retries",
17967 "TITLE: fix: retries\n日本語の説明を書きます。",
17968 );
17969 runner.state.repo = dir.path().to_owned();
17970 runner.state.candidates[0].worktree = dir.path().to_owned();
17971 let mut author = spec("alpha");
17972 author.command = vec![
17973 "sh".into(),
17974 "-c".into(),
17975 "printf '%s' \"$REWRITE_REPLY\"".into(),
17976 ];
17977 author.env.insert("REWRITE_REPLY".into(), reply.into());
17978 runner.state.config.agents = vec![author];
17979 seed_public_summary(&mut runner);
17980 let winner = runner.state.candidates[0].clone();
17981 let first = runner
17982 .guarded_pr_message(&winner, None, true)
17983 .await
17984 .unwrap();
17985 let message = if accepted {
17986 first.expect("a passing rewrite never asks")
17987 } else {
17988 assert!(first.is_none());
17990 assert!(runner.state.parked);
17991 let store = ask::Questions::open();
17992 let asked: Vec<_> = store
17993 .list()
17994 .into_iter()
17995 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
17996 .collect();
17997 assert_eq!(asked.len(), 1);
17998 assert!(!asked[0].detail.contains("secret"));
17999 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18000 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18001 assert_eq!(
18002 asked[0]
18003 .choices
18004 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18005 rewrite.is_some()
18006 );
18007 store
18010 .update(&asked[0].id, |q| {
18011 q.abandon("test");
18012 Ok(())
18013 })
18014 .unwrap();
18015 let again = runner
18016 .guarded_pr_message(&winner, None, true)
18017 .await
18018 .unwrap()
18019 .expect("an abandoned question falls back");
18020 let n = store
18021 .list()
18022 .into_iter()
18023 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18024 .count();
18025 assert_eq!(n, 1, "asked once per rejected text");
18026 again
18027 };
18028 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18029 assert!(!message.body.contains("secret"));
18030 assert!(!message.body.contains("prefix"));
18031 assert!(!message.title.contains("secret"));
18032 assert_eq!(
18033 message.body.contains("Add retries for failed requests."),
18034 accepted
18035 );
18036 assert_eq!(
18037 runner.state.seats["github-text-rewrite"].turns, 1,
18038 "only one rewrite invocation"
18039 );
18040 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18041 if accepted {
18042 assert!(runner.state.github_text.is_none());
18043 assert!(!runner.state.parked);
18044 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18045 assert!(message.body.contains("## Final verification"));
18046 assert!(message.body.contains("Evidence: final diff"));
18047 assert!(
18048 message
18049 .body
18050 .contains(&format!("magi:run/{}", runner.state.id))
18051 );
18052 }
18053 if !accepted {
18054 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18055 assert!(
18056 message
18057 .body
18058 .contains(&format!("magi:run/{}", runner.state.id))
18059 );
18060 }
18061 }
18062 }
18063
18064 fn scripted(id: &str, reply: &str) -> AgentSpec {
18065 let mut a = spec(id);
18066 a.command = vec![
18067 "sh".into(),
18068 "-c".into(),
18069 "printf '%s' \"$REWRITE_REPLY\"".into(),
18070 ];
18071 a.env.insert("REWRITE_REPLY".into(), reply.into());
18072 a
18073 }
18074
18075 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18076 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18077
18078 fn seed_public_summary(runner: &mut Runner) {
18080 let repo = &runner.state.repo;
18081 let g = |args: &[&str]| {
18082 let out = std::process::Command::new("git")
18083 .args(args)
18084 .current_dir(repo)
18085 .quiet()
18086 .output()
18087 .unwrap();
18088 assert!(out.status.success(), "git {args:?}");
18089 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18090 };
18091 g(&["init", "-q", "-b", "main"]);
18092 g(&[
18093 "-c",
18094 "user.name=Test",
18095 "-c",
18096 "user.email=test@example.invalid",
18097 "commit",
18098 "-q",
18099 "--allow-empty",
18100 "-m",
18101 "feat: fixture",
18102 ]);
18103 let head = g(&["rev-parse", "HEAD"]);
18104 g(&["update-ref", "refs/remotes/origin/main", &head]);
18105 let evidence = crate::pr_outcome::summary_evidence(
18106 &runner.state,
18107 crate::pr_outcome::FinalHead::At(&head),
18108 );
18109 let generated = &runner.state.candidates[0].summary;
18110 runner.state.change_summary = Some(crate::run::ChangeSummary {
18111 base: head.clone(),
18112 head,
18113 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18114 title: summary_title(generated).unwrap(),
18115 body: summary_without_title(generated),
18116 });
18117 }
18118
18119 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18120 let mut runner = runner_at(RunStatus::Gating);
18121 runner.state = state_with_summary("add retries", JA_SUMMARY);
18122 if review_only {
18123 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18124 }
18125 runner.state.repo = dir.to_owned();
18126 runner.state.candidates[0].worktree = dir.to_owned();
18127 runner.state.config.agents = agents;
18128 seed_public_summary(&mut runner);
18129 runner
18130 }
18131
18132 fn gate_questions(runner: &Runner) -> usize {
18133 ask::Questions::open()
18134 .list()
18135 .into_iter()
18136 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18137 .count()
18138 }
18139
18140 #[tokio::test]
18141 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18142 crate::run::pin_test_home();
18143 let dir = tempfile::tempdir().unwrap();
18144 let mut runner = rewrite_runner(
18145 dir.path(),
18146 true,
18147 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18148 );
18149 runner.state.config.roles.language_judge =
18150 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18151 let winner = runner.state.candidates[0].clone();
18152 let msg = runner
18153 .guarded_pr_message(&winner, None, true)
18154 .await
18155 .unwrap()
18156 .expect("an English rewrite posts without asking");
18157 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18158 assert!(msg.body.contains("<details>kept</details>"));
18159 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18160 assert_eq!(gate_questions(&runner), 0);
18161 assert!(runner.state.github_text.is_none());
18162 assert!(!runner.state.seats.contains_key("impl-A"));
18163 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18164 }
18165
18166 #[tokio::test]
18167 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18168 crate::run::pin_test_home();
18169 let dir = tempfile::tempdir().unwrap();
18170 let mut runner = rewrite_runner(
18171 dir.path(),
18172 false,
18173 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18174 );
18175 runner.state.config.roles.language_judge =
18176 Some(crate::config::AgentChoice::One("beta".into()));
18177 runner.state.seats.insert(
18178 "impl-A".into(),
18179 SeatState::new("impl-A", "alpha", runner.state.seed),
18180 );
18181 let winner = runner.state.candidates[0].clone();
18182 let msg = runner
18183 .guarded_pr_message(&winner, None, true)
18184 .await
18185 .unwrap()
18186 .expect("rewritten");
18187 assert!(msg.body.contains("<details>kept</details>"));
18188 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18189 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18190 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18191 }
18192
18193 #[tokio::test]
18194 async fn rewrite_reuses_a_resumable_implementer_session() {
18195 crate::run::pin_test_home();
18196 let dir = tempfile::tempdir().unwrap();
18197 let mut runner = rewrite_runner(
18198 dir.path(),
18199 false,
18200 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18201 );
18202 runner.state.config.roles.language_judge =
18203 Some(crate::config::AgentChoice::One("beta".into()));
18204 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18205 seat.turns = 1;
18206 runner.state.seats.insert("impl-A".into(), seat);
18207 let winner = runner.state.candidates[0].clone();
18208 runner
18209 .guarded_pr_message(&winner, None, true)
18210 .await
18211 .unwrap()
18212 .expect("rewritten");
18213 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18214 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18215 }
18216
18217 #[tokio::test]
18218 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18219 crate::run::pin_test_home();
18220 let dir = tempfile::tempdir().unwrap();
18221 let mut broken = spec("bad");
18222 broken.command = vec!["false".into()];
18223 let mut runner = rewrite_runner(
18224 dir.path(),
18225 true,
18226 vec![
18227 broken,
18228 scripted("noformat", "Add retries."),
18229 scripted("good", EN_REWRITE),
18230 ],
18231 );
18232 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18233 "bad".into(),
18234 "noformat".into(),
18235 "good".into(),
18236 ]));
18237 let winner = runner.state.candidates[0].clone();
18238 let msg = runner
18239 .guarded_pr_message(&winner, None, true)
18240 .await
18241 .unwrap()
18242 .expect("the third agent rewrites");
18243 assert!(msg.body.contains("`src/net.rs`"));
18244 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18245 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18246 }
18247
18248 #[tokio::test]
18249 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18250 crate::run::pin_test_home();
18251 for (agents, role, saved) in [
18253 (
18254 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18255 Some("beta"),
18256 true,
18257 ),
18258 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18259 (Vec::new(), None, false),
18260 ] {
18261 let dir = tempfile::tempdir().unwrap();
18262 let mut runner = rewrite_runner(dir.path(), true, agents);
18263 runner.state.config.roles.language_judge =
18264 role.map(|r| crate::config::AgentChoice::One(r.into()));
18265 let winner = runner.state.candidates[0].clone();
18266 let first = runner
18267 .guarded_pr_message(&winner, None, true)
18268 .await
18269 .unwrap();
18270 assert!(first.is_none(), "withheld text parks the run");
18271 assert!(runner.state.parked);
18272 assert_eq!(gate_questions(&runner), 1);
18273 let g = runner.state.github_text.as_ref().unwrap();
18274 assert_eq!(g.rewrite.is_some(), saved);
18275 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18276 }
18277 }
18278
18279 #[tokio::test]
18280 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18281 crate::run::pin_test_home();
18282 let dir = tempfile::tempdir().unwrap();
18283 let mut runner = runner_at(RunStatus::Gating);
18284 runner.state =
18285 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18286 runner.state.repo = dir.path().to_owned();
18287 runner.state.candidates[0].worktree = dir.path().to_owned();
18288 let mut author = spec("alpha");
18289 author.command = vec![
18290 "sh".into(),
18291 "-c".into(),
18292 "printf '%s' \"$REWRITE_REPLY\"".into(),
18293 ];
18294 author.env.insert(
18295 "REWRITE_REPLY".into(),
18296 format!(
18297 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18298 "English sentence. ".repeat(600)
18299 ),
18300 );
18301 runner.state.config.agents = vec![author];
18302 seed_public_summary(&mut runner);
18303 let winner = runner.state.candidates[0].clone();
18304 assert!(
18305 runner
18306 .guarded_pr_message(&winner, None, true)
18307 .await
18308 .unwrap()
18309 .is_none()
18310 );
18311 let record = runner.state.github_text.clone().unwrap();
18312 let rewrite = record.rewrite.clone().unwrap();
18313 assert_eq!(rewrite.failed, ["body-language"]);
18314 assert!(
18315 rewrite
18316 .body
18317 .ends_with(&format!("magi:run/{}", runner.state.id))
18318 );
18319 let store = ask::Questions::open();
18320 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18321 assert!(
18322 q.choices
18323 .contains(&crate::github_text::USE_REWRITE.to_owned())
18324 );
18325 for label in [
18326 "ORIGINAL",
18327 "FALLBACK",
18328 "REWRITE",
18329 "Rules still failed: body-language",
18330 "truncated",
18331 ] {
18332 assert!(q.detail.contains(label), "{label}");
18333 }
18334 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18335 &runner.state,
18336 &crate::github_text::artifact_name(&record.fingerprint),
18337 ))
18338 .unwrap();
18339 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18340 runner.state =
18342 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18343 store
18344 .update(&q.id, |q| {
18345 q.say("まだ日本語")?;
18346 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18347 })
18348 .unwrap();
18349 assert!(
18350 runner
18351 .guarded_pr_message(&winner, None, true)
18352 .await
18353 .unwrap()
18354 .is_none()
18355 );
18356 let retry = runner.state.github_text.as_ref().unwrap();
18357 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18358 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18359 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18360 assert!(
18361 q.posting_comparison
18362 .as_ref()
18363 .unwrap()
18364 .matches_detail(&q.detail)
18365 );
18366 store
18367 .update(&q.id, |q| {
18368 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18369 })
18370 .unwrap();
18371 let message = runner
18372 .guarded_pr_message(&winner, None, true)
18373 .await
18374 .unwrap()
18375 .unwrap();
18376 assert_eq!(message.title, rewrite.title);
18377 assert_eq!(message.body, rewrite.body);
18378 assert_eq!(
18379 runner.state.github_text.as_ref().unwrap().choice,
18380 Some(GithubTextChoice::Rewrite)
18381 );
18382 let again = runner
18383 .guarded_pr_message(&winner, None, true)
18384 .await
18385 .unwrap()
18386 .unwrap();
18387 assert_eq!(again.title, message.title);
18388 assert_eq!(again.body, message.body);
18389 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18390 assert!(
18391 !runner
18392 .state
18393 .events
18394 .iter()
18395 .any(|e| e.message.contains(&rewrite.title)
18396 || e.message.contains("English sentence."))
18397 );
18398 }
18399
18400 #[tokio::test]
18401 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18402 crate::run::pin_test_home();
18403 let dir = tempfile::tempdir().unwrap();
18404 let mut runner = runner_at(RunStatus::Gating);
18405 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18406 runner.state.repo = dir.path().to_owned();
18407 runner.state.candidates[0].worktree = dir.path().to_owned();
18408 seed_public_summary(&mut runner);
18409 let winner = runner.state.candidates[0].clone();
18410 assert!(
18411 runner
18412 .guarded_pr_message(&winner, None, true)
18413 .await
18414 .unwrap()
18415 .is_none()
18416 );
18417 let record = runner.state.github_text.as_ref().unwrap();
18418 assert!(record.rewrite.is_none());
18419 let mut old = serde_json::to_value(record).unwrap();
18421 old.as_object_mut().unwrap().remove("rewrite");
18422 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18423 assert!(old.rewrite.is_none());
18424 let id = record.question.clone().unwrap();
18425 ask::Questions::open()
18426 .update(&id, |q| {
18427 q.choices.push(crate::github_text::USE_REWRITE.into());
18428 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18429 })
18430 .unwrap();
18431 let message = runner
18432 .guarded_pr_message(&winner, None, true)
18433 .await
18434 .unwrap()
18435 .unwrap();
18436 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18437 assert_eq!(
18438 runner.state.github_text.as_ref().unwrap().choice,
18439 Some(GithubTextChoice::Fallback)
18440 );
18441 }
18442
18443 #[tokio::test]
18444 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18445 crate::run::pin_test_home();
18446 let dir = tempfile::tempdir().unwrap();
18447 let mut runner = runner_at(RunStatus::Gating);
18448 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18449 runner.state.repo = dir.path().to_owned();
18450 runner.state.candidates[0].worktree = dir.path().to_owned();
18451 seed_public_summary(&mut runner);
18452 let winner = runner.state.candidates[0].clone();
18453 assert!(
18454 runner
18455 .guarded_pr_message(&winner, None, true)
18456 .await
18457 .unwrap()
18458 .is_none()
18459 );
18460 let store = ask::Questions::open();
18461 let find = |run: &str| {
18462 store
18463 .list()
18464 .into_iter()
18465 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18466 .unwrap()
18467 };
18468 let run = runner.state.id.clone();
18469 let q = find(&run);
18470 let fp = runner
18471 .state
18472 .github_text
18473 .as_ref()
18474 .unwrap()
18475 .fingerprint
18476 .clone();
18477 let name = crate::github_text::artifact_name(&fp);
18478 let path = crate::run::artifact_path(&runner.state, &name);
18479 assert!(q.detail.contains(&path.display().to_string()));
18480 assert!(q.detail.contains("title-language: title"));
18481 let saved = std::fs::read_to_string(&path).expect("artifact written");
18482 assert!(saved.contains("日本語のタイトル"));
18483 assert!(q.cwd.is_none());
18484 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18485 let say = |id: &str, text: &str, choice: &str| {
18487 store
18488 .update(id, |q| {
18489 q.thread.push(ask::Turn {
18490 who: ask::Who::Operator,
18491 body: text.to_owned(),
18492 at: jiff::Timestamp::now(),
18493 note: None,
18494 });
18495 q.answer(ask::Answer::Choice(choice.to_owned()))
18496 })
18497 .unwrap();
18498 };
18499 say(&q.id, "まだ日本語", "use my text");
18500 assert!(
18501 runner
18502 .guarded_pr_message(&winner, None, true)
18503 .await
18504 .unwrap()
18505 .is_none()
18506 );
18507 let second = find(&run);
18508 assert_ne!(second.id, q.id);
18509 assert!(second.detail.contains("did not pass"));
18510 assert!(second.detail.contains("日本語のタイトル"));
18513 assert!(!second.detail.contains("まだ日本語"));
18514 assert!(second.detail.contains(&path.display().to_string()));
18515 say(&second.id, "fix: retry failed requests", "use my text");
18516 let message = runner
18517 .guarded_pr_message(&winner, None, true)
18518 .await
18519 .unwrap()
18520 .expect("a vetted replacement is posted");
18521 assert_eq!(message.title, "fix: retry failed requests");
18522 assert_eq!(
18523 runner
18524 .state
18525 .github_text
18526 .as_ref()
18527 .unwrap()
18528 .chosen_title
18529 .as_deref(),
18530 Some("fix: retry failed requests")
18531 );
18532 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18533 let again = runner
18535 .guarded_pr_message(&winner, None, true)
18536 .await
18537 .unwrap()
18538 .unwrap();
18539 assert_eq!(again.title, message.title);
18540 assert_eq!(
18541 store
18542 .list()
18543 .into_iter()
18544 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18545 .count(),
18546 2
18547 );
18548 }
18549
18550 #[test]
18551 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18552 let mut state = state_with_summary(
18556 "add retries",
18557 "TITLE: fix(web): batch reads\n- reads run.json once",
18558 );
18559 state.config.graph.language = "ja".to_owned();
18560 let m = pr_message(&state, 'A');
18561 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18562 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18563
18564 let task = "今回やってほしいこと: results projector を直す";
18567 let mut state = state_with_summary(task, "- no title line");
18568 state.config.graph.language = "ja".to_owned();
18569 let m = pr_message(&state, 'A');
18570 assert_eq!(
18571 m.title,
18572 format!("chore: land candidate A of run {}", state.id)
18573 );
18574 assert!(
18575 m.body.contains(&format!(
18576 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18577 )),
18578 "{}",
18579 m.body
18580 );
18581 }
18582
18583 #[test]
18584 fn pr_message_scrubs_home_paths_and_addresses() {
18585 let state = state_with_summary(
18586 "fix it in /Users/someone/src/x",
18587 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18588 );
18589 let m = pr_message(&state, 'A');
18590 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18591 assert!(!m.body.contains(leak), "{}", m.body);
18592 }
18593 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18594 }
18595
18596 #[test]
18597 fn pr_message_survives_a_task_that_closes_details() {
18598 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18599 let m = pr_message(&state, 'A');
18600 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18601 }
18602
18603 #[test]
18604 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18605 let cmd = manual_merge_command(
18606 MergeStyle::Squash,
18607 Path::new("/repo"),
18608 "b",
18609 "fix: \"quoted\" $(x) `y`\n\nbody",
18610 );
18611 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18612 }
18613
18614 #[test]
18615 fn manual_merge_command_matches_the_configured_style() {
18616 let repo = Path::new("/repo");
18617 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18618
18619 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18620 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18621
18622 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18623 assert_eq!(
18624 squash,
18625 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18626 \"Merge magi run 0832 (candidate A)\""
18627 );
18628
18629 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18630 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18631 }
18632
18633 #[test]
18634 fn a_nudge_gets_a_quarter_of_the_budget() {
18635 assert_eq!(retry_budget(secs(1200), true), secs(300));
18637 assert_eq!(retry_budget(secs(3600), true), secs(900));
18638 }
18639
18640 #[test]
18641 fn a_resent_prompt_keeps_the_whole_budget() {
18642 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18645 assert_eq!(retry_budget(secs(60), false), secs(60));
18646 }
18647
18648 #[test]
18649 fn the_floor_never_exceeds_the_original_budget() {
18650 assert_eq!(retry_budget(secs(60), true), secs(60));
18654 assert_eq!(retry_budget(secs(480), true), secs(120));
18655 assert_eq!(retry_budget(secs(0), true), secs(0));
18656 }
18657
18658 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18659 agent::CommandEvidence {
18660 id: "item1".to_owned(),
18661 description: "cargo test".to_owned(),
18662 exit_code,
18663 result_summary: String::new(),
18664 source: "codex".to_owned(),
18665 }
18666 }
18667
18668 #[test]
18669 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18670 assert!(!has_unconfirmed_command(&[]));
18674 }
18675
18676 #[test]
18677 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18678 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18682 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18683 assert!(!has_unconfirmed_command(&[
18684 evidence(Some(0)),
18685 evidence(Some(101))
18686 ]));
18687 }
18688
18689 #[test]
18690 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18691 assert!(has_unconfirmed_command(&[
18692 evidence(Some(0)),
18693 evidence(None)
18694 ]));
18695 }
18696
18697 #[test]
18698 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18699 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18700 assert_eq!(
18701 verified_noop_claim(true, &[], text).as_deref(),
18702 Some("already fixed by b32cfc4, on main.")
18703 );
18704 }
18705
18706 #[test]
18707 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18708 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18711 assert!(verified_noop_claim(false, &[], text).is_none());
18712 }
18713
18714 #[test]
18715 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18716 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18717 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18718 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18720 }
18721
18722 #[test]
18723 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18724 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18725 }
18726
18727 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18730 runner.state.candidates = shape
18731 .iter()
18732 .enumerate()
18733 .map(|(i, &(empty, verified))| Candidate {
18734 index: i,
18735 label: (b'A' + i as u8) as char,
18736 agent: "sonnet".to_owned(),
18737 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18738 worktree: PathBuf::from(format!("/wt/{i}")),
18739 summary: String::new(),
18740 stat: String::new(),
18741 files: 0,
18742 commits: 0,
18743 empty,
18744 failed: None,
18745 verified_noop: verified.map(str::to_owned),
18746 duration_ms: 0,
18747 folded: false,
18748 })
18749 .collect();
18750 }
18751
18752 #[test]
18753 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18754 ask_test_home();
18755 let mut runner = runner_at(RunStatus::Implementing);
18756 set_candidates(
18757 &mut runner,
18758 &[
18759 (true, Some("already on main at b32cfc4")),
18760 (true, Some("same fix, see the existing test")),
18761 ],
18762 );
18763
18764 runner
18765 .after_implement()
18766 .expect("a verified no-op is not an error");
18767
18768 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18769 }
18770
18771 #[test]
18772 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18773 ask_test_home();
18774 let mut runner = runner_at(RunStatus::Implementing);
18775 set_candidates(
18779 &mut runner,
18780 &[(true, Some("already on main at b32cfc4")), (true, None)],
18781 );
18782
18783 let err = runner
18784 .after_implement()
18785 .expect_err("an unverified empty candidate must still fail the run");
18786
18787 assert!(
18788 err.to_string().contains("no candidate produced a change"),
18789 "{err}"
18790 );
18791 assert_eq!(runner.state.status, RunStatus::Failed);
18792 }
18793
18794 #[test]
18795 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18796 ask_test_home();
18797 let mut runner = runner_at(RunStatus::Implementing);
18798 set_candidates(&mut runner, &[(true, None), (true, None)]);
18799
18800 let err = runner
18801 .after_implement()
18802 .expect_err("no candidate declared anything; this is an ordinary failure");
18803
18804 assert!(
18805 err.to_string().contains("no candidate produced a change"),
18806 "{err}"
18807 );
18808 assert_eq!(runner.state.status, RunStatus::Failed);
18809 }
18810}
18811
18812#[derive(Debug)]
18815struct GraftRestoreFailed(String);
18816
18817impl std::fmt::Display for GraftRestoreFailed {
18818 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18819 write!(
18820 f,
18821 "could not restore the winner to its pre-graft commit: {}",
18822 self.0
18823 )
18824 }
18825}
18826
18827impl std::error::Error for GraftRestoreFailed {}
18828
18829#[cfg(test)]
18833mod resource_wait_tests {
18834 use super::*;
18835 use std::collections::BTreeMap;
18836
18837 fn sh(id: &str, script: &str) -> AgentSpec {
18838 AgentSpec {
18839 id: id.to_owned(),
18840 kind: crate::config::AgentKind::Command,
18841 model: None,
18842 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18843 extra_args: Vec::new(),
18844 env: BTreeMap::new(),
18845 prompt_delivery: None,
18846 }
18847 }
18848
18849 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18850 SeatJob {
18851 seat: SeatState::new("impl-A", &spec.id, 1),
18852 spec,
18853 cwd: dir.to_path_buf(),
18854 prompt: "do it".to_owned(),
18855 timeout,
18856 allow_write: true,
18857 sessions: false,
18858 artifacts: dir.join("art"),
18859 stem: "impl-A".to_owned(),
18860 handover: None,
18861 }
18862 }
18863
18864 fn state(resource_wait_secs: u64) -> RunState {
18865 let mut config = Config::default();
18866 config.graph.timeout_resource_wait = resource_wait_secs;
18867 RunState::new(
18868 PathBuf::from("/repo"),
18869 "main".to_owned(),
18870 "abc1234".to_owned(),
18871 "task".to_owned(),
18872 config,
18873 )
18874 }
18875
18876 fn hold(cache: &Path) -> crate::cache::Guard {
18877 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18878 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18879 crate::cache::AcquireOutcome::Acquired(g) => g,
18880 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18881 }
18882 }
18883
18884 async fn run(
18885 jobs: Vec<SeatJob>,
18886 sem: Arc<Semaphore>,
18887 cache: Option<&Path>,
18888 st: &mut RunState,
18889 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18890 let prompts = Prompts::default();
18891 let ctx = WaveCtx {
18892 run: "r",
18893 node: "implement",
18894 prompts: &prompts,
18895 cache,
18896 round: None,
18897 carry_seats: false,
18898 pauses: None,
18899 };
18900 wave(jobs, sem, &ctx, st, 0).await
18901 }
18902
18903 #[tokio::test]
18904 async fn verification_cache_wait_preserves_the_command_budget() {
18905 crate::run::pin_test_home();
18906 let tmp = tempfile::tempdir().unwrap();
18907 let cache = tmp.path().join("target");
18908 let guard = hold(&cache);
18909 let release = tokio::spawn(async move {
18910 tokio::time::sleep(Duration::from_millis(100)).await;
18911 guard.release();
18912 });
18913 let mut st = state(10);
18914 let budget = Duration::from_millis(50);
18915 let wait = repository_wait(&st, None);
18916 let (outcomes, _) = with_cache_lease(
18917 &mut st,
18918 Some(&cache),
18919 "gate",
18920 "gate",
18921 tmp.path(),
18922 "abc1234",
18923 budget,
18924 "test gate",
18925 &wait,
18926 |_, available| async move {
18927 assert_eq!(
18928 available, budget,
18929 "contention must not shorten verification"
18930 );
18931 (Vec::new(), false, Vec::new())
18932 },
18933 )
18934 .await
18935 .unwrap();
18936 release.await.unwrap();
18937 assert!(outcomes.is_empty(), "{outcomes:?}");
18938 assert!(
18939 st.events
18940 .iter()
18941 .any(|event| event.message.contains("waiting for"))
18942 );
18943 assert!(st.resource_wait.is_none());
18944 hold(&cache).release();
18946 }
18947
18948 #[tokio::test]
18949 async fn verification_cache_wait_expires_without_starting_commands() {
18950 crate::run::pin_test_home();
18951 let tmp = tempfile::tempdir().unwrap();
18952 let cache = tmp.path().join("target");
18953 let _held = hold(&cache);
18954 let mut st = state(0);
18955 let wait = repository_wait(&st, None);
18956 let (outcomes, _) = with_cache_lease(
18957 &mut st,
18958 Some(&cache),
18959 "gate",
18960 "gate",
18961 tmp.path(),
18962 "abc1234",
18963 Duration::from_secs(60),
18964 "test gate",
18965 &wait,
18966 |_, _| async { panic!("commands must not start without the cache lease") },
18967 )
18968 .await
18969 .unwrap();
18970 assert_eq!(outcomes.len(), 1);
18971 assert!(outcomes[0].resource_blocked);
18972 assert!(st.resource_wait.is_none());
18973 }
18974
18975 #[tokio::test]
18976 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
18977 crate::run::pin_test_home();
18978 let tmp = tempfile::tempdir().unwrap();
18979 let cache = tmp.path().join("target-a");
18980 let guard = hold(&cache);
18981 tokio::spawn(async move {
18983 tokio::time::sleep(Duration::from_millis(1500)).await;
18984 guard.release();
18985 });
18986 let mut st = state(60);
18987 let results = run(
18988 vec![job(
18989 sh("alpha", "echo ok"),
18990 tmp.path(),
18991 Duration::from_secs(1),
18992 )],
18993 Arc::new(Semaphore::new(1)),
18994 Some(&cache),
18995 &mut st,
18996 )
18997 .await;
18998 assert!(
18999 matches!(results[0].2, AgentOutcome::Ok(_)),
19000 "the wait must not be charged to the agent"
19001 );
19002 assert!(st.resource_block.is_none());
19003 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19004 assert!(st.resource_wait.is_none() && st.active.is_empty());
19005 }
19006
19007 #[tokio::test]
19008 #[cfg(unix)]
19009 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19010 use std::os::fd::AsRawFd as _;
19011 crate::run::pin_test_home();
19012 let tmp = tempfile::tempdir().unwrap();
19013 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19014 let common = tmp.path().join(".git");
19015 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19016 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19017 assert_eq!(
19018 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19019 0
19020 );
19021 let mut st = state(1);
19022 st.config.coordination = crate::config::test_coordination();
19023 let mut reader = job(
19024 sh("reader", "echo must-not-run"),
19025 tmp.path(),
19026 Duration::from_secs(30),
19027 );
19028 reader.allow_write = false;
19029 let results = run(
19030 vec![reader.clone()],
19031 Arc::new(Semaphore::new(1)),
19032 None,
19033 &mut st,
19034 )
19035 .await;
19036 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19037 assert_eq!(results[0].1.turns, 0);
19038 assert!(FailClass::of(&results[0].2).is_none());
19039 assert_eq!(
19040 st.resource_block.as_ref().unwrap().kind,
19041 "repository coordination"
19042 );
19043 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19044 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19045 drop(external);
19046 let editor = crate::repo_lock::SeatLease::for_checkout(
19047 tmp.path(),
19048 true,
19049 crate::config::test_coordination().as_ref(),
19050 )
19051 .await
19052 .unwrap();
19053 let pause = Pause::new();
19054 let interrupt = Pause::new();
19055 let prompts = Prompts::default();
19056 let ctx = WaveCtx {
19057 run: "r",
19058 node: "review",
19059 prompts: &prompts,
19060 cache: None,
19061 round: None,
19062 carry_seats: false,
19063 pauses: Some((&pause, &interrupt)),
19064 };
19065 let (results, ()) = tokio::join!(
19066 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19067 async {
19068 tokio::time::sleep(Duration::from_millis(50)).await;
19069 pause.park();
19070 }
19071 );
19072 assert!(
19073 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19074 );
19075 assert_eq!(results[0].1.turns, 0);
19076 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19077 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19078 drop(editor);
19079 }
19080
19081 #[tokio::test]
19082 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19083 crate::run::pin_test_home();
19084 let tmp = tempfile::tempdir().unwrap();
19085 let cache = tmp.path().join("target-b");
19086 let _held = hold(&cache);
19087 let mut st = state(1);
19088 let results = run(
19089 vec![job(
19090 sh("alpha", "echo ok"),
19091 tmp.path(),
19092 Duration::from_secs(1),
19093 )],
19094 Arc::new(Semaphore::new(1)),
19095 Some(&cache),
19096 &mut st,
19097 )
19098 .await;
19099 let (_, seat, out) = &results[0];
19100 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19101 assert!(wave_blocked(&results));
19102 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19103 assert_eq!(seat.turns, 0, "nothing was asked");
19104 assert!(
19105 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19106 "no agent was spawned"
19107 );
19108 assert!(FailClass::of(out).is_none(), "not a failure class");
19109 let block = st.resource_block.as_ref().expect("recorded for the resume");
19110 assert_eq!(block.kind, "shared build cache");
19111 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19112 assert!(st.active.is_empty());
19113 }
19114
19115 #[tokio::test]
19116 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19117 crate::run::pin_test_home();
19118 let tmp = tempfile::tempdir().unwrap();
19119 let mut st = state(1);
19120 let results = run(
19121 vec![job(
19122 sh("alpha", "echo ok"),
19123 tmp.path(),
19124 Duration::from_secs(1),
19125 )],
19126 Arc::new(Semaphore::new(0)),
19127 None,
19128 &mut st,
19129 )
19130 .await;
19131 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19132 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19133 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19134 }
19135
19136 #[tokio::test]
19137 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19138 crate::run::pin_test_home();
19139 let tmp = tempfile::tempdir().unwrap();
19140 let mut st = state(60);
19141 let results = run(
19142 vec![job(
19143 sh("alpha", "sleep 30"),
19144 tmp.path(),
19145 Duration::from_secs(1),
19146 )],
19147 Arc::new(Semaphore::new(1)),
19148 None,
19149 &mut st,
19150 )
19151 .await;
19152 let out = &results[0].2;
19153 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19154 let class = FailClass::of(out).expect("a failure");
19155 assert_eq!(class, FailClass::Timeout);
19156 assert!(should_hand_over(None, &class), "existing handover policy");
19157 assert!(st.resource_block.is_none());
19158 }
19159}