1use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35 AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36 ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44 self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50 BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51 DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52 GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53 MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54 QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55 Tally, VoteRecord, tail, write_artifact,
56};
57use crate::verdict::{
58 self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59 ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63const OUTPUT_TAIL: usize = 8_000;
65
66const EVENT_OUTPUT_TAIL: usize = 2_000;
69
70const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
73
74const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
91
92pub(crate) const STAGNANT_LIMIT: usize = 2;
106
107const BASE_SYNC_ROUNDS: usize = 4;
120
121const MAX_FIX_CONTINUATIONS: usize = 2;
137
138#[derive(Clone)]
144struct SeatJob {
145 spec: AgentSpec,
146 seat: SeatState,
147 cwd: PathBuf,
148 prompt: String,
149 timeout: Duration,
150 allow_write: bool,
151 sessions: bool,
152 artifacts: PathBuf,
153 stem: String,
154 handover: Option<String>,
159}
160
161enum AgentOutcome {
173 Ok(AgentOutput),
175 Quota(AgentOutput),
177 Dropped(AgentOutput),
180 Failed(String),
182 ResourceBlocked(String),
189}
190
191#[derive(Debug, Clone, Default)]
218pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
219
220impl Pause {
221 #[must_use]
223 pub fn new() -> Self {
224 Self::default()
225 }
226
227 pub fn park(&self) {
229 self.0.store(true, Ordering::SeqCst);
230 }
231
232 pub fn park_because(&self, reason: impl Into<String>) {
238 self.park_for(ParkCause::Interrupt {
239 reason: reason.into(),
240 });
241 }
242
243 pub fn park_for(&self, cause: ParkCause) {
246 let mut reason_guard = self
247 .1
248 .lock()
249 .unwrap_or_else(std::sync::PoisonError::into_inner);
250 if reason_guard.is_none() {
251 *reason_guard = Some(cause);
252 }
253 drop(reason_guard);
254 self.park();
255 }
256
257 #[must_use]
259 pub fn parked(&self) -> bool {
260 self.0.load(Ordering::SeqCst)
261 }
262
263 #[must_use]
265 pub fn reason(&self) -> Option<String> {
266 self.cause().map(|c| c.label())
267 }
268
269 #[must_use]
271 pub fn cause(&self) -> Option<ParkCause> {
272 self.1
273 .lock()
274 .unwrap_or_else(std::sync::PoisonError::into_inner)
275 .clone()
276 }
277}
278
279pub struct Runner {
281 pub state: RunState,
283 roles: ResolvedRoles,
284 sem: Arc<Semaphore>,
285 pause: Pause,
289 interrupt: Pause,
295}
296
297async fn review_base(
302 repo: &Path,
303 remote: &str,
304 base_branch: &str,
305 base_commit: &str,
306 branch: &str,
307) -> String {
308 review_base_checked(repo, remote, base_branch, base_commit, branch)
309 .await
310 .0
311}
312
313async fn review_base_checked(
318 repo: &Path,
319 remote: &str,
320 base_branch: &str,
321 base_commit: &str,
322 branch: &str,
323) -> (String, bool) {
324 let tracking = format!("{remote}/{base_branch}");
325 let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
326 if git::rev_exists(repo, &tracking).await
327 && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
328 && !mb.is_empty()
329 {
330 return (mb, fresh);
331 }
332 let mb = git::merge_base(repo, base_commit, branch)
333 .await
334 .ok()
335 .filter(|mb| !mb.is_empty())
336 .unwrap_or_else(|| base_commit.to_owned());
337 (mb, false)
338}
339
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
344 if !is_review_run(state) {
345 return;
346 }
347 let base = review_base(
348 &state.repo,
349 &state.config.merge.remote,
350 &state.base_branch,
351 &state.base_commit,
352 branch,
353 )
354 .await;
355 if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
356 && !subjects.is_empty()
357 && state.reviewed_commits.as_ref() != Some(&subjects)
358 {
359 state.reviewed_commits = Some(subjects);
360 state.save().ok();
361 }
362}
363
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
368 let (base, trusted) = review_base_checked(
369 &state.repo,
370 &state.config.merge.remote,
371 &state.base_branch,
372 &state.base_commit,
373 branch,
374 )
375 .await;
376 if !trusted {
377 return None;
378 }
379 git::subjects(&state.repo, &state.base_commit, &base)
380 .await
381 .ok()
382}
383
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
388 let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
389 if is_leak(computed) {
390 return false;
391 }
392 let cur = current.trim();
393 cur.is_empty()
394 || cur.starts_with(REVIEW_PROMPT_OPENING)
395 || cur.starts_with("chore: land candidate")
396 || cur.starts_with("magi: candidate")
397 || is_leak(cur)
398}
399
400async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
429 let tracking = format!("{remote}/{branch}");
430 let fetched = git::fetch(repo, remote, branch).await;
431 let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
432 let local_exists = git::branch_exists(repo, branch).await?;
433 if !fresh {
434 if !local_exists {
435 bail!("no branch `{branch}` in {} or on {remote}", repo.display());
436 }
437 tracing::warn!(
438 "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
439 );
440 return Ok(());
441 }
442 let remote_sha = git::rev_parse(repo, &tracking).await?;
443 if !local_exists {
444 git::git(repo, &["branch", branch, &tracking]).await?;
445 return Ok(());
446 }
447 let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
448 if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
449 return Ok(());
450 }
451 if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
452 match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
458 .await?
459 {
460 crate::reconcile::Reconciliation::Pushed => {
461 tracing::warn!(
462 "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
463 short(&local_sha),
464 short(&remote_sha)
465 );
466 return Ok(());
467 }
468 crate::reconcile::Reconciliation::Placeholder => {}
469 crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
470 }
471 }
472 let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
473 if !out.ok() {
474 bail!(
475 "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
476 short(&local_sha),
477 short(&remote_sha),
478 out.stderr
479 );
480 }
481 tracing::warn!(
482 "local `{branch}` was stale: fast-forwarded {} -> {}",
483 short(&local_sha),
484 short(&remote_sha)
485 );
486 Ok(())
487}
488
489async fn resolve_base(
490 repo: &Path,
491 base_branch: &str,
492 remote: &str,
493 coordination: Option<&crate::config::Coordination>,
494) -> Result<crate::run::FetchedBase> {
495 let _lease = crate::repo_lock::SeatLease::operation_with(
496 repo,
497 coordination,
498 &crate::repo_lock::WaitPolicy::current(),
499 )
500 .await?;
501 let tracking = format!("{remote}/{base_branch}");
502 let fetched = git::fetch(repo, remote, base_branch).await;
503 if let Ok(out) = &fetched
504 && out.ok()
505 && git::rev_exists(repo, &tracking).await
506 {
507 return Ok(crate::run::FetchedBase {
508 remote: remote.to_owned(),
509 branch: base_branch.to_owned(),
510 commit: git::rev_parse(repo, &tracking).await?,
511 fetched_at: jiff::Timestamp::now(),
512 });
513 }
514 let why = match &fetched {
515 Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
516 Ok(_) => format!("{remote} has no {base_branch}"),
517 Err(e) => e.to_string(),
518 };
519 bail!(
522 "cannot read {tracking} ({why}); refusing to branch off the local \
523 `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
524 base / remote in magi.toml. This is launcher preparation: check the \
525 launcher's Git administration access, network and repository coordination \
526 before retrying; an unattended seat cannot approve those permissions"
527 )
528}
529
530struct FixClaim {
547 path: PathBuf,
548}
549
550impl FixClaim {
551 fn acquire(dir: &Path) -> Result<Self> {
552 std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
553 let path = dir.join("fix.lock");
554 let retry = crate::lock_file::Retry::new(None);
555 match Self::create(&path, retry) {
556 Ok(claim) => Ok(claim),
557 Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
558 if Self::reclaim_if_dead(&path) {
559 Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
560 } else {
561 bail!(
562 "another repair or `magi fix` is already running for this run ({} exists)",
563 path.display()
564 )
565 }
566 }
567 Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
568 }
569 }
570
571 fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
572 let mut f = retry.create(path)?;
573 use std::io::Write as _;
574 writeln!(f, "{}", std::process::id())?;
576 Ok(Self {
577 path: path.to_owned(),
578 })
579 }
580
581 fn reclaim_if_dead(path: &Path) -> bool {
585 let dead = std::fs::read_to_string(path)
586 .ok()
587 .and_then(|body| body.trim().parse::<u32>().ok())
588 .is_some_and(|pid| !crate::proc::pid_alive(pid));
589 dead && std::fs::remove_file(path).is_ok()
590 }
591}
592
593impl Drop for FixClaim {
594 fn drop(&mut self) {
595 let _ = std::fs::remove_file(&self.path);
596 }
597}
598
599struct DriverExecution<'a> {
602 runner: &'a mut Runner,
603 driver: crate::driver::DriverGuard,
604}
605
606impl Drop for DriverExecution<'_> {
607 fn drop(&mut self) {
608 self.driver.release_local(&mut self.runner.state);
609 }
610}
611
612impl Runner {
613 pub async fn start(
615 repo: &Path,
616 instruction: String,
617 config: Config,
618 origin: Origin,
619 ) -> Result<Self> {
620 Self::start_naming(repo, instruction, "", config, origin).await
621 }
622
623 pub async fn start_naming(
626 repo: &Path,
627 instruction: String,
628 _also_scan: &str,
629 config: Config,
630 origin: Origin,
631 ) -> Result<Self> {
632 let repo = git::toplevel(repo).await?;
633 let missing = agent::missing_programs(&config.agents);
634 if !missing.is_empty() {
635 bail!(
636 "these agent programs are not on PATH: {}. Fix the roster in \
637 magi.toml or install them.",
638 missing.join(", ")
639 );
640 }
641 let base_branch =
642 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
643 .await?;
644 let fetched_base = resolve_base(
645 &repo,
646 &base_branch,
647 &config.merge.remote,
648 config.coordination.as_ref(),
649 )
650 .await?;
651 let base_commit = fetched_base.commit.clone();
652 let roles = config.resolve_roles()?;
653 let max_parallel = config.graph.max_parallel.max(1);
654 let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
655 state.fetched_base = Some(fetched_base);
659 state.origin = Some(origin);
660 state.event("start", format!("run {} created", state.id));
661 state.save()?;
662 Ok(Self {
663 state,
664 roles,
665 sem: Arc::new(Semaphore::new(max_parallel)),
666 pause: Pause::new(),
667 interrupt: Pause::new(),
668 })
669 }
670
671 pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
685 Self::review_taking_over(repo, branch, config, None, origin).await
686 }
687
688 pub async fn review_taking_over(
695 repo: &Path,
696 branch: &str,
697 config: Config,
698 takeover: Option<crate::handover::Takeover>,
699 origin: Origin,
700 ) -> Result<Self> {
701 let repo = git::toplevel(repo).await?;
702 let missing = agent::missing_programs(&config.agents);
703 if !missing.is_empty() {
704 bail!(
705 "these agent programs are not on PATH: {}. Fix the roster in \
706 magi.toml or install them.",
707 missing.join(", ")
708 );
709 }
710 let base_branch =
711 git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
712 .await?;
713 if base_branch == branch {
714 bail!("`{branch}` is the base branch; there is nothing to review against");
715 }
716 let fetched_base = resolve_base(
717 &repo,
718 &base_branch,
719 &config.merge.remote,
720 config.coordination.as_ref(),
721 )
722 .await?;
723 let base_commit = fetched_base.commit.clone();
724
725 let roles = config.resolve_roles()?;
726 let max_parallel = config.graph.max_parallel.max(1);
727 let mut state = RunState::new(
728 repo.clone(),
729 base_branch,
730 base_commit.clone(),
731 String::new(),
732 config,
733 );
734 state.fetched_base = Some(fetched_base);
735 state.origin = Some(origin);
736
737 let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
744 earlier: Vec::new(),
745 home: crate::run::home(),
746 choice: None,
747 task_instruction: None,
748 });
749 let wait = repository_wait(&state, None);
750 let coordination_config = state.config.coordination.clone();
751 let coordination =
752 crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
753 .await?;
754 let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
755 if let Some(released) = &released {
756 state.event(
757 "release",
758 format!(
759 "took `{branch}` over from run {}: its worktree was released: {}",
760 crate::run::short_of(&released.old_id),
761 released.audit
762 ),
763 );
764 }
765 if let Some(choice) = takeover.choice.as_ref()
769 && let Err(e) =
770 crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
771 .await
772 {
773 if let Some(released) = &released {
774 released.restore(&repo, branch).await;
775 }
776 return Err(e.context("applying the owner's answer about the diverged branch"));
777 }
778 for id in takeover.earlier.iter().rev() {
784 if let Ok(old) = RunState::load_under(id, &takeover.home)
785 && let Some(head) = old.pr_head.clone()
786 && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
787 {
788 state.pr_head = Some(head);
789 break;
790 }
791 }
792 drop(coordination);
793 let opened = Self::open_review(
794 &repo,
795 branch,
796 state,
797 roles,
798 max_parallel,
799 base_commit,
800 &takeover,
801 )
802 .await;
803 if opened.is_err()
804 && let Some(released) = &released
805 {
806 let _coordination = crate::repo_lock::SeatLease::operation_with(
807 &repo,
808 coordination_config.as_ref(),
809 &wait,
810 )
811 .await?;
812 released.restore(&repo, branch).await;
813 }
814 opened
815 }
816
817 pub async fn recover_pr(
819 repo: &Path,
820 task: &str,
821 selection: crate::pr_recovery::Selection,
822 mut config: Config,
823 ) -> Result<Self> {
824 selection.configure(&mut config)?;
825 selection.check_task(task, &selection.instruction)?;
826 let base_branch =
827 git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
828 .await?;
829 let fetched_base = resolve_base(
830 repo,
831 &base_branch,
832 &config.merge.remote,
833 config.coordination.as_ref(),
834 )
835 .await?;
836 let base_commit = fetched_base.commit.clone();
837 selection
838 .verify(repo, &config.merge.remote, &base_branch, &selection.head)
839 .await?;
840 Box::pin(selection.verify_source(repo)).await?;
841 if let Some(proof) = &selection.source_proof {
842 anyhow::ensure!(
843 git::is_ancestor(repo, &proof.new_base, &base_commit).await,
844 "source rebase base is outside the fetched landing base"
845 );
846 }
847 let roles = config.resolve_roles()?;
848 let max_parallel = config.graph.max_parallel.max(1);
849 let mut state = RunState::new(
850 repo.to_path_buf(),
851 base_branch,
852 base_commit.clone(),
853 selection.instruction.clone(),
854 config,
855 );
856 state.fetched_base = Some(fetched_base);
857 state.origin = Some(crate::run::Origin::queue(task));
858 state.pr_head = Some(selection.branch.clone());
859 state.pr_recovery = Some(selection.clone());
860 crate::pr_head::check_predecessors(&state, &selection.branch).await?;
861 Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
862 }
863
864 async fn open_selected(
865 repo: &Path,
866 state: RunState,
867 roles: ResolvedRoles,
868 max_parallel: usize,
869 base_commit: String,
870 selection: crate::pr_recovery::Selection,
871 ) -> Result<Self> {
872 let mut state = state;
873 let branch = state.branch_for('A');
874 let wait = repository_wait(&state, None);
875 let coordination = coordinate_operation(&mut state, repo, &wait).await?;
876 git::git(repo, &["branch", &branch, selection.source()]).await?;
877 drop(coordination);
878 state.event(
879 "recovery",
880 format!(
881 "selected implementation {} for {} (public lease {})",
882 selection.source(),
883 selection.url,
884 selection.head
885 ),
886 );
887 let takeover = crate::handover::Takeover {
888 earlier: Vec::new(),
889 home: crate::run::home(),
890 choice: None,
891 task_instruction: Some(selection.instruction.clone()),
892 };
893 state.save()?;
894 let id = state.id.clone();
895 let opened = Self::open_review(
896 repo,
897 &branch,
898 state,
899 roles,
900 max_parallel,
901 base_commit,
902 &takeover,
903 )
904 .await;
905 match opened {
906 Ok(runner) => Ok(runner),
907 Err(e) => {
908 if let Ok(mut partial) = RunState::load(&id) {
909 partial.status = RunStatus::Blocked;
910 partial.driver_exited = true;
911 partial.event(
912 "recovery",
913 format!("startup failed; retained source and candidate {branch}: {e:#}"),
914 );
915 partial.save()?;
916 }
917 Err(crate::pr_recovery::StartupFailed {
918 run: id,
919 cause: format!("{e:#}"),
920 }
921 .into())
922 }
923 }
924 }
925
926 async fn inherit(
931 repo: &Path,
932 branch: &str,
933 base_commit: &str,
934 takeover: &crate::handover::Takeover,
935 ) -> Option<crate::recovery::RecoveryRecord> {
936 use crate::recovery::{Relation, assess};
937 let newest = takeover.earlier.last()?;
938 let mut chosen = None;
939 for id in takeover.earlier.iter().rev() {
940 if let Ok(old) = RunState::load_under(id, &takeover.home)
941 && !old.reviews.is_empty()
942 && (old.winner().is_some_and(|w| w.branch == branch)
943 || old.pr_head.as_deref() == Some(branch))
944 {
945 chosen = Some(old);
946 break;
947 }
948 }
949 let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
950 .await
951 .unwrap_or_default();
952 let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
953 Some(last) if last.head == tip => Relation::Same,
954 Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
955 _ => Relation::Diverged,
956 };
957 let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
958 Some(assess(from, chosen.as_ref(), base_commit, relation))
959 }
960
961 async fn open_review(
964 repo: &Path,
965 branch: &str,
966 mut state: RunState,
967 roles: ResolvedRoles,
968 max_parallel: usize,
969 base_commit: String,
970 takeover: &crate::handover::Takeover,
971 ) -> Result<Self> {
972 let wait = repository_wait(&state, None);
973 let _coordination = coordinate_operation(&mut state, repo, &wait).await?;
974 if state.pr_recovery.is_none() {
975 sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
976 }
977 let start = review_base(
980 repo,
981 &state.config.merge.remote,
982 &state.base_branch,
983 &base_commit,
984 branch,
985 )
986 .await;
987 let log = git::log_oneline(repo, &start, branch)
988 .await
989 .unwrap_or_default();
990 let log_text = if log.trim().is_empty() {
991 "(no commit messages)"
992 } else {
993 log.trim()
994 };
995 let mut instruction = format!(
996 "Review the work already on branch `{branch}`. There is no task \
997 statement: what the change claims to do is whatever its commits \
998 say.\n\n{log_text}"
999 );
1000 if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1004 if let Some(task) = takeover
1005 .task_instruction
1006 .as_deref()
1007 .filter(|t| !t.trim().is_empty())
1008 && rec.reviewed_head.is_some()
1009 {
1010 instruction = format!(
1011 "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1012 task.trim()
1013 );
1014 rec.criteria_inherited = true;
1015 }
1016 state.event("recovery", rec.summary());
1017 state.recovery = Some(rec);
1018 }
1019 if let Some(selected) = &state.pr_recovery {
1020 instruction = selected.instruction.clone();
1021 }
1022 state.task_instruction = takeover.task_instruction.clone();
1023 state.instruction = instruction;
1024 state.reviewed_commits = Some(
1025 git::subjects(repo, &start, branch)
1026 .await
1027 .unwrap_or_default(),
1028 );
1029
1030 for id in &takeover.earlier {
1033 if let Ok(old) = RunState::load_under(id, &takeover.home)
1034 && old.repo == state.repo
1035 && (old.pr_head.as_deref() == Some(branch)
1036 || old.winner().is_some_and(|w| w.branch == branch))
1037 {
1038 for hold in old.branch_owner_holds {
1039 if !state.branch_owner_holds.contains(&hold) {
1040 state.event(
1041 "approval-policy",
1042 format!(
1043 "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1044 hold.control.key(),
1045 hold.file,
1046 hold.head
1047 ),
1048 );
1049 state.branch_owner_holds.push(hold);
1050 }
1051 }
1052 }
1053 }
1054 let reviewed_head = git::rev_parse(repo, branch).await?;
1055 state.observe_branch_owner_holds(&reviewed_head).await;
1056
1057 let worktree = state.worktree_root().join("under-review");
1060 if let Some(parent) = worktree.parent() {
1061 tokio::fs::create_dir_all(parent).await.ok();
1062 }
1063 let path = worktree.to_string_lossy().to_string();
1064 git::git(repo, &["worktree", "add", &path, branch])
1065 .await
1066 .with_context(|| {
1067 format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1068 })?;
1069
1070 let setup = match ensure_setup_config(&mut state, repo).await {
1071 Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1072 Err(e) => Err(e),
1073 };
1074 if let Err(e) = setup {
1075 git::worktree_remove_clean(repo, &worktree).await.ok();
1076 return Err(e);
1077 }
1078 let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1079 .await
1080 .unwrap_or(0);
1081 if commits == 0 {
1082 git::worktree_remove_clean(repo, &worktree).await.ok();
1083 bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1084 }
1085 let files = git::changed_files(&worktree, &base_commit, "HEAD")
1086 .await
1087 .map(|f| f.len())
1088 .unwrap_or(0);
1089 if files == 0
1090 && let (Ok(head_tree), Ok(base_tree)) = (
1091 git::tree_of(&worktree, "HEAD").await,
1092 git::tree_of(&worktree, &base_commit).await,
1093 )
1094 && head_tree == base_tree
1095 {
1096 let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1097 git::worktree_remove_clean(repo, &worktree).await.ok();
1098 bail!(
1099 "`{branch}` at {} has a tree identical to base {}; this usually means \
1100 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1101 against `{}/{branch}`) rather than an empty change",
1102 short(&head),
1103 short(&base_commit),
1104 state.config.merge.remote
1105 );
1106 }
1107 let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1108 .await
1109 .unwrap_or_default();
1110
1111 state.candidates.push(Candidate {
1112 index: 0,
1113 label: 'A',
1114 agent: EXISTING_BRANCH.to_owned(),
1117 branch: branch.to_owned(),
1118 worktree,
1119 summary: String::new(),
1120 stat,
1121 files,
1122 commits,
1123 empty: false,
1124 failed: None,
1125 verified_noop: None,
1126 duration_ms: 0,
1127 folded: false,
1128 });
1129 state.tally = Some(Tally {
1130 first_choice: BTreeMap::from([('A', 0)]),
1131 borda: BTreeMap::new(),
1132 winner: 'A',
1133 rankings: 0,
1134 unanimous_initial: false,
1135 deliberated: false,
1136 changed_votes: 0,
1137 unanimous_final: false,
1138 tie_break: None,
1139 judges: 0,
1143 present: 0,
1144 quorum: 0,
1145 met_quorum: true,
1146 uncontested: Some("review-only run: nothing competed".to_owned()),
1147 });
1148 state.status = RunStatus::Reviewing;
1149 state.prep_complete = Some(true);
1150 state.event(
1151 "start",
1152 format!(
1153 "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1154 state.id
1155 ),
1156 );
1157 state.save()?;
1158 Ok(Self {
1159 state,
1160 roles,
1161 sem: Arc::new(Semaphore::new(max_parallel)),
1162 pause: Pause::new(),
1163 interrupt: Pause::new(),
1164 })
1165 }
1166
1167 pub fn resume(id: &str) -> Result<Self> {
1169 let state = RunState::load(id)?;
1170 crate::pr_head::ensure_owner(&state)?;
1171 if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1172 bail!(
1173 "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1174 );
1175 }
1176 if let Some(to) = &state.released_to {
1177 bail!(
1178 "run {} cannot be resumed: its worktree was released to run {}",
1179 state.short(),
1180 crate::run::short_of(to)
1181 );
1182 }
1183 let roles = state.config.resolve_roles()?;
1184 if let Some(id) = &state.config.graph.selected_implementer
1185 && state.candidates.iter().any(|c| c.agent != *id)
1186 {
1187 bail!(
1188 "replacement implementer pin does not match the saved candidates; task must stay held"
1189 );
1190 }
1191 let max_parallel = state.config.graph.max_parallel.max(1);
1192 Ok(Self {
1193 state,
1194 roles,
1195 sem: Arc::new(Semaphore::new(max_parallel)),
1196 pause: Pause::new(),
1197 interrupt: Pause::new(),
1198 })
1199 }
1200
1201 pub async fn execute(&mut self) -> Result<()> {
1208 if let Some(selection) = &self.state.pr_recovery
1209 && (selection.source_proof.is_some() || selection.source_run.is_some())
1210 {
1211 Box::pin(selection.verify_source(&self.state.repo)).await?;
1212 }
1213 let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1216 || self.state.land_approval.is_some()
1217 || self.state.merge_decision.is_some()
1218 || !self.state.merge_repairs.is_empty()
1219 || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1220 {
1221 let claim = FixClaim::acquire(&self.state.dir())?;
1222 let disk = RunState::load(&self.state.id)?;
1223 let checkpoint = |s: &RunState| {
1224 (
1225 s.merge_repairs.len(),
1226 s.merge_repairs
1227 .last()
1228 .map(|r| (r.question.clone(), r.phase)),
1229 s.land_approval
1230 .as_ref()
1231 .map(|a| (a.question.clone(), a.head.clone())),
1232 )
1233 };
1234 if checkpoint(&disk) != checkpoint(&self.state) {
1235 bail!(
1236 "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1237 );
1238 }
1239 Some(claim)
1240 } else {
1241 None
1242 };
1243 let mut execution = self.claim_driver().await?;
1244 let runner = &mut *execution.runner;
1245 let wait = runner.repository_wait();
1246 let result = wait.scope(Box::pin(runner.execute_graph())).await;
1249 let result = match result {
1250 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1251 record_repository_block(&mut runner.state, &e);
1252 runner.park_here()?;
1253 Ok(())
1254 }
1255 other => other,
1256 };
1257 execution.driver.finish(&mut runner.state).await?;
1258 let ended = if result.is_err() {
1259 Some(crate::notices::run_stopped(&runner.state.id, &runner.state))
1260 } else {
1261 crate::notices::run_ended(&runner.state)
1262 };
1263 if let Some(notice) = ended {
1264 crate::notices::raise(notice);
1265 }
1266 result
1267 }
1268
1269 fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1270 repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1271 }
1272
1273 async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1274 let wait = self.repository_wait();
1275 coordinate_operation(&mut self.state, cwd, &wait).await
1276 }
1277
1278 async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1280 let owner = match crate::pr_head::owner_lock_wait(
1281 &crate::run::home(),
1282 &self.state.id,
1283 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1284 )
1285 .await
1286 {
1287 Ok(owner) => owner,
1288 Err(error) => {
1289 let detail = format!("worktree maintenance or ownership update: {error:#}");
1290 self.state.resource_block = Some(crate::run::ResourceBlock {
1294 node: "start".into(),
1295 kind: "worktree maintenance".into(),
1296 detail: detail.clone(),
1297 at: Timestamp::now(),
1298 });
1299 return Err(crate::repo_lock::ResourceBlocked(detail).into());
1300 }
1301 };
1302 let disk = RunState::load(&self.state.id)?;
1303 crate::pr_head::ensure_owner(&disk)?;
1304 if disk.liveness(false) == crate::run::Liveness::Live {
1305 bail!("run already has a live driver; wait for its execution to end");
1306 }
1307 if let Some(to) = &disk.released_to {
1308 bail!(
1309 "run {} cannot continue: its worktree was released to run {}",
1310 self.state.short(),
1311 crate::run::short_of(to)
1312 );
1313 }
1314 let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1315 let mut execution = DriverExecution {
1316 runner: self,
1317 driver,
1318 };
1319 let _owner = owner;
1322 let state = &mut execution.runner.state;
1323 state.parked = false;
1328 state.resource_wait = None;
1329 state.resource_block = None;
1330 state.clear_active();
1337 let pid = std::process::id();
1352 state.driver_pid = Some(pid);
1353 state.driver_started_at = crate::proc::process_started_at(pid);
1354 state.driver_exited = false;
1355 state.begin_pass();
1356 execution.driver.bind(state);
1357 state.save()?;
1358 execution.driver.record(state)?;
1359 Ok(execution)
1360 }
1361
1362 async fn execute_graph(&mut self) -> Result<()> {
1363 self.state.configure_recovery_resume()?;
1364 self.persist_candidates().await?;
1367 if self
1368 .state
1369 .merge_decision
1370 .as_ref()
1371 .is_some_and(|decision| !decision.closed)
1372 && self.state.status != RunStatus::Landing
1373 {
1374 if crate::merge_decision::resumable_hold(&self.state) {
1375 self.state.merge = Some(MergeOutcome {
1376 mode: MergeMode::Pr,
1377 ok: false,
1378 detail: self
1379 .state
1380 .merge_decision
1381 .as_ref()
1382 .unwrap()
1383 .target
1384 .pr
1385 .clone(),
1386 empty: false,
1387 });
1388 self.run_land().await?;
1389 self.settle_questions();
1390 return Ok(());
1391 }
1392 bail!(
1393 "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1394 );
1395 }
1396 if self.state.config.graph.selected_implementer.is_some()
1397 && self.state.status == RunStatus::Blocked
1398 && self.state.tally.is_none()
1399 && self.state.candidate_preservation.is_empty()
1400 && self.state.interrupted_implementations.is_empty()
1401 {
1402 bail!(
1403 "selected implementation is held; explicitly release the owning task to retry it"
1404 );
1405 }
1406
1407 if self.state.merge_repairs.last().is_some_and(|r| {
1408 matches!(
1409 r.phase,
1410 crate::run::MergeRepairPhase::Requested
1411 | crate::run::MergeRepairPhase::Fixing
1412 | crate::run::MergeRepairPhase::Reviewing
1413 )
1414 }) {
1415 if self.repair_merge_review().await? {
1416 self.run_land().await?;
1417 }
1418 self.settle_questions();
1419 return Ok(());
1420 }
1421 if self.state.pr_publication_pending {
1424 self.merge().await?;
1425 if self.resource_halt()? {
1426 return Ok(());
1427 }
1428 self.state.save()?;
1429 self.settle_questions();
1430 return Ok(());
1431 }
1432 if self.state.status == RunStatus::Stalled {
1445 if self.recover_stall().await? {
1446 self.finish_after_tally().await?;
1447 } else {
1448 self.state.save()?;
1450 }
1451 return Ok(());
1452 }
1453 if self.state.status == RunStatus::Landing
1463 || crate::handover::landing_checkpoint(&self.state)
1464 {
1465 self.state.land_sync_pending = true;
1466 self.state.save()?;
1467 self.run_land().await?;
1468 self.settle_questions();
1473 return Ok(());
1474 }
1475 if self.state.github_text.is_some()
1480 && self.state.merge.is_none()
1481 && self.state.status == RunStatus::Gating
1482 {
1483 self.merge().await?;
1484 if self.resource_halt()? {
1485 return Ok(());
1486 }
1487 self.state.save()?;
1488 return Ok(());
1489 }
1490 if let Err(e) = self.prep().await {
1491 if e.is::<crate::repo_lock::ResourceBlocked>() {
1492 return Err(e);
1493 }
1494 self.state.status = RunStatus::Blocked;
1495 self.state
1496 .prep_failure
1497 .get_or_insert_with(|| format!("{e:#}"));
1498 self.state
1499 .event("prep", format!("preparation stopped: {e:#}"));
1500 self.state.save()?;
1501 return Err(e);
1502 }
1503 if self.park_here()? {
1504 return Ok(());
1505 }
1506 self.advise().await?;
1507 if self.park_here()? {
1508 return Ok(());
1509 }
1510 self.implement().await?;
1511 if self.park_here()? {
1512 return Ok(());
1513 }
1514 if self.state.status == RunStatus::VerifiedNoop {
1518 return Ok(());
1519 }
1520 self.judge().await?;
1521 if self.park_here()? {
1522 return Ok(());
1523 }
1524 self.deliberate().await?;
1525 if self.park_here()? {
1526 return Ok(());
1527 }
1528 self.vote().await?;
1529 if self.park_here()? {
1530 return Ok(());
1531 }
1532 self.tally()?;
1533 if self.state.status == RunStatus::Stalled {
1538 self.state.save()?;
1542 return Ok(());
1543 }
1544 self.finish_after_tally().await?;
1545 Ok(())
1546 }
1547
1548 fn stop_for_resource(&mut self) -> Result<()> {
1553 if self.state.resource_block.is_some() && !(self.pause.parked() || self.interrupt.parked())
1555 {
1556 self.state.status = RunStatus::Blocked;
1557 }
1558 self.state.save()
1559 }
1560
1561 fn resource_halt(&mut self) -> Result<bool> {
1563 if self.state.resource_block.is_none() {
1564 return Ok(false);
1565 }
1566 self.stop_for_resource()?;
1567 Ok(true)
1568 }
1569
1570 fn park_here(&mut self) -> Result<bool> {
1577 let parking = self.pause.parked() || self.interrupt.parked();
1578 if !parking && self.state.resource_block.is_some() {
1579 self.stop_for_resource()?;
1580 return Ok(true);
1581 }
1582 if !self.pause.parked() && !self.interrupt.parked() {
1587 return Ok(false);
1588 }
1589 let cause = self.interrupt.cause().or_else(|| self.pause.cause());
1590 let why = match cause.as_ref().map(ParkCause::label) {
1591 Some(reason) => format!(
1592 "parked after `{}` ({reason}) — resume to carry on from here",
1593 self.state.status.as_str()
1594 ),
1595 None => format!(
1596 "parked after `{}` — resume to carry on from here",
1597 self.state.status.as_str()
1598 ),
1599 };
1600 self.state.event("park", why);
1601 self.state
1602 .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
1603 self.state.parked = true;
1604 self.state.save()?;
1605 Ok(true)
1606 }
1607
1608 pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
1610 if self.state.prep_failure.is_none() {
1611 return Ok(());
1612 }
1613 let failed_at = self
1614 .state
1615 .events
1616 .iter()
1617 .rev()
1618 .find(|event| event.node == "prep")
1619 .map_or(self.state.updated_at, |event| event.at);
1620 anyhow::ensure!(
1621 approval_at > failed_at,
1622 "preparation retry requires a new explicit release after the failure"
1623 );
1624 self.state.prep_failure = None;
1625 self.state.prep_complete = Some(false);
1626 self.state
1627 .event("prep-retry", "operator approved retry of preparation");
1628 Ok(())
1629 }
1630
1631 pub fn on_pause(&mut self, pause: Pause) {
1633 self.pause = pause;
1634 }
1635
1636 pub fn watch_interrupt(&mut self, pause: Pause) {
1642 self.interrupt = pause;
1643 }
1644
1645 fn settle_questions(&mut self) {
1665 if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1666 tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1667 }
1668 }
1669
1670 async fn finish_after_tally(&mut self) -> Result<()> {
1673 self.graft().await?;
1676 if self.resource_halt()? {
1677 return Ok(());
1678 }
1679 self.fold_losers().await?;
1680 self.sync_to_base().await?;
1685 if self.resource_halt()? {
1686 return Ok(());
1687 }
1688 if self.state.status == RunStatus::AlreadyInBase {
1689 return Ok(());
1690 }
1691 self.review_loop().await?;
1692 if self.resource_halt()? {
1693 return Ok(());
1694 }
1695 self.sync_to_base().await?;
1696 if self.resource_halt()? {
1697 return Ok(());
1698 }
1699 if self.state.status == RunStatus::AlreadyInBase {
1700 return Ok(());
1701 }
1702 self.gate().await?;
1703 self.merge().await?;
1704 if self.resource_halt()? {
1705 return Ok(());
1706 }
1707 self.state.save()?;
1708 Ok(())
1709 }
1710
1711 async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
1718 let result = match ensure_setup_config(&mut self.state, repo).await {
1719 Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
1720 Err(e) => Err(e),
1721 };
1722 if let Err(e) = result {
1723 git::worktree_remove_clean(repo, wt).await.ok();
1724 if in_prep {
1725 self.rollback_prep(repo).await;
1726 }
1727 self.state.status = RunStatus::Blocked;
1728 self.state.event(
1729 "setup",
1730 format!("worktree setup failed in {}: {e:#}", wt.display()),
1731 );
1732 self.state.save()?;
1733 return Err(e);
1734 }
1735 Ok(())
1736 }
1737
1738 async fn rollback_prep(&mut self, repo: &Path) {
1740 let root = self.state.worktree_root();
1741 for c in std::mem::take(&mut self.state.candidates) {
1742 if matches!(
1743 git::worktree_remove_clean(repo, &c.worktree).await,
1744 Ok(true)
1745 ) {
1746 git::branch_delete(repo, &c.branch).await.ok();
1747 } else {
1748 self.state.candidates.push(c);
1749 }
1750 }
1751 let seats = self.roles.judges.len().max(self.state.config.graph.judges);
1752 for j in 1..=seats {
1753 git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
1754 .await
1755 .ok();
1756 }
1757 for k in 1..=self.state.config.graph.advisors {
1758 git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
1759 .await
1760 .ok();
1761 }
1762 }
1763
1764 async fn prep(&mut self) -> Result<()> {
1765 if let Some(reason) = &self.state.prep_failure {
1766 bail!(
1767 "candidate preparation previously failed: {reason}; correct the source before retrying"
1768 );
1769 }
1770 if self
1771 .state
1772 .prep_complete
1773 .unwrap_or(!self.state.candidates.is_empty())
1774 {
1775 return Ok(());
1776 }
1777 let repo = self.state.repo.clone();
1778 let _coordination = self.coordinate(&repo).await?;
1781 let base = self.state.base_commit.clone();
1782 let plan = self.state.sources.plan(&repo, &base).await?;
1783 self.rollback_prep(&repo).await;
1784 if !self.state.candidates.is_empty() {
1785 bail!(
1786 "preparation rollback retained worktrees; recover their uncommitted work before retrying"
1787 );
1788 }
1789 self.state.prep_complete = Some(false);
1790 self.state.status = RunStatus::Prep;
1791 self.state.save()?;
1792 let repo = self.state.repo.clone();
1793 let start = plan.start.clone().unwrap_or_else(|| base.clone());
1794 let root = self.state.worktree_root();
1795 let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1796
1797 let hooks_dir = self.state.dir().join("hooks");
1800 if self.state.config.blind.commit_msg_hook {
1801 std::fs::create_dir_all(&hooks_dir)
1802 .with_context(|| format!("create {}", hooks_dir.display()))?;
1803 let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1804 let path = hooks_dir.join("commit-msg");
1805 std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1806 make_executable(&path)?;
1807 git::acquire_worktree_config(&repo).await?;
1815 self.state.enabled_worktree_config = true;
1816 }
1817
1818 ensure_setup_config(&mut self.state, &repo).await?;
1821
1822 for (index, (spec, label)) in self
1823 .roles
1824 .implementers
1825 .clone()
1826 .into_iter()
1827 .zip(labels)
1828 .enumerate()
1829 {
1830 let branch = self.state.branch_for(label);
1831 let worktree = root.join(format!("cand-{label}"));
1832 self.state.event(
1836 "prep-intent",
1837 serde_json::json!({
1838 "branch": branch,
1839 "worktree": worktree,
1840 "start": start,
1841 "picks": plan.picks,
1842 })
1843 .to_string(),
1844 );
1845 self.state.save()?;
1846 git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1847 if self.state.config.blind.commit_msg_hook {
1848 git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1849 }
1850 git::local_exclude(&worktree, "/.magi/").await?;
1851 if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
1852 git::worktree_remove_clean(&repo, &worktree).await.ok();
1855 git::branch_delete(&repo, &branch).await.ok();
1856 self.rollback_prep(&repo).await;
1857 self.state.status = RunStatus::Blocked;
1858 self.state
1859 .event("prep", format!("worktree setup failed: {e:#}"));
1860 self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
1861 self.state.save()?;
1862 return Err(e);
1863 }
1864 self.state.candidates.push(Candidate {
1865 index,
1866 label,
1867 agent: spec.id.clone(),
1868 branch,
1869 worktree,
1870 summary: String::new(),
1871 stat: String::new(),
1872 files: 0,
1873 commits: 0,
1874 empty: false,
1875 failed: None,
1876 verified_noop: None,
1877 duration_ms: 0,
1878 folded: false,
1879 });
1880 self.state.save()?;
1881 for pick in &plan.picks {
1882 if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
1883 {
1884 self.state.status = RunStatus::Blocked;
1885 self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
1886 self.state
1887 .event("prep", format!("cannot apply referenced commit: {e}"));
1888 self.state.save()?;
1889 return Err(e);
1890 }
1891 }
1892 }
1893
1894 for j in 1..=self.roles.judges.len() {
1895 let wt = root.join(format!("judge-{j}"));
1896 if !wt.exists() {
1897 git::worktree_add_detached(&repo, &wt, &base).await?;
1898 self.setup_seat_worktree(&repo, &wt, true).await?;
1899 }
1900 }
1901
1902 if self.state.config.graph.advise {
1910 for k in 1..=self.state.config.graph.advisors {
1911 let wt = root.join(format!("advisor-{k}"));
1912 if !wt.exists() {
1913 git::worktree_add_detached(&repo, &wt, &base).await?;
1914 self.setup_seat_worktree(&repo, &wt, true).await?;
1915 }
1916 }
1917 }
1918
1919 let authors: Vec<&str> = self
1924 .roles
1925 .implementers
1926 .iter()
1927 .map(|a| a.id.as_str())
1928 .collect();
1929 let overlap: Vec<String> = self
1930 .roles
1931 .judges
1932 .iter()
1933 .enumerate()
1934 .filter(|(_, j)| authors.contains(&j.id.as_str()))
1935 .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1936 .collect();
1937 if !overlap.is_empty() {
1938 let note = format!(
1939 "{} also authored a candidate; blind, but the panel is less \
1940 independent than {} distinct agents would be",
1941 overlap.join(", "),
1942 self.roles.judges.len()
1943 );
1944 self.state.event("prep", note);
1945 }
1946
1947 self.state.event(
1948 "prep",
1949 format!(
1950 "{} candidates, {} judges, base {} ({})",
1951 self.state.candidates.len(),
1952 self.roles.judges.len(),
1953 &self.state.base_commit[..7.min(self.state.base_commit.len())],
1954 self.state.base_branch
1955 ),
1956 );
1957 self.state.prep_complete = Some(true);
1958 self.state.status = RunStatus::Implementing;
1959 self.state.save()?;
1960 Ok(())
1961 }
1962
1963 async fn advise(&mut self) -> Result<()> {
1996 let implement_untouched = self
1997 .state
1998 .candidates
1999 .iter()
2000 .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2001 if !self.state.config.graph.advise || self.state.advise_attempted {
2002 return Ok(());
2003 }
2004 if !implement_untouched {
2005 self.state.event(
2006 "advise",
2007 "skipping the design-deliberation stage: at least one \
2008 candidate already shows implementation progress, so this \
2009 run is past the point the stage exists to run before"
2010 .to_owned(),
2011 );
2012 self.state.advise_attempted = true;
2013 self.state.save()?;
2014 return Ok(());
2015 }
2016 let run_id = self.state.id.clone();
2017 let prompts = self.state.config.prompts.clone();
2018 let instruction = self.state.instruction.clone();
2019 let language = self.state.config.graph.language.clone();
2020 let root = self.state.worktree_root();
2021 let n = self.state.config.graph.advisors;
2022 let where_recorded = self.state.dir().join("run.json");
2023
2024 let seats = match self.state.config.advisors() {
2025 Ok(seats) if !seats.is_empty() => seats,
2026 Ok(_) => {
2027 self.state.event(
2028 "advise",
2029 format!(
2030 "[graph] advisors is 0; skipping the design-deliberation \
2031 stage and continuing without a synthesis brief (see {})",
2032 where_recorded.display()
2033 ),
2034 );
2035 self.state.advise_attempted = true;
2036 self.state.save()?;
2037 return Ok(());
2038 }
2039 Err(e) => {
2040 self.state.event(
2041 "advise",
2042 format!(
2043 "could not resolve advisor seats ({e:#}); continuing \
2044 without a design-deliberation brief (see {})",
2045 where_recorded.display()
2046 ),
2047 );
2048 self.state.advise_attempted = true;
2049 self.state.save()?;
2050 return Ok(());
2051 }
2052 };
2053
2054 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2055 let artifacts = agent::artifacts_dir(&self.state.dir());
2056 let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2057
2058 let mut jobs = Vec::new();
2059 for (i, spec) in seats.iter().cloned().enumerate() {
2060 let seat_key = format!("advisor-{}", i + 1);
2061 let seat = self.seat(&seat_key, &spec.id);
2062 jobs.push(SeatJob {
2063 prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2064 spec,
2065 seat,
2066 cwd: worktrees[i % worktrees.len()].clone(),
2067 timeout,
2068 allow_write: false,
2069 sessions: false,
2070 artifacts: artifacts.clone(),
2071 stem: seat_key,
2072 handover: None,
2073 });
2074 }
2075
2076 self.state.event(
2077 "advise",
2078 format!(
2079 "{} advisor seat(s) sketching a design in parallel",
2080 jobs.len()
2081 ),
2082 );
2083 let mut quota_losses = Vec::new();
2084 let cache = self.state.config.cache_dir();
2085 let ctx = WaveCtx {
2086 carry_seats: false,
2087 pauses: Some((&self.pause, &self.interrupt)),
2088 run: &run_id,
2089 node: "advise",
2090 prompts: &prompts,
2091 cache: cache.as_deref(),
2092 round: None,
2093 };
2094 let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2095 let results = ask_json_wave::<Proposal>(
2096 jobs,
2097 Arc::clone(&self.sem),
2098 self.state.config.graph.retries,
2099 &advisor_roster,
2100 &ctx,
2101 &mut quota_losses,
2102 &mut self.state,
2103 &|p: &Proposal| p.validate(),
2104 )
2105 .await;
2106 if self.state.resource_block.is_some() {
2107 return self.stop_for_resource();
2108 }
2109 self.state.quota.extend(quota_losses);
2110
2111 let mut records = Vec::with_capacity(results.len());
2112 for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2113 let agent_id = seat.agent.clone();
2114 self.state.seats.insert(seat.key.clone(), seat);
2115 match res {
2116 Ok((proposal, out)) => {
2117 self.state
2118 .event("advise", format!("advisor-{} proposed a design", i + 1));
2119 records.push(advise::AdvisorRecord::proposed(
2120 i + 1,
2121 agent_id,
2122 proposal,
2123 out.duration_ms,
2124 ));
2125 }
2126 Err(e) => {
2127 self.state.event(
2128 "advise",
2129 format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2130 );
2131 records.push(advise::AdvisorRecord::failed(
2132 i + 1,
2133 agent_id,
2134 e.to_string(),
2135 ));
2136 }
2137 }
2138 }
2139
2140 let mut advice = advise::Advice {
2141 records,
2142 synthesis: None,
2143 };
2144 if advice.proposals().is_empty() {
2145 self.state.event(
2146 "advise",
2147 "no advisor produced a usable proposal; continuing without a \
2148 synthesis brief"
2149 .to_owned(),
2150 );
2151 } else {
2152 match self
2153 .synthesize_brief(
2154 &advice,
2155 &instruction,
2156 &language,
2157 &worktrees[0],
2158 &artifacts,
2159 &run_id,
2160 &prompts,
2161 cache.as_deref(),
2162 )
2163 .await
2164 {
2165 Ok(Some(text)) => {
2166 self.state.event(
2167 "advise",
2168 "synthesized a design brief for the implementer".to_owned(),
2169 );
2170 advice.synthesis = Some(text);
2171 }
2172 Ok(None) => {
2173 self.state.event(
2174 "advise",
2175 "the synthesis seat produced nothing usable; continuing \
2176 without a design brief"
2177 .to_owned(),
2178 );
2179 }
2180 Err(e) => {
2181 if e.is::<crate::repo_lock::ResourceBlocked>() {
2182 return Err(e);
2183 }
2184 self.state.event(
2185 "advise",
2186 format!("could not synthesize a design brief: {e:#}"),
2187 );
2188 }
2189 }
2190 }
2191 advise::apply_reflection(&mut advice);
2192
2193 self.state.advice = Some(advice);
2194 self.state.advise_attempted = true;
2195 self.state.save()?;
2196 Ok(())
2197 }
2198
2199 #[allow(clippy::too_many_arguments)]
2211 async fn synthesize_brief(
2212 &mut self,
2213 advice: &advise::Advice,
2214 instruction: &str,
2215 language: &str,
2216 cwd: &Path,
2217 artifacts: &Path,
2218 run_id: &str,
2219 prompts: &Prompts,
2220 cache: Option<&Path>,
2221 ) -> Result<Option<String>> {
2222 let chain = agent::pick_chain(
2223 &self.state.config.agents,
2224 self.state.config.roles.synthesizer.as_ref(),
2225 &agent::installed,
2226 "synthesizer",
2227 )?;
2228 let proposals = advice.proposals();
2229 let mut prompt = prompt::with_overlay(
2230 prompt::synthesize_brief(instruction, &proposals, language),
2231 prompts.overlay("advise"),
2232 );
2233 if cache.is_some() {
2234 prompt.push('\n');
2239 prompt.push_str(&prompt::build_cache_note("advise", false));
2240 }
2241 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2242 let mut last = None;
2247 for (n, spec) in chain.iter().enumerate() {
2248 if n > 0 {
2249 self.state
2250 .event("advise", format!("synthesis falling back to {}", spec.id));
2251 }
2252 let mut seat = self.seat("advise-synthesis", &spec.id);
2253 let wait = self.repository_wait();
2254 let outcome = wait
2255 .scope(agent::invoke(
2256 spec,
2257 &mut seat,
2258 &Invocation {
2259 scope: crate::agent::InvocationScope::Repository,
2260 coordination: self.state.config.coordination.as_ref(),
2261 fetched_base: None,
2262 cwd,
2263 prompt: &prompt,
2264 timeout,
2265 allow_write: false,
2266 unsandboxed: false,
2267 sessions: false,
2268 artifacts,
2269 stem: &if n == 0 {
2270 "advise-synthesis".to_owned()
2271 } else {
2272 format!("advise-synthesis-{}", spec.id)
2273 },
2274 run: run_id,
2275 node: "advise",
2276 cache_dir: None,
2277 attachments: &[],
2278 writable: &[],
2279 },
2280 ))
2281 .await;
2282 if outcome.is_ok() {
2283 self.state.seats.insert(seat.key.clone(), seat);
2284 }
2285 let advance = agent::chain_advances(&outcome);
2286 last = Some(outcome);
2287 if !advance {
2288 break;
2289 }
2290 }
2291 let out = last.expect("a chain holds at least one agent")?;
2294 if !out.usable() {
2295 return Ok(None);
2296 }
2297 let text =
2298 verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2299 Ok((!text.trim().is_empty()).then_some(text))
2300 }
2301
2302 async fn implement(&mut self) -> Result<()> {
2305 self.persist_candidates().await?;
2306 let run_id = self.state.id.clone();
2311 let prompts = self.state.config.prompts.clone();
2312 let todo: Vec<usize> = self
2313 .state
2314 .candidates
2315 .iter()
2316 .enumerate()
2317 .filter(|(_, c)| {
2318 (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2319 && c.failed.is_none()
2320 && !c.empty
2321 })
2322 .map(|(i, _)| i)
2323 .collect();
2324 if todo.is_empty() {
2325 return self.after_implement();
2326 }
2327 self.state.status = RunStatus::Implementing;
2328
2329 let language = self.state.config.graph.language.clone();
2330 let github_language = self.state.config.graph.github_language.clone();
2331 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2332 let sessions = self.state.config.graph.sessions;
2333 let artifacts = agent::artifacts_dir(&self.state.dir());
2334 let brief = self
2338 .state
2339 .advice
2340 .as_ref()
2341 .and_then(|a| a.synthesis.as_deref())
2342 .map(str::to_owned);
2343 let attachments = self.state.attachments.clone();
2344
2345 let mut jobs = Vec::new();
2346 for &i in &todo {
2347 let (index, label, worktree) = {
2348 let c = &self.state.candidates[i];
2349 (c.index, c.label, c.worktree.clone())
2350 };
2351 let spec = self.roles.implementers[index].clone();
2352 let seat_key = format!("impl-{label}");
2353 let seat = self.seat(&seat_key, &spec.id);
2354 let instruction = seeded_instruction(&self.state);
2355 jobs.push(SeatJob {
2356 spec,
2357 seat,
2358 prompt: prompt::implement_in(
2359 &instruction,
2360 &worktree.to_string_lossy(),
2361 &language,
2362 &github_language,
2363 brief.as_deref(),
2364 &attachments,
2365 ),
2366 cwd: worktree,
2367 timeout,
2368 allow_write: true,
2369 sessions,
2370 artifacts: artifacts.clone(),
2371 stem: format!("impl-{label}"),
2372 handover: None,
2373 });
2374 }
2375
2376 self.state.event(
2377 "implement",
2378 format!("{} candidates in parallel", jobs.len()),
2379 );
2380 let mut sent = jobs.clone();
2386 let cache = self.state.config.cache_dir();
2387 let ctx = WaveCtx {
2388 carry_seats: false,
2389 pauses: Some((&self.pause, &self.interrupt)),
2390 run: &run_id,
2391 node: "implement",
2392 prompts: &prompts,
2393 cache: cache.as_deref(),
2394 round: None,
2395 };
2396 let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2397 if !wave_blocked(&results) {
2398 self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
2399 .await;
2400 self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
2401 .await?;
2402 self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
2403 .await;
2404 }
2405 let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
2406
2407 let selected_failed = self.state.config.graph.selected_implementer.is_some()
2408 && results
2409 .iter()
2410 .any(|(_, _, out)| FailClass::of(out).is_some());
2411 for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
2412 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
2413 if self
2417 .state
2418 .candidate_preservation
2419 .contains_key(&self.state.candidates[i].label)
2420 {
2421 self.state.seats.insert(seat.key.clone(), seat);
2422 }
2423 continue;
2424 }
2425 let seat_key = seat.key.clone();
2426 let agent = seat.agent.clone();
2435 let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
2436 self.state.seats.insert(seat.key.clone(), seat);
2437 let label = self.state.candidates[i].label;
2438 self.state.interrupted_implementations.remove(&label);
2439
2440 let (summary, duration, failed, verified_claim) = match out {
2441 AgentOutcome::Ok(o) => {
2442 let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
2443 let failed = (!o.usable()).then(|| {
2444 if o.timed_out {
2445 "agent timed out".to_owned()
2446 } else {
2447 format!(
2448 "agent exited with {}",
2449 crate::notices::exit_status(o.exit_code)
2450 )
2451 }
2452 });
2453 let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
2454 (text, o.duration_ms, failed, verified_claim)
2455 }
2456 AgentOutcome::Dropped(o) => {
2462 let why = o
2463 .dropped
2464 .as_ref()
2465 .map(|d| d.why.as_str())
2466 .unwrap_or("the CLI ended the stream without delivering its answer");
2467 (
2468 String::new(),
2469 o.duration_ms,
2470 Some(format!("the CLI dropped the stream ({why})")),
2471 None,
2472 )
2473 }
2474 AgentOutcome::Quota(o) => {
2475 self.state.quota.push(QuotaLoss {
2476 seat: seat_key,
2477 node: "implement".to_owned(),
2478 at: Timestamp::now(),
2479 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2480 });
2481 (
2482 String::new(),
2483 o.duration_ms,
2484 Some("rate limited (quota); produced no change".to_owned()),
2485 None,
2486 )
2487 }
2488 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
2489 (String::new(), 0, Some(e), None)
2490 }
2491 };
2492
2493 let c = &mut self.state.candidates[i];
2496 if !exhausted_the_fallback_chain {
2497 c.agent = agent;
2498 }
2499 c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
2500 c.duration_ms = duration;
2501 c.failed = failed;
2502 c.verified_noop = verified_claim;
2503 self.state
2504 .candidate_preservation
2505 .insert(label, "awaiting persistence".to_owned());
2506 }
2507 self.state.save()?;
2508 self.persist_candidates().await?;
2509
2510 if resource_blocked {
2511 return self.stop_for_resource();
2512 }
2513 if selected_failed {
2514 self.state.status = RunStatus::Blocked;
2515 self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
2516 self.state.save()?;
2517 bail!(
2518 "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
2519 );
2520 }
2521 self.after_implement()
2522 }
2523
2524 async fn persist_candidates(&mut self) -> Result<()> {
2526 let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
2527 for label in labels {
2528 let i = self
2529 .state
2530 .candidates
2531 .iter()
2532 .position(|c| c.label == label)
2533 .context("pending candidate is missing")?;
2534 let c = self.state.candidates[i].clone();
2535 let interrupted = self.state.interrupted_implementations.contains(&label);
2536 let result = async {
2537 let wait = crate::repo_lock::WaitPolicy::new(
2538 Duration::from_secs(self.state.config.graph.timeout_resource_wait),
2539 || false,
2540 );
2541 let _coordination =
2542 coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
2543 if let Err(e) = git::snapshot_uncommitted(
2546 &c.worktree,
2547 &self
2548 .state
2549 .dir()
2550 .join("artifacts")
2551 .join(format!("candidate-{label}-recovery")),
2552 )
2553 .await
2554 {
2555 self.state.event(
2556 "preserve",
2557 format!("candidate {label} snapshot incomplete: {e:#}"),
2558 );
2559 }
2560 let rescued = git::rescue_commit(
2561 &c.worktree,
2562 &format!("magi: candidate {label} (uncommitted work)"),
2563 )
2564 .await?;
2565 self.state.note_withheld("implement", &rescued.withheld);
2566 let base = &self.state.base_commit;
2567 let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
2568 let patch = git::diff(&c.worktree, base, "HEAD").await?;
2569 let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
2570 let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
2571 write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
2572 if c.empty && commits > 0 && !patch.trim().is_empty()
2573 && (self.state.judge_skipped || !self.state.judgements.is_empty()
2574 || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
2575 || self.state.tally.is_some())
2576 {
2577 bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
2581 }
2582 let c = &mut self.state.candidates[i];
2583 c.commits = commits;
2584 c.stat = stat;
2585 c.files = files;
2586 let no_changes = commits == 0 || patch.trim().is_empty();
2587 c.empty = no_changes && !interrupted;
2588 if interrupted && no_changes {
2589 c.failed = None;
2590 c.verified_noop = None;
2591 }
2592 if !c.empty {
2593 c.failed = None;
2594 c.verified_noop = None;
2595 }
2596 self.state.event(
2597 "preserve",
2598 format!("candidate {label}: {files} files, {commits} commits"),
2599 );
2600 Ok::<_, anyhow::Error>(())
2601 }
2602 .await;
2603 match result {
2604 Ok(()) => {
2605 self.state.candidate_preservation.remove(&label);
2606 if !interrupted || self.state.candidates[i].files > 0 {
2607 self.state.interrupted_implementations.remove(&label);
2608 }
2609 }
2610 Err(e) => {
2611 let reason = format!("candidate {label} preservation failed: {e:#}");
2612 self.state.event("preserve", &reason);
2613 self.state.candidate_preservation.insert(label, reason);
2614 }
2615 }
2616 if self.state.candidate_preservation.is_empty()
2617 && self.state.status == RunStatus::Blocked
2618 && self.state.candidates.iter().all(|c| c.failed.is_none())
2619 {
2620 self.state.status = RunStatus::Implementing;
2624 self.state.event(
2625 "preserve",
2626 "candidate persistence recovered; implementation can continue",
2627 );
2628 }
2629 self.state.save()?;
2630 }
2631 if !self.state.candidate_preservation.is_empty() {
2632 self.state.status = RunStatus::Blocked;
2633 self.state.save()?;
2634 bail!(
2635 "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
2636 );
2637 }
2638 Ok(())
2639 }
2640
2641 async fn resume_undelivered(
2669 &mut self,
2670 results: &mut [(usize, SeatState, AgentOutcome)],
2671 sent: &[SeatJob],
2672 prompts: &Prompts,
2673 run_id: &str,
2674 ) {
2675 for (wi, seat, out) in results.iter_mut() {
2676 let Some(dropped) = (match &*out {
2677 AgentOutcome::Dropped(o) => o.dropped.clone(),
2678 _ => None,
2679 }) else {
2680 continue;
2681 };
2682 let Some(job) = sent.get(*wi) else { continue };
2683 if !git::is_clean(&job.cwd).await.unwrap_or(true) {
2685 self.state.event(
2686 "implement",
2687 format!(
2688 "{}: the CLI dropped the stream after {} output tokens ({}), but the \
2689 work is in the tree",
2690 seat.key, dropped.output_tokens, dropped.why
2691 ),
2692 );
2693 continue;
2694 }
2695 if !has_context(&job.spec, seat, job.sessions) {
2703 self.state.event(
2704 "implement",
2705 format!(
2706 "{}: the CLI dropped the stream after {} output tokens ({}), but there \
2707 is no session left to resume",
2708 seat.key, dropped.output_tokens, dropped.why
2709 ),
2710 );
2711 continue;
2712 }
2713 self.state.event(
2714 "implement",
2715 format!(
2716 "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
2717 conversation",
2718 seat.key, dropped.output_tokens, dropped.why
2719 ),
2720 );
2721 let mut retry = job.clone();
2722 retry.seat = seat.clone();
2723 retry.prompt = prompt::resume_after_drop(&dropped.why);
2724 retry.timeout = retry_budget(job.timeout, true);
2725 retry.stem = format!("{}-resume", job.stem);
2726 let cache = self.state.config.cache_dir();
2727 let ctx = WaveCtx {
2728 carry_seats: false,
2729 pauses: Some((&self.pause, &self.interrupt)),
2730 run: run_id,
2731 node: "implement",
2732 prompts,
2733 cache: cache.as_deref(),
2734 round: None,
2735 };
2736 let (resumed_seat, resumed) =
2737 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2738 *seat = resumed_seat;
2739 *out = resumed;
2740 }
2741 }
2742
2743 async fn resume_seat_handovers(
2808 &mut self,
2809 results: &mut [(usize, SeatState, AgentOutcome)],
2810 sent: &mut [SeatJob],
2811 prompts: &Prompts,
2812 run_id: &str,
2813 ) -> Result<()> {
2814 let instruction = seeded_instruction(&self.state);
2815 let language = self.state.config.graph.language.clone();
2816 let github_language = self.state.config.graph.github_language.clone();
2817 let brief = self
2818 .state
2819 .advice
2820 .as_ref()
2821 .and_then(|a| a.synthesis.as_deref())
2822 .map(str::to_owned);
2823 let attachments = self.state.attachments.clone();
2824 for (wi, seat, out) in results.iter_mut() {
2825 let others: BTreeSet<String> = sent
2828 .iter()
2829 .enumerate()
2830 .filter(|(j, _)| j != wi)
2831 .map(|(_, j)| j.spec.id.clone())
2832 .collect();
2833 let Some(job) = sent.get_mut(*wi) else {
2834 continue;
2835 };
2836 let start = self
2841 .roles
2842 .implementer_roster
2843 .iter()
2844 .position(|s| s.id == job.spec.id)
2845 .unwrap_or(0);
2846 let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
2847 let mut fallback_attempt = 0usize;
2848 let mut prev: Option<FailClass> = None;
2849 while let Some(cur) = FailClass::of(&*out) {
2850 if !should_hand_over(prev.as_ref(), &cur) {
2851 break;
2852 }
2853 let Some(next) =
2854 pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
2855 .cloned()
2856 else {
2857 break;
2858 };
2859 tried.insert(next.id.clone());
2860 fallback_attempt += 1;
2861
2862 let coordination = self.coordinate(&job.cwd).await?;
2863 if let Ok(r) = git::rescue_commit(
2864 &job.cwd,
2865 &format!(
2866 "magi: candidate {} (uncommitted work before {} fallback)",
2867 seat.key,
2868 if cur == FailClass::Quota {
2869 "quota"
2870 } else {
2871 "handover"
2872 }
2873 ),
2874 )
2875 .await
2876 {
2877 self.state.note_withheld("implement", &r.withheld);
2878 }
2879 drop(coordination);
2880
2881 record_handover(
2882 &mut self.state,
2883 "implement",
2884 &seat.key,
2885 &seat.agent,
2886 &next.id,
2887 &cur,
2888 &fail_reason(&*out),
2889 );
2890 prev = Some(cur.clone());
2891
2892 let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2893 self.state.seats.insert(seat.key.clone(), new_seat.clone());
2894 job.spec = next.clone();
2902 let mut retry = job.clone();
2903 retry.seat = new_seat;
2904 retry.prompt = prompt::implement_in(
2905 &instruction,
2906 &job.cwd.to_string_lossy(),
2907 &language,
2908 &github_language,
2909 brief.as_deref(),
2910 &attachments,
2911 );
2912 retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2913 let cache = self.state.config.cache_dir();
2914 let ctx = WaveCtx {
2915 carry_seats: false,
2916 pauses: Some((&self.pause, &self.interrupt)),
2917 run: run_id,
2918 node: "implement",
2919 prompts,
2920 cache: cache.as_deref(),
2921 round: None,
2922 };
2923 let (fallback_seat, fallback_out) = run_one(
2924 retry,
2925 Arc::clone(&self.sem),
2926 &ctx,
2927 &mut self.state,
2928 fallback_attempt,
2929 )
2930 .await;
2931 *seat = fallback_seat;
2932 *out = fallback_out;
2933 }
2934 }
2935 Ok(())
2936 }
2937
2938 async fn resume_unconfirmed_commands(
2962 &mut self,
2963 results: &mut [(usize, SeatState, AgentOutcome)],
2964 sent: &[SeatJob],
2965 prompts: &Prompts,
2966 run_id: &str,
2967 ) {
2968 for (wi, seat, out) in results.iter_mut() {
2969 let AgentOutcome::Ok(o) = &*out else {
2970 continue;
2971 };
2972 if !has_unconfirmed_command(&o.commands) {
2973 continue;
2974 }
2975 let Some(job) = sent.get(*wi) else { continue };
2976 if !has_context(&job.spec, seat, job.sessions) {
2977 self.state.event(
2978 "implement",
2979 format!(
2980 "{}: the reply named a command whose own CLI never confirmed the exit \
2981 status of, but there is no session left to resume",
2982 seat.key
2983 ),
2984 );
2985 continue;
2986 }
2987 self.state.event(
2988 "implement",
2989 format!(
2990 "{}: the reply named a command whose own CLI never confirmed the exit \
2991 status of; resuming the conversation",
2992 seat.key
2993 ),
2994 );
2995 let mut retry = job.clone();
2996 retry.seat = seat.clone();
2997 retry.prompt = prompt::resume_incomplete(
2998 "a command in your last reply had no confirmed exit status",
2999 );
3000 retry.timeout = retry_budget(job.timeout, true);
3001 retry.stem = format!("{}-confirm", job.stem);
3002 let cache = self.state.config.cache_dir();
3003 let ctx = WaveCtx {
3004 carry_seats: false,
3005 pauses: Some((&self.pause, &self.interrupt)),
3006 run: run_id,
3007 node: "implement",
3008 prompts,
3009 cache: cache.as_deref(),
3010 round: None,
3011 };
3012 let (resumed_seat, resumed) =
3013 run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3014 *seat = resumed_seat;
3015 *out = resumed;
3016 }
3017 }
3018
3019 async fn continue_fix_report(
3040 &mut self,
3041 mut seat: SeatState,
3042 parse_err: String,
3043 job: &SeatJob,
3044 prompts: &Prompts,
3045 run_id: &str,
3046 round: usize,
3047 ) -> (
3048 SeatState,
3049 Option<FixReport>,
3050 Option<String>,
3051 ContinuationRecord,
3052 ) {
3053 let mut last_err = parse_err;
3054 let mut cumulative_wait_ms = 0u64;
3055 let mut attempts = 0usize;
3056 loop {
3057 if !has_context(&job.spec, &seat, job.sessions) {
3058 self.state.event(
3059 "fix",
3060 format!(
3061 "round {round}: fixer's reply had no adoption report ({last_err}); no \
3062 session left to resume into"
3063 ),
3064 );
3065 let outcome = if attempts == 0 {
3066 ContinuationOutcome::NoSession
3067 } else {
3068 ContinuationOutcome::Exhausted
3069 };
3070 return (
3071 seat,
3072 None,
3073 Some(format!("unparsable fix report: {last_err}")),
3074 ContinuationRecord {
3075 attempts,
3076 cumulative_wait_ms,
3077 outcome,
3078 },
3079 );
3080 }
3081 if attempts >= MAX_FIX_CONTINUATIONS {
3082 self.state.event(
3083 "fix",
3084 format!(
3085 "round {round}: fixer's reply still had no adoption report after \
3086 {attempts} continuation(s) ({last_err}); giving up"
3087 ),
3088 );
3089 return (
3090 seat,
3091 None,
3092 Some(format!(
3093 "unparsable fix report after {attempts} continuation(s): {last_err}"
3094 )),
3095 ContinuationRecord {
3096 attempts,
3097 cumulative_wait_ms,
3098 outcome: ContinuationOutcome::Exhausted,
3099 },
3100 );
3101 }
3102 attempts += 1;
3103 self.state.event(
3104 "fix",
3105 format!(
3106 "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3107 the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3108 ),
3109 );
3110 let mut retry = job.clone();
3111 retry.seat = seat.clone();
3112 retry.prompt = prompt::resume_incomplete(&last_err);
3113 retry.timeout = retry_budget(job.timeout, true);
3114 retry.stem = format!("{}-continue{attempts}", job.stem);
3115 let cache = self.state.config.cache_dir();
3116 let ctx = WaveCtx {
3117 carry_seats: false,
3118 pauses: Some((&self.pause, &self.interrupt)),
3119 run: run_id,
3120 node: "fix",
3121 prompts,
3122 cache: cache.as_deref(),
3123 round: Some(round),
3124 };
3125 let (resumed_seat, resumed_out) = run_one(
3126 retry,
3127 Arc::clone(&self.sem),
3128 &ctx,
3129 &mut self.state,
3130 attempts,
3131 )
3132 .await;
3133 seat = resumed_seat;
3134 match resumed_out {
3135 AgentOutcome::Ok(o) => {
3136 cumulative_wait_ms += o.duration_ms;
3137 match verdict::extract_json::<FixReport>(&o.text) {
3138 Ok(report) if !has_unconfirmed_command(&o.commands) => {
3139 self.state.event(
3140 "fix",
3141 format!(
3142 "round {round}: fixer's adoption report recovered after \
3143 {attempts} continuation(s)"
3144 ),
3145 );
3146 return (
3147 seat,
3148 Some(report),
3149 None,
3150 ContinuationRecord {
3151 attempts,
3152 cumulative_wait_ms,
3153 outcome: ContinuationOutcome::Resumed,
3154 },
3155 );
3156 }
3157 Ok(_) => {
3165 last_err = "the reply parsed, but it reported a command whose own CLI \
3166 never confirmed an exit status"
3167 .to_owned();
3168 }
3169 Err(e) => last_err = e.to_string(),
3170 }
3171 }
3172 AgentOutcome::Quota(o) => {
3173 cumulative_wait_ms += o.duration_ms;
3174 self.state.quota.push(QuotaLoss {
3175 seat: seat.key.clone(),
3176 node: "fix".to_owned(),
3177 at: Timestamp::now(),
3178 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3179 });
3180 self.state.event(
3181 "fix",
3182 format!(
3183 "round {round}: continuation rate limited (quota); not retrying now"
3184 ),
3185 );
3186 return (
3187 seat,
3188 None,
3189 Some("rate limited (quota) while recovering the fix report".to_owned()),
3190 ContinuationRecord {
3191 attempts,
3192 cumulative_wait_ms,
3193 outcome: ContinuationOutcome::QuotaLost,
3194 },
3195 );
3196 }
3197 AgentOutcome::Dropped(o) => {
3198 cumulative_wait_ms += o.duration_ms;
3199 let why = o
3200 .dropped
3201 .as_ref()
3202 .map(|d| d.why.as_str())
3203 .unwrap_or("the CLI ended the stream without delivering its answer");
3204 last_err = format!("the CLI dropped the stream ({why})");
3205 }
3206 AgentOutcome::Failed(e) => last_err = e,
3207 AgentOutcome::ResourceBlocked(e) => {
3210 return (
3211 seat,
3212 None,
3213 Some(format!("the continuation was not started: {e}")),
3214 ContinuationRecord {
3215 attempts: attempts.saturating_sub(1),
3216 cumulative_wait_ms,
3217 outcome: ContinuationOutcome::NoSession,
3218 },
3219 );
3220 }
3221 }
3222 }
3223 }
3224
3225 fn after_implement(&mut self) -> Result<()> {
3226 if !self.state.candidate_preservation.is_empty() {
3227 bail!("candidate preservation incomplete; judging is unsafe");
3228 }
3229 if self.state.leaks.is_empty() {
3231 let cfg = self.state.config.blind.clone();
3232 let mut leaks = Vec::new();
3233 for c in &self.state.candidates {
3234 let Some(patch) =
3235 crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3236 else {
3237 continue;
3238 };
3239 leaks.extend(blind::scan(
3240 &format!("candidate {} patch", c.label),
3241 &patch,
3242 &cfg.vendor_tokens,
3243 ));
3244 }
3245 if !leaks.is_empty() {
3246 let summary = leaks
3247 .iter()
3248 .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3249 .collect::<Vec<_>>()
3250 .join(", ");
3251 match cfg.on_leak {
3252 LeakPolicy::Fail => {
3253 self.state.status = RunStatus::Failed;
3254 self.state
3255 .event("blind", format!("vendor text in a patch: {summary}"));
3256 self.state.leaks = leaks;
3257 self.state.save()?;
3258 self.settle_questions();
3259 bail!(
3260 "blind.on_leak = \"fail\" and vendor text reached a \
3261 judged patch: {summary}"
3262 );
3263 }
3264 LeakPolicy::Redact => self.state.event(
3265 "blind",
3266 format!("redacting vendor text for judging: {summary}"),
3267 ),
3268 LeakPolicy::Warn => self.state.event(
3269 "blind",
3270 format!("vendor text present in a judged patch (shown as-is): {summary}"),
3271 ),
3272 }
3273 self.state.leaks = leaks;
3274 }
3275 }
3276
3277 if self.state.viable().is_empty() {
3278 if self.state.all_candidates_verified_noop() {
3279 self.state.status = RunStatus::VerifiedNoop;
3290 self.state.save()?;
3291 self.settle_questions();
3292 return Ok(());
3293 }
3294 self.state.status = RunStatus::Failed;
3295 self.state.save()?;
3296 self.settle_questions();
3297 bail!("no candidate produced a change; nothing to judge");
3298 }
3299 self.state.status = RunStatus::Judging;
3300 self.state.save()?;
3301 Ok(())
3302 }
3303
3304 async fn judge(&mut self) -> Result<()> {
3307 let run_id = self.state.id.clone();
3312 let prompts = self.state.config.prompts.clone();
3313 if !self.state.judgements.is_empty() || self.state.judge_skipped {
3314 return Ok(());
3315 }
3316 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3317 if viable.len() == 1 {
3318 self.state.judge_skipped = true;
3325 self.state.event(
3326 "judge",
3327 format!(
3328 "only candidate {} produced a change; judging skipped",
3329 viable[0].label
3330 ),
3331 );
3332 self.state.save()?;
3333 return Ok(());
3334 }
3335 self.state.status = RunStatus::Judging;
3336
3337 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3338 let language = self.state.config.graph.language.clone();
3339 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3340 let sessions = self.state.config.graph.sessions;
3341 let artifacts = agent::artifacts_dir(&self.state.dir());
3342 let root = self.state.worktree_root();
3343 let base_short = short(&self.state.base_commit);
3344
3345 let mut jobs = Vec::new();
3346 let mut orders = Vec::new();
3347 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3348 let order = blind::presentation_order(viable.len(), j, self.state.seed);
3349 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3350 orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3351 let seat_key = format!("judge-{}", j + 1);
3352 let seat = self.seat(&seat_key, &spec.id);
3353 jobs.push(SeatJob {
3354 prompt: prompt::judge(
3355 &self.state.instruction,
3356 &views,
3357 self.roles.judges.len(),
3358 &base_short,
3359 &language,
3360 ),
3361 spec,
3362 seat,
3363 cwd: root.join(format!("judge-{}", j + 1)),
3364 timeout,
3365 allow_write: false,
3366 sessions,
3367 artifacts: artifacts.clone(),
3368 stem: format!("judge-{}", j + 1),
3369 handover: None,
3370 });
3371 }
3372
3373 self.state.event(
3374 "judge",
3375 format!(
3376 "{} judges ranking {} candidates blind",
3377 jobs.len(),
3378 viable.len()
3379 ),
3380 );
3381 let labels_for_check = labels.clone();
3382 let mut quota_losses = Vec::new();
3383 let cache = self.state.config.cache_dir();
3384 let ctx = WaveCtx {
3385 carry_seats: false,
3386 pauses: Some((&self.pause, &self.interrupt)),
3387 run: &run_id,
3388 node: "judge",
3389 prompts: &prompts,
3390 cache: cache.as_deref(),
3391 round: None,
3392 };
3393 let results = ask_json_wave::<Ranking>(
3394 jobs,
3395 Arc::clone(&self.sem),
3396 self.state.config.graph.retries,
3397 &self.roles.judge_roster,
3398 &ctx,
3399 &mut quota_losses,
3400 &mut self.state,
3401 &move |r: &Ranking| r.validate(&labels_for_check),
3402 )
3403 .await;
3404 if self.state.resource_block.is_some() {
3405 return self.stop_for_resource();
3406 }
3407 self.state.quota.extend(quota_losses);
3408
3409 for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
3410 let agent_id = seat.agent.clone();
3411 self.state.seats.insert(seat.key.clone(), seat);
3412 let mut record = Judgement {
3413 judge: j + 1,
3414 seat: format!("judge-{}", j + 1),
3415 agent: agent_id,
3416 ranking: Vec::new(),
3417 reasons: BTreeMap::new(),
3418 confidence: None,
3419 strengths: BTreeMap::new(),
3420 order: orders[j].clone(),
3421 failed: None,
3422 duration_ms: 0,
3423 };
3424 match res {
3425 Ok((ranking, out)) => {
3426 record.ranking = ranking.normalized();
3427 record.reasons = ranking.reasons;
3428 record.strengths = ranking.strengths;
3429 record.confidence = ranking.confidence;
3430 record.duration_ms = out.duration_ms;
3431 self.state.event(
3432 "judge",
3433 format!(
3434 "judge {} ranked {}",
3435 j + 1,
3436 record.ranking.iter().collect::<String>()
3437 ),
3438 );
3439 }
3440 Err(e) => {
3441 record.failed = Some(e.to_string());
3442 self.state
3443 .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
3444 }
3445 }
3446 self.state.judgements.push(record);
3447 self.state.save()?;
3448 }
3449 Ok(())
3450 }
3451
3452 async fn deliberate(&mut self) -> Result<()> {
3455 let run_id = self.state.id.clone();
3460 let prompts = self.state.config.prompts.clone();
3461 if !self.state.deliberation.is_empty() {
3462 return Ok(());
3463 }
3464 let tops: Vec<char> = self
3465 .state
3466 .judgements
3467 .iter()
3468 .filter_map(|j| j.ranking.first().copied())
3469 .collect();
3470 let rounds = self.state.config.graph.deliberate_rounds;
3471 if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
3472 if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
3473 self.state.event(
3474 "deliberate",
3475 format!("judges agreed on {} outright; no deliberation", tops[0]),
3476 );
3477 }
3478 self.state.status = RunStatus::Voting;
3479 self.state.save()?;
3480 return Ok(());
3481 }
3482
3483 self.state.status = RunStatus::Deliberating;
3484 self.state.event(
3485 "deliberate",
3486 format!(
3487 "split: first choices were {} — opening {rounds} round(s)",
3488 tops.iter().collect::<String>()
3489 ),
3490 );
3491
3492 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3493 let language = self.state.config.graph.language.clone();
3494 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3495 let sessions = self.state.config.graph.sessions;
3496 let artifacts = agent::artifacts_dir(&self.state.dir());
3497 let root = self.state.worktree_root();
3498 let base_short = short(&self.state.base_commit);
3499
3500 for round in 1..=rounds {
3504 let mut turns: Vec<DeliberationTurn> = Vec::new();
3505 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3506 if self.state.judgements[j].failed.is_some() {
3507 continue;
3508 }
3509 let seat_key = format!("judge-{}", j + 1);
3510 let spec = self.occupant(&seat_key, spec);
3511 let mut seat = self.seat(&seat_key, &spec.id);
3512 let transcript = self.transcript(&turns, j);
3513 let build = |context: Option<&str>| {
3514 prompt::deliberate(
3515 &self.state.instruction,
3516 context,
3517 &transcript,
3518 round,
3519 rounds,
3520 &language,
3521 )
3522 };
3523 let block = self.candidate_block(&viable, &base_short);
3524 let full = build(Some(&block));
3525 let text = if has_context(&spec, &seat, sessions) {
3526 build(None)
3527 } else {
3528 full.clone()
3529 };
3530 let job = SeatJob {
3531 spec,
3532 seat: seat.clone(),
3533 prompt: text,
3534 cwd: root.join(format!("judge-{}", j + 1)),
3535 timeout,
3536 allow_write: false,
3537 sessions,
3538 artifacts: artifacts.clone(),
3539 stem: format!("delib-{round}-judge-{}", j + 1),
3540 handover: Some(full),
3541 };
3542 let cache = self.state.config.cache_dir();
3543 let ctx = WaveCtx {
3544 carry_seats: false,
3545 pauses: Some((&self.pause, &self.interrupt)),
3546 run: &run_id,
3547 node: "deliberate",
3548 prompts: &prompts,
3549 cache: cache.as_deref(),
3550 round: None,
3551 };
3552 let mut losses = Vec::new();
3556 let mut results = ask_wave_with::<String>(
3557 vec![job],
3558 Arc::clone(&self.sem),
3559 0,
3560 &self.roles.judge_roster,
3561 &ctx,
3562 &mut losses,
3563 &mut self.state,
3564 &|text: &str| {
3565 Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
3566 },
3567 )
3568 .await;
3569 self.state.quota.extend(losses);
3570 let (updated, res, _) = results.pop().expect("one job in, one result out");
3571 seat = updated;
3572 let agent_id = seat.agent.clone();
3573 self.state.seats.insert(seat.key.clone(), seat);
3574 let body = match res {
3575 Ok((body, _)) => body,
3576 Err(e) => {
3579 self.state
3580 .event("deliberate", format!("judge {} skipped: {e}", j + 1));
3581 continue;
3582 }
3583 };
3584 let tentative = verdict::extract_json::<Position>(&body)
3585 .ok()
3586 .and_then(|p| p.tentative)
3587 .and_then(|s| s.trim().chars().next())
3588 .map(|c| c.to_ascii_uppercase());
3589 self.state.event(
3590 "deliberate",
3591 format!(
3592 "round {round}: judge {} now favours {}",
3593 j + 1,
3594 tentative.map_or("—".to_owned(), |c| c.to_string())
3595 ),
3596 );
3597 turns.push(DeliberationTurn {
3598 judge: j + 1,
3599 agent: agent_id,
3600 body: blind::sanitize_prose(&body, &self.state.config.blind),
3601 tentative,
3602 });
3603 }
3604 self.state
3605 .deliberation
3606 .push(DeliberationRound { round, turns });
3607 self.state.save()?;
3608 }
3609
3610 self.state.status = RunStatus::Voting;
3611 self.state.save()?;
3612 Ok(())
3613 }
3614
3615 async fn vote(&mut self) -> Result<()> {
3618 let run_id = self.state.id.clone();
3623 let prompts = self.state.config.prompts.clone();
3624 if !self.state.votes.is_empty() {
3625 return Ok(());
3626 }
3627 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3628 if viable.len() == 1 {
3629 return Ok(());
3630 }
3631 self.state.status = RunStatus::Voting;
3632
3633 let language = self.state.config.graph.language.clone();
3634 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3635 let sessions = self.state.config.graph.sessions;
3636 let artifacts = agent::artifacts_dir(&self.state.dir());
3637 let root = self.state.worktree_root();
3638 let base_short = short(&self.state.base_commit);
3639 let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3640
3641 let mut jobs = Vec::new();
3642 let mut seats_at = Vec::new();
3643 for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3644 if self
3645 .state
3646 .judgements
3647 .get(j)
3648 .is_some_and(|r| r.failed.is_some())
3649 {
3650 continue;
3651 }
3652 let seat_key = format!("judge-{}", j + 1);
3653 let spec = self.occupant(&seat_key, spec);
3654 let seat = self.seat(&seat_key, &spec.id);
3655 let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
3656 let text = if has_context(&spec, &seat, sessions) {
3657 prompt::final_vote(&viable, &language)
3658 } else {
3659 full.clone()
3660 };
3661 jobs.push(SeatJob {
3662 spec,
3663 seat,
3664 prompt: text,
3665 cwd: root.join(format!("judge-{}", j + 1)),
3666 timeout,
3667 allow_write: false,
3668 sessions,
3669 artifacts: artifacts.clone(),
3670 stem: format!("vote-judge-{}", j + 1),
3671 handover: Some(full),
3672 });
3673 seats_at.push(j);
3674 }
3675
3676 self.state.event(
3677 "vote",
3678 format!(
3679 "collecting {} final votes one by one, privately",
3680 jobs.len()
3681 ),
3682 );
3683 let allowed = viable.clone();
3684 let mut quota_losses = Vec::new();
3685 let cache = self.state.config.cache_dir();
3686 let ctx = WaveCtx {
3687 carry_seats: false,
3688 pauses: Some((&self.pause, &self.interrupt)),
3689 run: &run_id,
3690 node: "vote",
3691 prompts: &prompts,
3692 cache: cache.as_deref(),
3693 round: None,
3694 };
3695 let results = ask_json_wave::<FinalVote>(
3696 jobs,
3697 Arc::clone(&self.sem),
3698 self.state.config.graph.retries,
3699 &self.roles.judge_roster,
3700 &ctx,
3701 &mut quota_losses,
3702 &mut self.state,
3703 &move |v: &FinalVote| match v.label() {
3704 Some(c) if allowed.contains(&c) => Ok(()),
3705 other => bail!("vote {other:?} is not one of {allowed:?}"),
3706 },
3707 )
3708 .await;
3709 if self.state.resource_block.is_some() {
3710 return self.stop_for_resource();
3711 }
3712 self.state.quota.extend(quota_losses);
3713
3714 for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
3715 let agent_id = seat.agent.clone();
3716 self.state.seats.insert(seat.key.clone(), seat);
3717 let initial = self
3718 .state
3719 .judgements
3720 .get(j)
3721 .and_then(|r| r.ranking.first().copied());
3722 let mut record = VoteRecord {
3723 judge: j + 1,
3724 agent: agent_id,
3725 vote: None,
3726 reason: String::new(),
3727 changed: false,
3728 };
3729 match res {
3730 Ok((v, _)) => {
3731 record.vote = v.label();
3732 record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3733 record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
3734 self.state.event(
3735 "vote",
3736 format!(
3737 "judge {} voted {}{}",
3738 j + 1,
3739 record.vote.unwrap_or('?'),
3740 if record.changed { " (changed)" } else { "" }
3741 ),
3742 );
3743 }
3744 Err(e) => {
3745 self.state
3746 .event("vote", format!("judge {} cast no vote: {e}", j + 1));
3747 }
3748 }
3749 self.state.votes.push(record);
3750 self.state.save()?;
3751 }
3752 Ok(())
3753 }
3754
3755 fn tally(&mut self) -> Result<()> {
3758 if self.state.tally.is_some() {
3759 return Ok(());
3760 }
3761 let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
3762 let tops: Vec<char> = self
3763 .state
3764 .judgements
3765 .iter()
3766 .filter_map(|j| j.ranking.first().copied())
3767 .collect();
3768 let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
3769
3770 let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3773 let mut cast: Vec<char> = Vec::new();
3774 for (i, j) in self.state.judgements.iter().enumerate() {
3775 let vote = self
3776 .state
3777 .votes
3778 .iter()
3779 .find(|v| v.judge == i + 1)
3780 .and_then(|v| v.vote)
3781 .or_else(|| j.ranking.first().copied());
3782 if let Some(v) = vote {
3783 *first_choice.entry(v).or_insert(0) += 1;
3784 cast.push(v);
3785 }
3786 }
3787
3788 let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
3789 for j in &self.state.judgements {
3790 let n = j.ranking.len();
3791 for (pos, label) in j.ranking.iter().enumerate() {
3792 *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
3793 }
3794 }
3795
3796 let best = first_choice.values().copied().max().unwrap_or(0);
3797 let mut leaders: Vec<char> = first_choice
3798 .iter()
3799 .filter(|(_, v)| **v == best)
3800 .map(|(k, _)| *k)
3801 .collect();
3802 let mut tie_break = None;
3803 if leaders.len() > 1 {
3804 let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
3805 let borda_leaders: Vec<char> = leaders
3806 .iter()
3807 .copied()
3808 .filter(|l| borda[l] == top_borda)
3809 .collect();
3810 tie_break = Some(if borda_leaders.len() == 1 {
3811 format!(
3812 "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
3813 leaders.len()
3814 )
3815 } else {
3816 format!(
3817 "{} way tie on both first-choice votes and Borda points, broken by label order",
3818 leaders.len()
3819 )
3820 });
3821 leaders = borda_leaders;
3822 leaders.sort_unstable();
3823 }
3824 let winner = *leaders
3825 .first()
3826 .or(viable.first())
3827 .context("no candidate to declare a winner from")?;
3828
3829 let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
3830 let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
3831 let deliberated = !self.state.deliberation.is_empty();
3832
3833 let quota_seats: std::collections::BTreeSet<&str> =
3837 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3838 let mut present = 0usize;
3839 for (i, j) in self.state.judgements.iter().enumerate() {
3840 if quota_seats.contains(j.seat.as_str()) {
3841 continue;
3842 }
3843 let ranked = !j.ranking.is_empty() && j.failed.is_none();
3844 let voted = self
3845 .state
3846 .votes
3847 .iter()
3848 .any(|v| v.judge == i + 1 && v.vote.is_some());
3849 if ranked || voted {
3850 present += 1;
3851 }
3852 }
3853 let needs_quorum = viable.len() > 1;
3859 let judges_total = if needs_quorum {
3860 self.roles.judges.len()
3861 } else {
3862 0
3863 };
3864 let quorum = if needs_quorum {
3865 judges_total / 2 + 1
3866 } else {
3867 0
3868 };
3869 let met_quorum = !needs_quorum || present >= quorum;
3870 let uncontested = (!needs_quorum).then(|| {
3871 format!("only one candidate ({winner}) produced a usable change; no panel was asked")
3872 });
3873
3874 self.state.event(
3875 "tally",
3876 match &uncontested {
3877 Some(reason) => format!("winner {winner} — {reason}"),
3878 None => format!(
3879 "winner {winner} — votes {} | initial {} | {} changed | \
3880 {present}/{judges_total} judges{}",
3881 first_choice
3882 .iter()
3883 .map(|(k, v)| format!("{k}:{v}"))
3884 .collect::<Vec<_>>()
3885 .join(" "),
3886 if unanimous_initial {
3887 "unanimous"
3888 } else {
3889 "split"
3890 },
3891 changed_votes,
3892 if met_quorum {
3893 String::new()
3894 } else {
3895 format!(" — below quorum ({quorum} required)")
3896 },
3897 ),
3898 },
3899 );
3900 if !met_quorum {
3901 self.state.event(
3902 "stall",
3903 format!(
3904 "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
3905 the run stops here, resumable"
3906 ),
3907 );
3908 }
3909 self.state.tally = Some(Tally {
3910 first_choice,
3911 borda,
3912 winner,
3913 rankings: tops.len(),
3914 unanimous_initial,
3915 deliberated,
3916 changed_votes,
3917 unanimous_final,
3918 tie_break,
3919 judges: judges_total,
3920 present,
3921 quorum,
3922 met_quorum,
3923 uncontested,
3924 });
3925 self.state.status = if met_quorum {
3926 RunStatus::Reviewing
3927 } else {
3928 RunStatus::Stalled
3929 };
3930 self.state.save()?;
3931 Ok(())
3932 }
3933
3934 #[allow(clippy::too_many_lines)]
3955 async fn recover_stall(&mut self) -> Result<bool> {
3956 let run_id = self.state.id.clone();
3961 let prompts = self.state.config.prompts.clone();
3962 let quota_seats: BTreeSet<&str> =
3967 self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3968 let absent: Vec<String> = self
3969 .state
3970 .judgements
3971 .iter()
3972 .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3973 .map(|j| j.seat.clone())
3974 .collect();
3975 if absent.is_empty() {
3976 return Ok(false);
3977 }
3978 let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3979 if viable.len() <= 1 {
3980 return Ok(false);
3981 }
3982 let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983 let language = self.state.config.graph.language.clone();
3984 let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985 let sessions = self.state.config.graph.sessions;
3986 let artifacts = agent::artifacts_dir(&self.state.dir());
3987 let root = self.state.worktree_root();
3988 let base_short = short(&self.state.base_commit);
3989 let candidates: Vec<Candidate> = viable.clone();
3990
3991 let mut positions: Vec<usize> = absent
3993 .iter()
3994 .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3995 .collect();
3996 if positions.is_empty() {
3997 return Ok(false);
3998 }
3999 positions.sort_unstable();
4000 positions.dedup();
4001
4002 let mut judge_jobs = Vec::new();
4004 for &j in &positions {
4005 let order = blind::presentation_order(viable.len(), j, self.state.seed);
4006 let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4007 let seat_key = format!("judge-{}", j + 1);
4008 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4009 let seat = self.seat(&seat_key, &spec.id);
4010 judge_jobs.push(SeatJob {
4011 spec,
4012 seat,
4013 prompt: prompt::judge(
4014 &self.state.instruction,
4015 &views,
4016 self.roles.judges.len(),
4017 &base_short,
4018 &language,
4019 ),
4020 cwd: root.join(seat_key),
4021 timeout,
4022 allow_write: false,
4023 sessions,
4024 artifacts: artifacts.clone(),
4025 stem: format!("judge-{}-recover", j + 1),
4026 handover: None,
4027 });
4028 }
4029
4030 let labels_for_check = labels.clone();
4031 let mut judge_losses = Vec::new();
4032 let retries = self.state.config.graph.retries;
4033 let cache = self.state.config.cache_dir();
4034 let ctx = WaveCtx {
4035 carry_seats: false,
4036 pauses: Some((&self.pause, &self.interrupt)),
4037 run: &run_id,
4038 node: "judge",
4039 prompts: &prompts,
4040 cache: cache.as_deref(),
4041 round: None,
4042 };
4043 let results = ask_json_wave::<Ranking>(
4044 judge_jobs,
4045 Arc::clone(&self.sem),
4046 retries,
4047 &self.roles.judge_roster,
4048 &ctx,
4049 &mut judge_losses,
4050 &mut self.state,
4051 &move |r: &Ranking| r.validate(&labels_for_check),
4052 )
4053 .await;
4054 if self.state.resource_block.is_some() {
4055 self.stop_for_resource()?;
4056 return Ok(false);
4057 }
4058
4059 let mut recovered: BTreeSet<usize> = BTreeSet::new();
4061 for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4062 let agent_id = seat.agent.clone();
4063 self.state.seats.insert(seat.key.clone(), seat);
4064 let record = &mut self.state.judgements[j];
4065 match res {
4066 Ok((ranking, out)) => {
4067 record.agent = agent_id;
4068 record.ranking = ranking.normalized();
4069 record.reasons = ranking.reasons;
4070 record.strengths = ranking.strengths;
4071 record.confidence = ranking.confidence;
4072 record.failed = None;
4073 record.duration_ms = out.duration_ms;
4074 recovered.insert(j);
4075 self.state.event(
4076 "recover",
4077 format!("judge {} ranked again after the limit", j + 1),
4078 );
4079 }
4080 Err(e) => {
4081 self.state
4082 .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4083 }
4084 }
4085 }
4086
4087 let mut vote_jobs = Vec::new();
4089 let mut vote_pos: Vec<usize> = Vec::new();
4090 for &j in &recovered {
4091 let seat_key = format!("judge-{}", j + 1);
4092 let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4093 let seat = self.seat(&seat_key, &spec.id);
4094 let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4095 let text = if has_context(&spec, &seat, sessions) {
4096 prompt::final_vote(&labels, &language)
4097 } else {
4098 full.clone()
4099 };
4100 vote_jobs.push(SeatJob {
4101 spec,
4102 seat,
4103 prompt: text,
4104 cwd: root.join(seat_key),
4105 timeout,
4106 allow_write: false,
4107 sessions,
4108 artifacts: artifacts.clone(),
4109 stem: format!("vote-judge-{}-recover", j + 1),
4110 handover: Some(full),
4111 });
4112 vote_pos.push(j);
4113 }
4114 let allowed = labels.clone();
4115 let mut vote_losses = Vec::new();
4116 let vote_retries = self.state.config.graph.retries;
4117 let vote_cache = self.state.config.cache_dir();
4118 let ctx = WaveCtx {
4119 carry_seats: false,
4120 pauses: Some((&self.pause, &self.interrupt)),
4121 run: &run_id,
4122 node: "vote",
4123 prompts: &prompts,
4124 cache: vote_cache.as_deref(),
4125 round: None,
4126 };
4127 let votes = ask_json_wave::<FinalVote>(
4128 vote_jobs,
4129 Arc::clone(&self.sem),
4130 vote_retries,
4131 &self.roles.judge_roster,
4132 &ctx,
4133 &mut vote_losses,
4134 &mut self.state,
4135 &move |v: &FinalVote| match v.label() {
4136 Some(c) if allowed.contains(&c) => Ok(()),
4137 other => bail!("vote {other:?} is not one of {allowed:?}"),
4138 },
4139 )
4140 .await;
4141 if self.state.resource_block.is_some() {
4142 self.stop_for_resource()?;
4143 return Ok(false);
4144 }
4145 for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4146 let agent_id = seat.agent.clone();
4147 self.state.seats.insert(seat.key.clone(), seat);
4148 match res {
4149 Ok((v, _)) => {
4150 if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4151 rec.vote = v.label();
4152 rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4153 } else {
4154 self.state.votes.push(VoteRecord {
4155 judge: j + 1,
4156 agent: agent_id,
4157 vote: v.label(),
4158 reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4159 changed: false,
4160 });
4161 }
4162 self.state.event(
4163 "recover",
4164 format!("judge {} voted again after the limit", j + 1),
4165 );
4166 }
4167 Err(e) => {
4168 self.state
4169 .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4170 }
4171 }
4172 }
4173
4174 let recovered_keys: BTreeSet<String> = recovered
4178 .iter()
4179 .map(|&j| format!("judge-{}", j + 1))
4180 .collect();
4181 self.state
4182 .quota
4183 .retain(|q| !recovered_keys.contains(&q.seat));
4184 for loss in judge_losses.into_iter().chain(vote_losses) {
4188 if recovered_keys.contains(&loss.seat) {
4189 continue;
4190 }
4191 self.state.quota.retain(|q| q.seat != loss.seat);
4192 self.state.quota.push(loss);
4193 }
4194
4195 self.state.tally = None;
4197 self.tally()?;
4198 Ok(self
4199 .state
4200 .tally
4201 .as_ref()
4202 .map(|t| t.met_quorum)
4203 .unwrap_or(false))
4204 }
4205
4206 async fn graft(&mut self) -> Result<()> {
4222 let Err(e) = self.graft_inner().await else {
4223 return Ok(());
4224 };
4225 if e.is::<crate::repo_lock::ResourceBlocked>() {
4226 return Err(e);
4227 }
4228 if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4229 self.state.status = RunStatus::Blocked;
4230 self.state.event("graft", format!("blocked: {e:#}"));
4231 self.state.save()?;
4232 return Err(e);
4233 }
4234 if let Some(rec) = self.state.graft.clone()
4239 && !rec.outcome.settled()
4240 {
4241 if let Some(winner) = self.state.winner().cloned() {
4242 if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4243 if re.is::<crate::repo_lock::ResourceBlocked>() {
4244 return Err(re);
4245 }
4246 self.state.status = RunStatus::Blocked;
4247 self.state.event("graft", format!("blocked: {re:#}"));
4248 self.state.save()?;
4249 return Err(re);
4250 }
4251 }
4252 if let Some(g) = self.state.graft.as_mut() {
4253 g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4254 }
4255 }
4256 self.state.event("graft", format!("skipped: {e:#}"));
4257 if let Err(e) = self.state.save() {
4258 tracing::warn!("save after graft: {e:#}");
4259 }
4260 Ok(())
4261 }
4262
4263 async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4266 let _coordination = self.coordinate(wt).await?;
4267 crate::recovery::restore_repair(wt, pre)
4268 .await
4269 .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4270 }
4271
4272 fn graft_skipped(&mut self, why: &str) {
4274 let message = format!("skipped: {why}");
4275 if !self
4276 .state
4277 .events
4278 .iter()
4279 .any(|e| e.node == "graft" && e.message == message)
4280 {
4281 self.state.event("graft", message);
4282 }
4283 }
4284
4285 async fn graft_inner(&mut self) -> Result<()> {
4286 if let Some(rec) = self.state.graft.clone()
4289 && !rec.outcome.settled()
4290 {
4291 if self.state.reviews.is_empty()
4295 && let Some(winner) = self.state.winner().cloned()
4296 {
4297 self.graft_restore(&winner.worktree, &rec.pre).await?;
4298 }
4299 if rec.outcome == GraftOutcome::WaitingForRestore {
4300 self.state.graft = None;
4301 self.state.save()?;
4302 } else {
4303 if let Some(g) = self.state.graft.as_mut() {
4304 g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4305 }
4306 self.state.event(
4307 "graft",
4308 "rolled back: the previous round was interrupted; the winner is untouched",
4309 );
4310 self.state.save()?;
4311 return Ok(());
4312 }
4313 }
4314 let plan = match graft::plan(&self.state) {
4315 Ok(plan) => plan,
4316 Err(skip) => {
4317 if skip != graft::Skip::AlreadyDone {
4318 self.graft_skipped(skip.reason());
4319 self.state.save()?;
4320 }
4321 return Ok(());
4322 }
4323 };
4324 let Some(winner) = self.state.winner().cloned() else {
4325 return Ok(());
4326 };
4327 let wt = winner.worktree.clone();
4328 if !git::is_clean(&wt).await? {
4329 self.graft_skipped("the winner's worktree has uncommitted changes");
4330 self.state.save()?;
4331 return Ok(());
4332 }
4333 let pre = git::rev_parse(&wt, "HEAD").await?;
4334 self.state.graft = Some(GraftRecord {
4337 pre: pre.clone(),
4338 post: None,
4339 lines: None,
4340 limit: None,
4341 agent: None,
4342 outcome: GraftOutcome::Running,
4343 requested: graft::requested(&plan),
4344 adoption: None,
4345 adoption_error: None,
4346 });
4347 self.state.event(
4348 "graft",
4349 format!(
4350 "porting strengths of candidate(s) {} onto {}",
4351 plan.strengths
4352 .iter()
4353 .map(|(l, _, _)| l.to_string())
4354 .collect::<Vec<_>>()
4355 .join(", "),
4356 plan.winner
4357 ),
4358 );
4359 self.state.save()?;
4360
4361 let fix_prompt = prompt::graft(
4362 &self.state.instruction,
4363 plan.winner,
4364 &plan.strengths,
4365 &self.state.config.graph.language,
4366 );
4367 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
4368 let sessions = self.state.config.graph.sessions;
4369 let artifacts = agent::artifacts_dir(&self.state.dir());
4370 let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
4371 let (_, seat, out) = self
4372 .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
4373 SeatJob {
4374 prompt: fix_prompt.clone(),
4375 spec,
4376 seat,
4377 cwd: wt.clone(),
4378 timeout,
4379 allow_write: true,
4380 sessions,
4381 artifacts: artifacts.clone(),
4382 stem: "graft".to_owned(),
4383 handover: None,
4384 }
4385 })
4386 .await;
4387 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
4388 self.state.graft.as_mut().expect("graft record").outcome =
4391 GraftOutcome::WaitingForRestore;
4392 return self.stop_for_resource();
4393 }
4394 let mut why: Option<String> = None;
4395 match &out {
4396 AgentOutcome::Ok(o) if agent::output_advances(o) => {
4397 why = Some("the fixer produced nothing usable".to_owned());
4398 }
4399 AgentOutcome::Ok(_) => {}
4400 AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
4401 AgentOutcome::Quota(o) => {
4402 self.state.quota.push(QuotaLoss {
4403 seat: seat.key.clone(),
4404 node: "graft".to_owned(),
4405 at: Timestamp::now(),
4406 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4407 });
4408 why = Some("rate limited (quota)".to_owned());
4409 }
4410 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
4411 why = Some(format!(
4412 "the fixer failed: {}",
4413 e.lines()
4414 .next()
4415 .unwrap_or("")
4416 .chars()
4417 .take(160)
4418 .collect::<String>()
4419 ));
4420 }
4421 }
4422 let agent_id = seat.agent.clone();
4423 self.state.seats.insert(seat.key.clone(), seat);
4424
4425 let mut post = None;
4426 let mut lines = None;
4427 let _coordination = self.coordinate(&wt).await?;
4428 if why.is_none() {
4429 let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
4430 if let Ok(r) = &rescued {
4431 self.state.note_withheld("graft", &r.withheld);
4432 }
4433 let head = git::rev_parse(&wt, "HEAD").await?;
4434 let dirty = match &rescued {
4438 Ok(r) => git::status_porcelain(&wt)
4439 .await
4440 .map(|st| {
4441 st.lines().any(|l| {
4442 !r.withheld
4443 .iter()
4444 .any(|w| l.trim_end().ends_with(w.path.as_str()))
4445 })
4446 })
4447 .unwrap_or(true),
4448 Err(_) => true,
4449 };
4450 if dirty {
4451 why = Some("the fixer left changes that could not be committed".to_owned());
4452 } else if head == pre {
4453 why = Some("the fixer changed nothing".to_owned());
4454 } else {
4455 lines = git::changed_lines(&wt, &pre, &head).await;
4456 post = Some(head);
4457 }
4458 }
4459 let (adoption, adoption_error) = match &out {
4460 AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
4461 _ => (None, None),
4462 };
4463 let requested = graft::requested(&plan);
4464 let outcome = match why {
4465 None => {
4466 self.state.event(
4467 "graft",
4468 format!(
4469 "applied: {}",
4470 lines.map_or_else(
4471 || "changed lines unavailable".to_owned(),
4472 |n| format!("{n} changed lines")
4473 )
4474 ),
4475 );
4476 GraftOutcome::Applied
4477 }
4478 Some(why) => {
4479 crate::recovery::restore_repair(&wt, &pre)
4482 .await
4483 .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
4484 self.state.event(
4485 "graft",
4486 format!("rolled back: {why}; the winner is untouched"),
4487 );
4488 GraftOutcome::RolledBack(why)
4489 }
4490 };
4491 self.state.graft = Some(GraftRecord {
4492 pre,
4493 post,
4494 lines,
4495 limit: None,
4496 agent: Some(agent_id),
4497 outcome,
4498 requested,
4499 adoption,
4500 adoption_error,
4501 });
4502 self.state.save()?;
4503 Ok(())
4504 }
4505
4506 async fn fold_losers(&mut self) -> Result<()> {
4509 let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
4510 return Ok(());
4511 };
4512 if !self.state.candidate_preservation.is_empty() {
4513 bail!("candidate preservation incomplete; refusing cleanup");
4514 }
4515 let repo = self.state.repo.clone();
4516 let _coordination = self.coordinate(&repo).await?;
4517 let mut folded = Vec::new();
4518 for i in 0..self.state.candidates.len() {
4519 let c = &self.state.candidates[i];
4520 if c.label == winner || c.folded {
4521 continue;
4522 }
4523 let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
4524 remove_preserved_worktree(&repo, &wt).await?;
4525 if git::branch_exists(&repo, &branch).await?
4526 && !git::branch_delete(&repo, &branch).await?
4527 {
4528 bail!("candidate branch removal refused");
4529 }
4530 self.state.candidates[i].folded = true;
4531 self.state.save()?;
4532 folded.push(label.to_string());
4533 }
4534 let root = self.state.worktree_root();
4536 for j in 1..=self.roles.judges.len() {
4537 let wt = root.join(format!("judge-{j}"));
4538 if wt.exists() {
4539 remove_preserved_worktree(&repo, &wt).await?;
4540 }
4541 }
4542 if self.state.config.graph.advise {
4545 for k in 1..=self.state.config.graph.advisors {
4546 let wt = root.join(format!("advisor-{k}"));
4547 if wt.exists() {
4548 remove_preserved_worktree(&repo, &wt).await?;
4549 }
4550 }
4551 }
4552 if !folded.is_empty() {
4553 self.state
4554 .event("fold", format!("folded candidates {}", folded.join(", ")));
4555 self.state.save()?;
4556 }
4557 Ok(())
4558 }
4559
4560 async fn sync_to_base(&mut self) -> Result<()> {
4600 if self.state.status == RunStatus::AlreadyInBase {
4601 return Ok(());
4602 }
4603 let conflicted = self
4604 .state
4605 .base_sync
4606 .as_ref()
4607 .is_some_and(|s| s.conflict.is_some());
4608 let Some(winner) = self.state.winner().cloned() else {
4609 return Ok(());
4610 };
4611
4612 let repo = self.state.repo.clone();
4613 let remote = self.state.config.merge.remote.clone();
4614 let base_branch = self.state.base_branch.clone();
4615 let tracking = format!("{remote}/{base_branch}");
4616
4617 git::fetch(&repo, &remote, &base_branch).await.ok();
4618 let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
4623 return Ok(());
4624 };
4625
4626 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4627 let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
4628 let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
4629
4630 if (behind > 0 || conflicted || head == tip)
4639 && self
4640 .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
4641 .await?
4642 {
4643 return Ok(());
4644 }
4645 if conflicted {
4646 return Ok(());
4647 }
4648
4649 if behind == 0 {
4650 if let Some(from) = self
4659 .state
4660 .rebase_fixes
4661 .iter()
4662 .rev()
4663 .find_map(|r| r.from.clone())
4664 && from != head
4665 && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
4666 .await
4667 .is_ok_and(|o| o.ok())
4668 {
4669 git::sync_to_head(&winner.worktree).await?;
4670 }
4671 let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
4677 if let Some(why) = &conflict {
4678 self.state.status = RunStatus::Blocked;
4679 self.state.event("land", why.clone());
4680 }
4681 self.state.base_sync = Some(BaseSync {
4682 tip,
4683 behind: 0,
4684 attempts,
4685 conflict,
4686 already_in: None,
4687 });
4688 self.state.save()?;
4689 return Ok(());
4690 }
4691
4692 if attempts >= BASE_SYNC_ROUNDS {
4693 let why = format!(
4694 "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
4695 rebase(s); rebasing again would only race it",
4696 winner.branch
4697 );
4698 self.state.status = RunStatus::Blocked;
4699 self.state.base_sync = Some(BaseSync {
4700 tip,
4701 behind,
4702 attempts,
4703 conflict: Some(why.clone()),
4704 already_in: None,
4705 });
4706 self.state.event("land", why);
4707 self.state.save()?;
4708 return Ok(());
4709 }
4710
4711 self.state.event(
4712 "land",
4713 format!(
4714 "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
4715 winner.branch
4716 ),
4717 );
4718 self.state.save()?;
4719
4720 let branch_tracking = format!("{remote}/{}", winner.branch);
4726 let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
4727 let remote_tip = if self.state.task_pr_branch().is_none()
4728 && matches!(&fetched_branch, Ok(o) if o.ok())
4729 {
4730 git::rev_parse(&repo, &branch_tracking).await.ok()
4731 } else {
4732 None
4733 };
4734 if let Some(theirs) = &remote_tip
4738 && !git::is_ancestor(&repo, theirs, &head).await
4739 && !crate::reconcile::origin_missing(&repo, &head, theirs)
4740 .await
4741 .is_ok_and(|missing| missing.is_empty())
4742 {
4743 let why = format!(
4744 "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
4745 them",
4746 short(theirs),
4747 winner.branch
4748 );
4749 self.state.status = RunStatus::Blocked;
4750 self.state.base_sync = Some(BaseSync {
4751 tip,
4752 behind,
4753 attempts,
4754 conflict: Some(why.clone()),
4755 already_in: None,
4756 });
4757 self.state.event("land", why);
4758 self.state.save()?;
4759 return Ok(());
4760 }
4761
4762 let scratch = self.state.dir().join("base-sync");
4763 let rebased = match crate::rebase::rebase_with_fixer(
4764 &mut self.state,
4765 &scratch,
4766 &winner.branch,
4767 &tracking,
4768 )
4769 .await
4770 {
4771 Ok(crate::rebase::Rebased::Applied) => Ok(None),
4772 Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
4773 Err(e) => Err(e),
4774 };
4775 let attempts = attempts + 1;
4776 match rebased {
4777 Ok(None) => {
4778 git::sync_to_head(&winner.worktree).await?;
4782 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
4783 let mut conflict = None;
4784 if let Some(pinned) = &remote_tip {
4785 let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
4786 match pushed {
4787 Ok(o) if o.ok() => self.state.event(
4788 "land",
4789 format!("pushed rebased {} to {remote}", winner.branch),
4790 ),
4791 Ok(o) => {
4792 conflict = Some(format!(
4793 "rebased {} locally but {remote} refused the push (it moved since {}; someone may have pushed): {}",
4794 winner.branch,
4795 short(pinned),
4796 o.stderr.chars().take(600).collect::<String>()
4797 ));
4798 }
4799 Err(e) => {
4800 conflict = Some(format!(
4801 "rebased {} locally but could not push it: {e:#}",
4802 winner.branch
4803 ));
4804 }
4805 }
4806 }
4807 if let Some(why) = &conflict {
4808 self.state.status = RunStatus::Blocked;
4809 self.state.event("land", why.clone());
4810 }
4811 self.state.base_sync = Some(BaseSync {
4812 tip: tip.clone(),
4813 behind: 0,
4814 attempts,
4815 conflict,
4816 already_in: None,
4817 });
4818 self.state
4819 .event("land", format!("rebased {} onto {tracking}", winner.branch));
4820 }
4821 Ok(Some(conflict)) => {
4822 let why = format!(
4823 "{} conflicts with {tracking} and did not rebase: {}",
4824 winner.branch,
4825 conflict.chars().take(600).collect::<String>()
4826 );
4827 self.state.status = RunStatus::Blocked;
4828 self.state.base_sync = Some(BaseSync {
4829 tip,
4830 behind,
4831 attempts,
4832 conflict: Some(why.clone()),
4833 already_in: None,
4834 });
4835 self.state.event("land", why);
4836 }
4837 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
4841 self.state.save()?;
4842 return Ok(());
4843 }
4844 Err(e) => {
4845 let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
4846 self.state.status = RunStatus::Blocked;
4847 self.state.base_sync = Some(BaseSync {
4848 tip,
4849 behind,
4850 attempts,
4851 conflict: Some(why.clone()),
4852 already_in: None,
4853 });
4854 self.state.event("land", why);
4855 }
4856 }
4857 self.state.save()?;
4858 Ok(())
4859 }
4860
4861 async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
4867 if self.state.task_pr_branch().is_some() {
4870 return None;
4871 }
4872 let repo = self.state.repo.clone();
4873 let remote = self.state.config.merge.remote.clone();
4874 let fetched = git::fetch(&repo, &remote, branch).await;
4875 if !matches!(&fetched, Ok(o) if o.ok()) {
4876 return None;
4877 }
4878 let branch_tracking = format!("{remote}/{branch}");
4879 let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
4880 if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
4881 return None;
4882 }
4883 if !crate::reconcile::origin_missing(&repo, head, &theirs)
4884 .await
4885 .is_ok_and(|missing| missing.is_empty())
4886 {
4887 return Some(format!(
4888 "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
4889 them",
4890 short(&theirs)
4891 ));
4892 }
4893 match git::push_pinned(&repo, &remote, branch, &theirs).await {
4894 Ok(o) if o.ok() => {
4895 self.state
4896 .event("land", format!("pushed rebased {branch} to {remote}"));
4897 None
4898 }
4899 Ok(o) => Some(format!(
4900 "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
4901 someone may have pushed): {}",
4902 short(&theirs),
4903 o.stderr.chars().take(600).collect::<String>()
4904 )),
4905 Err(e) => Some(format!(
4906 "{branch} is rebased locally but could not be pushed: {e:#}"
4907 )),
4908 }
4909 }
4910
4911 async fn settle_already_in(
4928 &mut self,
4929 branch: &str,
4930 tip: &str,
4931 head: &str,
4932 attempts: usize,
4933 behind: usize,
4934 ) -> Result<bool> {
4935 let repo = self.state.repo.clone();
4936 let remote = self.state.config.merge.remote.clone();
4937 let start = self.state.base_commit.clone();
4938 let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
4939 Ok(Some(e)) => e,
4940 Ok(None) => return Ok(false),
4941 Err(e) => {
4942 tracing::warn!("already-in-base check for {branch}: {e:#}");
4943 return Ok(false);
4944 }
4945 };
4946 let mut verified = vec![head.to_owned()];
4947 let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
4948 let fetched = git::fetch(&repo, &remote, &public_branch).await;
4949 if matches!(&fetched, Ok(o) if o.ok())
4950 && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
4951 && theirs != head
4952 {
4953 match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
4954 Ok(Some(_)) => verified.push(theirs),
4955 _ => return Ok(false),
4956 }
4957 }
4958 let base_branch = self.state.base_branch.clone();
4959 let message = format!(
4960 "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
4961 land",
4962 evidence.names(),
4963 evidence.proof.as_str()
4964 );
4965 let closed =
4966 crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
4967 .await;
4968 match closed {
4969 Ok(Ok(url)) => self
4970 .state
4971 .event("land", format!("closed {url}: superseded on {base_branch}")),
4972 Ok(Err(why)) => self
4973 .state
4974 .event("land", format!("did not close a pull request: {why}")),
4975 Err(e) => {
4976 let why = format!(
4977 "{branch} is already in {remote}/{base_branch}, but its pull request could \
4978 not be closed ({e:#}); resume to retry"
4979 );
4980 self.state.status = RunStatus::Blocked;
4981 self.state.base_sync = Some(BaseSync {
4982 tip: tip.to_owned(),
4983 behind,
4984 attempts,
4985 conflict: Some(why.clone()),
4986 already_in: None,
4987 });
4988 self.state.event("land", why);
4989 self.state.save()?;
4990 return Ok(true);
4991 }
4992 }
4993 self.state.status = RunStatus::AlreadyInBase;
4994 self.state.base_sync = Some(BaseSync {
4995 tip: tip.to_owned(),
4996 behind,
4997 attempts,
4998 conflict: None,
4999 already_in: Some(evidence),
5000 });
5001 self.state.event("land", message);
5002 self.state.save()?;
5003 self.settle_questions();
5004 Ok(true)
5005 }
5006
5007 fn landing_base(&self) -> String {
5017 self.state
5018 .base_sync
5019 .as_ref()
5020 .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
5021 }
5022
5023 pub async fn fix_selected(
5056 &mut self,
5057 ids: &[String],
5058 reason: &str,
5059 allow_stale: bool,
5060 ) -> Result<()> {
5061 let reason = reason.trim();
5062 if reason.is_empty() {
5063 bail!("a fix request needs a reason — that is the operator's own record of why");
5064 }
5065 if ids.is_empty() {
5066 bail!("no finding id given");
5067 }
5068 if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
5069 bail!(
5070 "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
5071 has already concluded — can be given a targeted fix. A run still \
5072 in progress should simply be resumed; a `merged` run's branch has \
5073 already landed, so its answer is a fresh `magi review <branch>`, \
5074 not reopening this run's own record",
5075 self.state.id,
5076 self.state.status.as_str()
5077 );
5078 }
5079 let Some(winner) = self.state.winner().cloned() else {
5080 bail!("run {} has no winning candidate to fix", self.state.id);
5081 };
5082 if !git::branch_exists(&self.state.repo, &winner.branch).await? {
5083 bail!(
5084 "branch `{}` no longer exists; this run cannot be extended",
5085 winner.branch
5086 );
5087 }
5088 let home = crate::run::home();
5089 if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
5090 bail!(
5091 "run {} is currently being worked on by another magi process",
5092 self.state.id
5093 );
5094 }
5095 let _claim = FixClaim::acquire(&self.state.dir())?;
5101
5102 let mut seen = BTreeSet::new();
5106 let mut findings = Vec::new();
5107 let mut missing = Vec::new();
5108 for id in ids {
5109 if !seen.insert(id.clone()) {
5110 continue;
5111 }
5112 match self.state.finding(id) {
5113 Some((round, rec, f)) => findings.push(OperatorFixFinding {
5114 id: f.id.clone(),
5115 severity: f.severity,
5116 reviewer_vote: rec.vote,
5117 round: round.round,
5118 round_head: round.head.clone(),
5119 reviewer: rec.reviewer,
5120 agent: rec.agent.clone(),
5121 file: f.file.clone(),
5122 line: f.line,
5123 title: f.title.clone(),
5124 detail: f.detail.clone(),
5125 outcome: OperatorFixOutcome::Pending,
5126 }),
5127 None => missing.push(id.clone()),
5128 }
5129 }
5130 if !missing.is_empty() {
5131 bail!(
5132 "unknown finding id(s): {}; nothing was changed",
5133 missing.join(", ")
5134 );
5135 }
5136
5137 let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
5138 let stale_details: Vec<(String, String)> = findings
5139 .iter()
5140 .filter(|f| f.round_head != head_at_request)
5141 .map(|f| (f.id.clone(), f.round_head.clone()))
5142 .collect();
5143 let stale = !stale_details.is_empty();
5144 if stale && !allow_stale {
5145 bail!(
5146 "the branch has moved since some finding(s) were raised — {} — now \
5147 at {}; pass --allow-stale to fix anyway, or re-run review first",
5148 stale_details
5149 .iter()
5150 .map(|(id, head)| format!("{id} (raised against {})", short(head)))
5151 .collect::<Vec<_>>()
5152 .join(", "),
5153 short(&head_at_request)
5154 );
5155 }
5156
5157 let request = OperatorFixRequest {
5158 requested_at: Timestamp::now(),
5159 reason: reason.to_owned(),
5160 findings,
5161 head_at_request: head_at_request.clone(),
5162 allow_stale,
5163 stale,
5164 fix: None,
5165 result_head: None,
5166 follow_up_review_run: None,
5167 };
5168 self.state.event(
5169 "fix",
5170 format!(
5171 "operator requested a targeted fix on {} finding(s) ({}): {reason}",
5172 request.findings.len(),
5173 request
5174 .findings
5175 .iter()
5176 .map(|f| f.id.as_str())
5177 .collect::<Vec<_>>()
5178 .join(", "),
5179 ),
5180 );
5181 self.state.operator_fixes.push(request);
5188 self.state.save()?;
5189 let request_index = self.state.operator_fixes.len() - 1;
5190
5191 let repo = self.state.repo.clone();
5200 let coordination = self.coordinate(&repo).await?;
5201 if winner.worktree.exists() {
5202 let dirty = git::git(
5205 &winner.worktree,
5206 &["status", "--porcelain", "--untracked-files=all"],
5207 )
5208 .await?;
5209 let only_withheld = dirty.lines().all(|l| {
5210 l.strip_prefix("?? ")
5211 .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
5212 });
5213 if !only_withheld {
5214 bail!(
5215 "`{}` has uncommitted changes; refusing to touch it — commit or \
5216 discard them first",
5217 winner.worktree.display()
5218 );
5219 }
5220 git::worktree_remove(&self.state.repo, &winner.worktree)
5221 .await
5222 .ok();
5223 }
5224 let fix_worktree = self.state.worktree_root().join("operator-fix");
5225 let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
5226 git::git(
5227 &self.state.repo,
5228 &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
5229 )
5230 .await
5231 .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
5232 if !git::is_clean(&fix_worktree).await? {
5233 git::worktree_remove(&self.state.repo, &fix_worktree)
5234 .await
5235 .ok();
5236 bail!(
5237 "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
5238 winner.branch
5239 );
5240 }
5241 let repo_for_setup = self.state.repo.clone();
5242 let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
5243 Ok(()) => {
5244 worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
5245 }
5246 Err(e) => Err(e),
5247 };
5248 if let Err(e) = setup {
5249 git::worktree_remove(&self.state.repo, &fix_worktree)
5250 .await
5251 .ok();
5252 return Err(e);
5253 }
5254
5255 let run_id = self.state.id.clone();
5256 let prompts = self.state.config.prompts.clone();
5257 let language = self.state.config.graph.language.clone();
5258 let sessions = self.state.config.graph.sessions;
5259 let artifacts = agent::artifacts_dir(&self.state.dir());
5260 let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
5261 .findings
5262 .iter()
5263 .map(|f| Finding {
5264 id: f.id.clone(),
5265 severity: f.severity,
5266 file: f.file.clone(),
5267 line: f.line,
5268 title: f.title.clone(),
5269 detail: f.detail.clone(),
5270 })
5271 .collect();
5272 let fix_prompt = prompt::operator_fix_in(
5273 &self.state.instruction,
5274 &finding_list,
5275 reason,
5276 &stale_details,
5277 &head_at_request,
5278 &language,
5279 &self.state.config.graph.github_language,
5280 );
5281 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5282 drop(coordination);
5283 let (job, seat, out) = self
5284 .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
5285 prompt: fix_prompt.clone(),
5286 spec,
5287 seat,
5288 cwd: fix_worktree.clone(),
5289 timeout,
5290 allow_write: true,
5291 sessions,
5292 artifacts: artifacts.clone(),
5293 stem: "operator-fix".to_owned(),
5294 handover: None,
5295 })
5296 .await;
5297 let agent_id = seat.agent.clone();
5298
5299 let mut fix = FixRecord {
5300 agent: agent_id,
5301 addressed: Vec::new(),
5302 rejected: Vec::new(),
5303 notes: String::new(),
5304 committed: false,
5305 failed: None,
5306 duration_ms: 0,
5307 continuation: None,
5308 };
5309 let mut final_seat = seat.clone();
5310 match out {
5311 AgentOutcome::Ok(o) => {
5312 fix.duration_ms = o.duration_ms;
5313 let parsed = verdict::extract_json::<FixReport>(&o.text);
5314 let incomplete_reason = match &parsed {
5315 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
5316 "the reply parsed, but it reported a command whose own CLI \
5317 never confirmed an exit status"
5318 .to_owned(),
5319 ),
5320 Ok(_) => None,
5321 Err(e) => Some(e.to_string()),
5322 };
5323 match incomplete_reason {
5324 None => {
5325 let report = parsed.expect("checked Ok above");
5326 fix.addressed = report.addressed;
5327 fix.rejected = report.rejected;
5328 fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5329 }
5330 Some(reason) => {
5331 let (resumed_seat, resolved, failure, cont) = self
5332 .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
5333 .await;
5334 fix.duration_ms += cont.cumulative_wait_ms;
5335 fix.continuation = Some(cont);
5336 final_seat = resumed_seat;
5337 match resolved {
5338 Some(report) => {
5339 fix.addressed = report.addressed;
5340 fix.rejected = report.rejected;
5341 fix.notes =
5342 blind::sanitize_prose(&report.notes, &self.state.config.blind);
5343 }
5344 None => fix.failed = failure,
5345 }
5346 }
5347 }
5348 }
5349 AgentOutcome::Dropped(o) => {
5350 fix.duration_ms = o.duration_ms;
5351 let why = o
5352 .dropped
5353 .as_ref()
5354 .map(|d| d.why.as_str())
5355 .unwrap_or("the CLI ended the stream without delivering its answer");
5356 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5357 }
5358 AgentOutcome::Quota(o) => {
5359 self.state.quota.push(QuotaLoss {
5360 seat: final_seat.key.clone(),
5361 node: "fix".to_owned(),
5362 at: Timestamp::now(),
5363 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5364 });
5365 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5366 }
5367 AgentOutcome::Failed(e) => fix.failed = Some(e),
5368 AgentOutcome::ResourceBlocked(e) => {
5369 fix.failed = Some(format!("the fixer was not started: {e}"));
5370 }
5371 }
5372 if fix.continuation.is_none() {
5373 fix.continuation = Some(ContinuationRecord::not_needed());
5374 }
5375 self.state.seats.insert(final_seat.key.clone(), final_seat);
5376
5377 let rescue_message = format!(
5378 "magi: operator-selected fix ({}) (uncommitted work)",
5379 self.state.operator_fixes[request_index]
5380 .findings
5381 .iter()
5382 .map(|f| f.id.as_str())
5383 .collect::<Vec<_>>()
5384 .join(", ")
5385 );
5386 let repo = self.state.repo.clone();
5387 let coordination = self.coordinate(&repo).await?;
5388 if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
5389 self.state.note_withheld("fix", &r.withheld);
5390 }
5391 let after = git::rev_parse(&fix_worktree, "HEAD").await?;
5392 fix.committed = after != head_at_request;
5393 git::worktree_remove(&self.state.repo, &fix_worktree)
5394 .await
5395 .ok();
5396 drop(coordination);
5397
5398 self.state.event(
5399 "fix",
5400 match &fix.failed {
5401 Some(reason) => format!(
5402 "operator fix: adoption report was lost ({reason}); {}",
5403 if fix.committed {
5404 "committed"
5405 } else {
5406 "NO new commit"
5407 }
5408 ),
5409 None => format!(
5410 "operator fix: {} addressed, {} rejected, {}",
5411 fix.addressed.len(),
5412 fix.rejected.len(),
5413 if fix.committed {
5414 "committed"
5415 } else {
5416 "NO new commit"
5417 }
5418 ),
5419 },
5420 );
5421
5422 for f in &mut self.state.operator_fixes[request_index].findings {
5429 f.outcome = if fix.failed.is_some() {
5430 OperatorFixOutcome::Unreported
5431 } else if fix.addressed.contains(&f.id) {
5432 OperatorFixOutcome::Addressed
5433 } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
5434 OperatorFixOutcome::Rejected { why: r.why.clone() }
5435 } else {
5436 OperatorFixOutcome::Unreported
5437 };
5438 }
5439
5440 let committed = fix.committed;
5441 if committed {
5442 self.state.operator_fixes[request_index].result_head = Some(after.clone());
5443 }
5444 self.state.operator_fixes[request_index].fix = Some(fix);
5445 self.state.save()?;
5448
5449 if committed {
5450 self.state.event(
5451 "fix",
5452 format!(
5453 "operator fix committed {}; opening a follow-up review-only run",
5454 short(&after)
5455 ),
5456 );
5457 let origin =
5460 Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
5461 match Self::review(
5462 &self.state.repo,
5463 &winner.branch,
5464 self.state.config.clone(),
5465 origin,
5466 )
5467 .await
5468 {
5469 Ok(mut follow_up) => {
5470 follow_up.state.event(
5471 "start",
5472 format!(
5473 "requested by an operator fix on run {} for finding(s) {}",
5474 self.state.id,
5475 self.state.operator_fixes[request_index]
5476 .findings
5477 .iter()
5478 .map(|f| f.id.as_str())
5479 .collect::<Vec<_>>()
5480 .join(", "),
5481 ),
5482 );
5483 follow_up.state.save()?;
5484 let follow_up_id = follow_up.state.id.clone();
5485 let adopted = match crate::direct::adopt(
5488 &follow_up.state,
5489 self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
5490 ) {
5491 Ok(a) => a,
5492 Err(e) => {
5493 self.state.event(
5496 "fix",
5497 format!(
5498 "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
5499 ),
5500 );
5501 self.state.save()?;
5502 return Ok(());
5503 }
5504 };
5505 let executed = follow_up.execute().await;
5506 let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
5507 if let Some(a) = adopted {
5508 a.finish(&follow_up.state, executed);
5509 }
5510 if let Some(e) = failure {
5511 self.state.event(
5512 "fix",
5513 format!(
5514 "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
5515 ),
5516 );
5517 }
5518 self.state.operator_fixes[request_index].follow_up_review_run =
5519 Some(follow_up_id);
5520 }
5521 Err(e) => {
5522 self.state.event(
5523 "fix",
5524 format!("committed the fix but could not open a follow-up review: {e:#}"),
5525 );
5526 }
5527 }
5528 self.state.save()?;
5529 }
5530
5531 Ok(())
5532 }
5533
5534 async fn ask_fixer(
5554 &mut self,
5555 winner: &Candidate,
5556 node: &'static str,
5557 round: Option<usize>,
5558 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5559 ) -> (SeatJob, SeatState, AgentOutcome) {
5560 let attempts = fixer::attempts(&self.state, &self.roles, winner);
5561 self.ask_fixer_among(attempts, node, round, None, build)
5562 .await
5563 }
5564
5565 async fn ask_fixer_among(
5570 &mut self,
5571 attempts: Vec<(AgentSpec, String)>,
5572 node: &'static str,
5573 round: Option<usize>,
5574 restore: Option<(&Path, &str)>,
5575 build: impl Fn(AgentSpec, SeatState) -> SeatJob,
5576 ) -> (SeatJob, SeatState, AgentOutcome) {
5577 let run_id = self.state.id.clone();
5578 let prompts = self.state.config.prompts.clone();
5579 let cache = self.state.config.cache_dir();
5580 let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
5581 let mut last = None;
5582 for (i, (spec, key)) in attempts.into_iter().enumerate() {
5583 let seat = self.seat(&key, &spec.id);
5584 let mut job = build(spec.clone(), seat);
5585 if i > 0 {
5586 job.stem = format!("{}-{}", job.stem, spec.id);
5587 }
5588 let ctx = WaveCtx {
5589 carry_seats: false,
5590 pauses: Some((&self.pause, &self.interrupt)),
5591 run: &run_id,
5592 node,
5593 prompts: &prompts,
5594 cache: cache.as_deref(),
5595 round,
5596 };
5597 let (seat, out) =
5598 run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5599 let advances = match &out {
5602 AgentOutcome::Ok(o) => agent::output_advances(o),
5603 AgentOutcome::ResourceBlocked(_) => false,
5604 _ => true,
5605 };
5606 if let Some(next) = ids.get(i + 1)
5607 && advances
5608 {
5609 self.state.seats.insert(seat.key.clone(), seat.clone());
5610 let class =
5611 FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
5612 let reason = match &out {
5613 AgentOutcome::Ok(_) => "nothing usable".to_owned(),
5614 other => fail_reason(other),
5615 };
5616 if let Some((wt, rev)) = restore
5620 && let Err(e) = self.graft_restore(wt, rev).await
5621 {
5622 let why =
5623 format!("could not restore the worktree before the next fixer: {e:#}");
5624 tracing::warn!("{why}");
5625 let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
5626 AgentOutcome::ResourceBlocked(why)
5627 } else {
5628 AgentOutcome::Failed(why)
5629 };
5630 last = Some((job, seat, out));
5631 break;
5632 }
5633 record_handover(
5634 &mut self.state,
5635 node,
5636 &seat.key,
5637 &spec.id,
5638 next,
5639 &class,
5640 &reason,
5641 );
5642 continue;
5643 }
5644 last = Some((job, seat, out));
5645 break;
5646 }
5647 last.expect("the fixer chain always has an entry")
5648 }
5649
5650 async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
5661 use crate::recovery::{Mode, Repair};
5662 if !self.state.reviews.is_empty() {
5663 return Ok(());
5664 }
5665 let Some(mut rec) = self.state.recovery.clone() else {
5666 return Ok(());
5667 };
5668 if rec.mode != Mode::VerifyRepair {
5669 return Ok(());
5670 }
5671 let wt = winner.worktree.clone();
5672 let mut head = git::rev_parse(&wt, "HEAD").await?;
5673 match rec.repair.clone() {
5674 Repair::Done { .. } => return Ok(()),
5675 Repair::WaitingForRestore { before } => {
5676 self.graft_restore(&wt, &before).await?;
5677 head = before;
5678 rec.repair = Repair::NotStarted;
5679 self.state.recovery = Some(rec.clone());
5680 self.state.save()?;
5681 }
5682 Repair::Started { before } => {
5683 if let Err(e) = self.graft_restore(&wt, &before).await {
5687 if e.is::<crate::repo_lock::ResourceBlocked>() {
5688 return Err(e);
5689 }
5690 return self
5691 .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
5692 .await;
5693 }
5694 rec.repair = Repair::Done {
5695 before: before.clone(),
5696 after: before,
5697 committed: false,
5698 failed: Some("interrupted; rolled back".to_owned()),
5699 };
5700 self.state
5701 .event("recovery", "the repair was interrupted and rolled back");
5702 self.state.recovery = Some(rec);
5703 self.state.save()?;
5704 return Ok(());
5705 }
5706 Repair::NotStarted => {}
5707 }
5708 if crate::recovery::recheck(&mut rec, &head) {
5709 self.state.event("recovery", rec.summary());
5710 self.state.recovery = Some(rec);
5711 self.state.save()?;
5712 return Ok(());
5713 }
5714 rec.repair = Repair::Started {
5715 before: head.clone(),
5716 };
5717 self.state.recovery = Some(rec.clone());
5718 self.state.save()?;
5719
5720 let language = self.state.config.graph.language.clone();
5721 let sessions = self.state.config.graph.sessions;
5722 let artifacts = agent::artifacts_dir(&self.state.dir());
5723 let fix_prompt = prompt::recovery_fix(
5724 &self.state.instruction,
5725 &rec,
5726 self.state.config.graph.review_rounds,
5727 &language,
5728 );
5729 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5730 self.state.event(
5731 "recovery",
5732 format!(
5733 "repair: the fixer starts from the failed verification on {}",
5734 short(&head)
5735 ),
5736 );
5737 let (_job, seat, out) = self
5738 .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
5739 prompt: fix_prompt.clone(),
5740 spec,
5741 seat,
5742 cwd: wt.clone(),
5743 timeout,
5744 allow_write: true,
5745 sessions,
5746 artifacts: artifacts.clone(),
5747 stem: "fix-recovery".to_owned(),
5748 handover: None,
5749 })
5750 .await;
5751 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5752 rec.repair = Repair::WaitingForRestore { before: head };
5753 self.state.recovery = Some(rec);
5754 return self.stop_for_resource();
5755 }
5756 let failed = match &out {
5757 AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
5758 AgentOutcome::Quota(o) => {
5759 self.state.quota.push(QuotaLoss {
5760 seat: seat.key.clone(),
5761 node: "fix".to_owned(),
5762 at: Timestamp::now(),
5763 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5764 });
5765 Some("rate limited (quota); the fixer could not run".to_owned())
5766 }
5767 AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
5768 AgentOutcome::Failed(e) => Some(e.clone()),
5769 };
5770 self.state.seats.insert(seat.key.clone(), seat);
5771 let coordination = self.coordinate(&wt).await?;
5772 let failed = match failed {
5773 Some(why) => {
5775 if let Err(e) = git::restore_to(&wt, &head).await {
5776 return self
5777 .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
5778 .await;
5779 }
5780 Some(why)
5781 }
5782 None => {
5783 if let Ok(r) = git::rescue_commit(
5784 &wt,
5785 "magi: recovery repair of the failed verification (uncommitted work)",
5786 )
5787 .await
5788 {
5789 self.state.note_withheld("fix", &r.withheld);
5790 }
5791 None
5792 }
5793 };
5794 let after = git::rev_parse(&wt, "HEAD").await?;
5795 drop(coordination);
5796 self.state.event(
5797 "recovery",
5798 match &failed {
5799 Some(why) => format!("repair: nothing kept ({why})"),
5800 None if after != head => {
5801 format!("repair: committed {}", short(&after))
5802 }
5803 None => "repair: the fixer made no commit".to_owned(),
5804 },
5805 );
5806 rec.repair = Repair::Done {
5807 before: head.clone(),
5808 committed: after != head,
5809 after,
5810 failed,
5811 };
5812 self.state.recovery = Some(rec);
5813 self.state.save()?;
5814 Ok(())
5815 }
5816
5817 async fn stop_recovery(
5820 &mut self,
5821 mut rec: crate::recovery::RecoveryRecord,
5822 head: &str,
5823 why: String,
5824 ) -> Result<()> {
5825 rec.repair = crate::recovery::Repair::Started {
5827 before: head.to_owned(),
5828 };
5829 self.state.recovery = Some(rec);
5830 self.state
5831 .event("recovery", format!("repair could not be undone: {why}"));
5832 self.state.status = RunStatus::Blocked;
5833 self.state.save()?;
5834 bail!("recovery repair could not be rolled back: {why}")
5835 }
5836
5837 async fn review_loop(&mut self) -> Result<()> {
5838 if self
5843 .state
5844 .base_sync
5845 .as_ref()
5846 .is_some_and(|s| s.conflict.is_some())
5847 {
5848 return Ok(());
5849 }
5850 let run_id = self.state.id.clone();
5855 let prompts = self.state.config.prompts.clone();
5856 let Some(winner) = self.state.winner().cloned() else {
5857 return Ok(());
5858 };
5859 let max_rounds = self.state.config.graph.review_rounds;
5860 if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
5870 if status == RunStatus::Gating {
5873 self.record_contested_handoff();
5874 }
5875 self.state.status = status;
5876 self.state.save()?;
5877 return Ok(());
5878 }
5879 self.state.status = RunStatus::Reviewing;
5880 if self
5890 .state
5891 .reviews
5892 .last()
5893 .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
5894 {
5895 let shell = self.state.config.shell();
5896 return self
5897 .stop_reviewing(
5898 "the last round's own verification never resolved",
5899 &shell,
5900 &winner.worktree,
5901 )
5902 .await;
5903 }
5904
5905 self.recovery_repair(&winner).await?;
5906 if self.resource_halt()? {
5907 return Ok(());
5908 }
5909
5910 let repo = self.state.repo.clone();
5911 let root = self.state.worktree_root();
5912 let language = self.state.config.graph.language.clone();
5913 let sessions = self.state.config.graph.sessions;
5914 let artifacts = self.merge_repair_artifacts();
5915 let base = self.landing_base();
5916 let base_short = short(&base);
5917 let reviewers = self.roles.reviewers.clone();
5918 let shell = self.state.config.shell();
5919
5920 for round in (self.state.reviews.len() + 1)..=max_rounds {
5921 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5922 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5923 let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
5924 let prev_verification = self
5933 .state
5934 .reviews
5935 .last()
5936 .and_then(|r| r.verification_summary(&head));
5937 let prior_section = match (&self.state.recovery, round) {
5940 (Some(rec), 1) if self.state.reviews.is_empty() => {
5941 let since = match (rec.mode, rec.reviewed_head.as_deref()) {
5942 (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
5943 Some(crate::recovery::head_of(
5944 &git::diff(&winner.worktree, reviewed, "HEAD")
5945 .await
5946 .unwrap_or_default(),
5947 crate::recovery::SINCE_DIFF_MAX,
5948 ))
5949 }
5950 _ => None,
5951 };
5952 Some(prompt::recovery_review(rec, since.as_deref()))
5953 }
5954 _ => None,
5955 };
5956
5957 let mut jobs = Vec::new();
5961 let coordination = self.coordinate(&repo).await?;
5962 for (r, spec) in reviewers.iter().cloned().enumerate() {
5963 let wt = root.join(format!("review-{}", r + 1));
5964 if wt.exists() {
5967 git::reset_detached(&wt, &head).await?;
5968 } else {
5969 git::worktree_add_detached(&repo, &wt, &head).await?;
5970 }
5971 self.setup_seat_worktree(&repo, &wt, false).await?;
5972 let seat_key = format!("review-{}", r + 1);
5973 let spec = pick_start_spec(
5976 &self.roles.reviewer_roster,
5977 spec,
5978 self.state.seat_history.get(&seat_key),
5979 );
5980 let seat = self.seat(&seat_key, &spec.id);
5981 jobs.push(SeatJob {
5982 prompt: prompt::review(&prompt::ReviewCtx {
5983 instruction: &self.state.instruction,
5984 branch: &winner.branch,
5985 base_short: &base_short,
5986 stat: &stat,
5987 patch: &patch,
5988 verification: prev_verification.as_ref(),
5989 prior: prior_section.as_deref(),
5990 reviewers: reviewers.len(),
5991 round,
5992 rounds: max_rounds,
5993 competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
5996 lens: Lens::for_seat(r),
5997 language: &language,
5998 github_language: &self.state.config.graph.github_language,
5999 }),
6000 spec,
6001 seat,
6002 cwd: wt,
6003 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6004 allow_write: false,
6005 sessions,
6006 artifacts: artifacts.clone(),
6007 stem: format!("review-{round}-{}", r + 1),
6008 handover: None,
6009 });
6010 }
6011
6012 drop(coordination);
6013 self.state.event(
6014 "review",
6015 format!(
6016 "round {round}: {} reviewers on {}",
6017 jobs.len(),
6018 short(&head)
6019 ),
6020 );
6021 let mut quota_losses = Vec::new();
6022 let review_retries = self.state.config.graph.retries;
6023 let review_cache = self.state.config.cache_dir();
6024 let ctx = WaveCtx {
6025 carry_seats: true,
6026 pauses: Some((&self.pause, &self.interrupt)),
6027 run: &run_id,
6028 node: "review",
6029 prompts: &prompts,
6030 cache: review_cache.as_deref(),
6031 round: Some(round),
6032 };
6033 let results = ask_json_wave::<Review>(
6034 jobs,
6035 Arc::clone(&self.sem),
6036 review_retries,
6037 &self.roles.reviewer_roster,
6038 &ctx,
6039 &mut quota_losses,
6040 &mut self.state,
6041 &|_: &Review| Ok(()),
6042 )
6043 .await;
6044 if self.state.resource_block.is_some() {
6045 return self.stop_for_resource();
6046 }
6047 let round_quota_missing = quota_losses.len();
6051 self.state.quota.extend(quota_losses);
6052
6053 let mut records = Vec::new();
6054 let mut all_findings = Vec::new();
6055 for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
6056 let agent_id = seat.agent.clone();
6057 self.state.seats.insert(seat.key.clone(), seat);
6058 let mut record = ReviewRecord {
6059 reviewer: r + 1,
6060 agent: agent_id,
6061 summary: String::new(),
6062 findings: Vec::new(),
6063 vote: None,
6064 failed: None,
6065 duration_ms: 0,
6066 attempts,
6072 };
6073 match res {
6074 Ok((review, out)) => {
6075 record.summary =
6083 blind::sanitize_prose(&review.summary, &self.state.config.blind);
6084 record.vote = Some(review.vote);
6085 record.duration_ms = out.duration_ms;
6086 for (n, mut f) in review.findings.into_iter().enumerate() {
6087 f.id = format!("R{round}-{}-{}", r + 1, n + 1);
6090 f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
6091 f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
6092 f.file = f
6098 .file
6099 .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
6100 all_findings.push(f.clone());
6101 record.findings.push(f);
6102 }
6103 self.state.event(
6104 "review",
6105 format!(
6106 "round {round}: reviewer {} voted {} with {} finding(s)",
6107 r + 1,
6108 review.vote.label(),
6109 record.findings.len()
6110 ),
6111 );
6112 }
6113 Err(e) => {
6114 record.failed = Some(e.to_string());
6115 self.state.event(
6116 "review",
6117 format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
6118 );
6119 }
6120 }
6121 records.push(record);
6122 }
6123
6124 let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
6131 let vote_split =
6132 initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
6133 let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
6134 if vote_split {
6135 self.state.event(
6136 "review",
6137 format!(
6138 "round {round}: votes split ({}) — one round of reconsideration",
6139 initial_votes
6140 .iter()
6141 .map(|v| v.label())
6142 .collect::<Vec<_>>()
6143 .join(", ")
6144 ),
6145 );
6146 let panel: Vec<ReviewSeatReport<'_>> = records
6149 .iter()
6150 .filter_map(|r| {
6151 r.vote.map(|vote| ReviewSeatReport {
6152 reviewer: r.reviewer,
6153 vote,
6154 summary: &r.summary,
6155 findings: &r.findings,
6156 })
6157 })
6158 .collect();
6159
6160 let mut jobs = Vec::new();
6161 let mut seats_at = Vec::new();
6162 for (r, spec) in reviewers.iter().cloned().enumerate() {
6163 if records[r].vote.is_none() {
6167 continue;
6168 }
6169 let wt = root.join(format!("review-{}", r + 1));
6170 let seat_key = format!("review-{}", r + 1);
6171 let spec = self.occupant(&seat_key, spec);
6172 let seat = self.seat(&seat_key, &spec.id);
6173 let build = |with_patch: bool| {
6181 prompt::review_reconsider(&ReviewReconsiderCtx {
6182 instruction: &self.state.instruction,
6183 reviewer: r + 1,
6184 lens: Lens::for_seat(r),
6185 panel: &panel,
6186 patch: with_patch.then_some(ReviewPatch {
6187 branch: &winner.branch,
6188 base_short: &base_short,
6189 stat: &stat,
6190 patch: &patch,
6191 }),
6192 round,
6193 rounds: max_rounds,
6194 language: &language,
6195 })
6196 };
6197 let full = build(true);
6198 let prompt = if has_context(&spec, &seat, sessions) {
6199 build(false)
6200 } else {
6201 full.clone()
6202 };
6203 jobs.push(SeatJob {
6204 prompt,
6205 spec,
6206 seat,
6207 cwd: wt,
6208 timeout: Duration::from_secs(self.state.config.graph.timeout_review),
6209 allow_write: false,
6210 sessions,
6211 artifacts: artifacts.clone(),
6212 stem: format!("review-{round}-reconsider-{}", r + 1),
6213 handover: Some(full),
6214 });
6215 seats_at.push(r);
6216 }
6217
6218 let mut recon_quota_losses = Vec::new();
6219 let recon_cache = self.state.config.cache_dir();
6220 let recon_ctx = WaveCtx {
6221 carry_seats: true,
6222 pauses: Some((&self.pause, &self.interrupt)),
6223 run: &run_id,
6224 node: "review",
6225 prompts: &prompts,
6226 cache: recon_cache.as_deref(),
6227 round: Some(round),
6228 };
6229 let recon_results = ask_json_wave::<ReviewRevote>(
6230 jobs,
6231 Arc::clone(&self.sem),
6232 review_retries,
6233 &self.roles.reviewer_roster,
6234 &recon_ctx,
6235 &mut recon_quota_losses,
6236 &mut self.state,
6237 &|_: &ReviewRevote| Ok(()),
6238 )
6239 .await;
6240 if self.state.resource_block.is_some() {
6241 return self.stop_for_resource();
6242 }
6243 self.state.quota.extend(recon_quota_losses);
6244
6245 for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
6246 let agent_id = seat.agent.clone();
6247 self.state.seats.insert(seat.key.clone(), seat);
6248 let mut rec = ReviewRevoteRecord {
6249 reviewer: r + 1,
6250 agent: agent_id,
6251 vote: None,
6252 reason: String::new(),
6253 failed: None,
6254 };
6255 match res {
6256 Ok((rv, _)) => {
6257 rec.vote = Some(rv.vote);
6258 rec.reason =
6259 blind::sanitize_prose(&rv.reason, &self.state.config.blind);
6260 self.state.event(
6261 "review",
6262 format!(
6263 "round {round}: reviewer {} revoted {}",
6264 r + 1,
6265 rv.vote.label()
6266 ),
6267 );
6268 }
6269 Err(e) => {
6270 rec.failed = Some(e.to_string());
6271 self.state.event(
6272 "review",
6273 format!("round {round}: reviewer {} did not revote: {e}", r + 1),
6274 );
6275 }
6276 }
6277 reconsideration.push(rec);
6278 }
6279 } else if initial_votes.len() > 1 {
6280 self.state.event(
6281 "review",
6282 format!(
6283 "round {round}: votes agreed ({}) — no reconsideration",
6284 initial_votes[0].label()
6285 ),
6286 );
6287 }
6288
6289 let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
6290 let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6291 let defer_e2e =
6302 blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
6303 let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
6304 let reason =
6305 format!("{blocking} blocking finding(s) already required a fix this round");
6306 self.state.event(
6307 "verify",
6308 format!(
6309 "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
6310 {}); it will run once a round has none left",
6311 short(&head)
6312 ),
6313 );
6314 (Vec::new(), false, true, Some(reason))
6315 } else {
6316 let e2e_commands = self.state.config.verify.e2e.clone();
6317 let cache_dir = self.state.config.cache_dir();
6318 let context = format!("round {round}");
6319 let wait = self.repository_wait();
6320 let (e2e, verify_retried) = with_cache_lease(
6321 &mut self.state,
6322 cache_dir.as_deref(),
6323 "e2e",
6324 "e2e",
6325 &winner.worktree,
6326 &head,
6327 verify_timeout,
6328 &context,
6329 &wait,
6330 |state, budget| {
6331 let shell = shell.clone();
6332 let e2e_commands = e2e_commands.clone();
6333 let worktree = winner.worktree.clone();
6334 let context = context.clone();
6335 async move {
6336 run_e2e_with_retry(
6337 state,
6338 &shell,
6339 &e2e_commands,
6340 &worktree,
6341 budget,
6342 &context,
6343 )
6344 .await
6345 }
6346 },
6347 )
6348 .await?;
6349 (e2e, verify_retried, false, None)
6350 };
6351
6352 let expected = records.len();
6353 let answered = records.iter().filter(|r| r.failed.is_none()).count();
6354 let incomplete = answered < expected;
6355 let e2e_ok = e2e.iter().all(CommandOutcome::ok);
6356 let policy = self.state.config.graph.incomplete_review;
6357 let clean = round_is_clean(
6358 blocking,
6359 e2e_ok,
6360 answered,
6361 expected,
6362 round_quota_missing,
6363 policy,
6364 );
6365
6366 let mut round_record = ReviewRound {
6367 round,
6368 head: head.clone(),
6369 verified_head: None,
6370 verified_at: None,
6371 reviews: records,
6372 e2e,
6373 verify_retried,
6374 e2e_deferred,
6375 e2e_defer_reason,
6376 fix: None,
6377 blocking,
6378 answered,
6379 expected,
6380 clean,
6381 progressed: false,
6382 vote_split,
6383 reconsideration,
6384 verdict: None,
6385 };
6386 round_record.verdict = ReviewVote::worst(
6390 round_record
6391 .final_votes()
6392 .into_iter()
6393 .map(|(_, _, vote)| vote),
6394 );
6395 if !matches!(
6406 round_record.e2e_status(),
6407 E2eStatus::Deferred | E2eStatus::NotConfigured
6408 ) {
6409 round_record.verified_head = Some(head.clone());
6410 round_record.verified_at = Some(Timestamp::now());
6411 }
6412 let this_round_verification = round_record.verification_summary(&head);
6413
6414 if incomplete {
6415 let missing: Vec<String> = round_record
6416 .reviews
6417 .iter()
6418 .filter(|r| r.failed.is_some())
6419 .map(|r| format!("review-{}", r.reviewer))
6420 .collect();
6421 self.state.event(
6422 "review",
6423 format!(
6424 "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
6425 missing.join(", ")
6426 ),
6427 );
6428 }
6429
6430 if clean {
6431 self.state.event(
6432 "review",
6433 if incomplete && policy == IncompleteReviewPolicy::Warn {
6434 format!(
6435 "round {round}: clean (warn policy, incomplete panel) — no \
6436 blocking findings from the seats that answered, verification green"
6437 )
6438 } else if incomplete {
6439 format!(
6440 "round {round}: clean ({} rate-limited reviewer(s) excluded from \
6441 quorum) — no blocking findings from the seats that answered, \
6442 verification green",
6443 expected - answered
6444 )
6445 } else {
6446 format!("round {round}: clean — no blocking findings, verification green")
6447 },
6448 );
6449 self.state.reviews.push(round_record);
6450 self.state.status = RunStatus::Gating;
6451 self.state.save()?;
6452 return Ok(());
6453 }
6454
6455 if incomplete && blocking == 0 && e2e_ok {
6463 self.state.reviews.push(round_record);
6464 self.state.save()?;
6465 if round == max_rounds {
6466 self.state.status = RunStatus::Blocked;
6467 self.state.event(
6468 "review",
6469 format!(
6470 "{} reviewer seat(s) never answered after {max_rounds} rounds; \
6471 refusing to call it clean",
6472 expected - answered
6473 ),
6474 );
6475 return Ok(());
6476 }
6477 continue;
6478 }
6479
6480 if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
6492 self.state.reviews.push(round_record);
6493 return self
6494 .stop_reviewing(
6495 "the round's own verification could not run",
6496 &shell,
6497 &winner.worktree,
6498 )
6499 .await;
6500 }
6501
6502 if round == max_rounds {
6503 self.state.reviews.push(round_record);
6504 return self
6505 .stop_reviewing(
6506 &format!(
6507 "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
6508 ),
6509 &shell,
6510 &winner.worktree,
6511 )
6512 .await;
6513 }
6514
6515 let blocking_findings: Vec<_> = all_findings
6518 .iter()
6519 .filter(|f| f.severity.blocks())
6520 .cloned()
6521 .collect();
6522 let fix_prompt = prompt::fix_in(
6523 &self.state.instruction,
6524 &blocking_findings,
6525 this_round_verification.as_ref(),
6526 round,
6527 max_rounds,
6528 &language,
6529 &self.state.config.graph.github_language,
6530 );
6531 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6532 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
6533 let (job, seat, out) = self
6534 .ask_fixer(&winner, "fix", Some(round), |spec, seat| SeatJob {
6535 prompt: fix_prompt.clone(),
6536 spec,
6537 seat,
6538 cwd: winner.worktree.clone(),
6539 timeout,
6540 allow_write: true,
6541 sessions,
6542 artifacts: artifacts.clone(),
6543 stem: format!("fix-{round}"),
6544 handover: None,
6545 })
6546 .await;
6547 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6550 return self.stop_for_resource();
6551 }
6552 let agent_id = seat.agent.clone();
6553
6554 let mut fix = FixRecord {
6555 agent: agent_id,
6556 addressed: Vec::new(),
6557 rejected: Vec::new(),
6558 notes: String::new(),
6559 committed: false,
6560 failed: None,
6561 duration_ms: 0,
6562 continuation: None,
6563 };
6564 let mut continuation = ContinuationRecord::not_needed();
6565 let mut final_seat = seat.clone();
6566 match out {
6567 AgentOutcome::Ok(o) => {
6568 fix.duration_ms = o.duration_ms;
6569 let parsed = verdict::extract_json::<FixReport>(&o.text);
6570 let incomplete_reason = match &parsed {
6577 Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6578 "the reply parsed, but it reported a command whose own CLI never \
6579 confirmed an exit status"
6580 .to_owned(),
6581 ),
6582 Ok(_) => None,
6583 Err(e) => Some(e.to_string()),
6584 };
6585 match incomplete_reason {
6586 None => {
6587 let report = parsed.expect("checked Ok above");
6588 fix.addressed = report.addressed;
6589 fix.rejected = report.rejected;
6590 fix.notes =
6591 blind::sanitize_prose(&report.notes, &self.state.config.blind);
6592 }
6593 Some(reason) => {
6594 let (resumed_seat, resolved, failure, cont) = self
6595 .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
6596 .await;
6597 fix.duration_ms += cont.cumulative_wait_ms;
6598 continuation = cont;
6599 final_seat = resumed_seat;
6600 match resolved {
6601 Some(report) => {
6602 fix.addressed = report.addressed;
6603 fix.rejected = report.rejected;
6604 fix.notes = blind::sanitize_prose(
6605 &report.notes,
6606 &self.state.config.blind,
6607 );
6608 }
6609 None => fix.failed = failure,
6610 }
6611 }
6612 }
6613 }
6614 AgentOutcome::Dropped(o) => {
6616 fix.duration_ms = o.duration_ms;
6617 let why = o
6618 .dropped
6619 .as_ref()
6620 .map(|d| d.why.as_str())
6621 .unwrap_or("the CLI ended the stream without delivering its answer");
6622 fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6623 }
6624 AgentOutcome::Quota(o) => {
6625 self.state.quota.push(QuotaLoss {
6626 seat: final_seat.key.clone(),
6627 node: "fix".to_owned(),
6628 at: Timestamp::now(),
6629 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6630 });
6631 fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6632 }
6633 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
6634 }
6635 fix.continuation = Some(continuation);
6636 self.state.seats.insert(final_seat.key.clone(), final_seat);
6637 let coordination = self.coordinate(&winner.worktree).await?;
6638 if let Ok(r) = git::rescue_commit(
6639 &winner.worktree,
6640 &format!("magi: review round {round} fixes (uncommitted work)"),
6641 )
6642 .await
6643 {
6644 self.state.note_withheld("fix", &r.withheld);
6645 }
6646 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
6647 fix.committed = after != before;
6648 let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
6656 drop(coordination);
6657 let progressed = diff_after != patch;
6658 let commit_note = if fix.committed {
6659 "committed"
6660 } else {
6661 "NO new commit"
6662 };
6663 let tree_note = if progressed {
6664 "changed vs base"
6665 } else {
6666 "unchanged vs base"
6667 };
6668 self.state.event(
6669 "fix",
6670 match &fix.failed {
6671 Some(reason) => {
6677 format!(
6678 "round {round}: fixer's adoption report was lost ({reason}); \
6679 {commit_note}, tree {tree_note}"
6680 )
6681 }
6682 None => format!(
6683 "round {round}: {} addressed, {} rejected, {commit_note}, tree \
6684 {tree_note}{}",
6685 fix.addressed.len(),
6686 fix.rejected.len(),
6687 if continuation.outcome == ContinuationOutcome::Resumed {
6688 format!(
6689 " (adoption report recovered after {} continuation(s))",
6690 continuation.attempts
6691 )
6692 } else {
6693 String::new()
6694 },
6695 ),
6696 },
6697 );
6698 round_record.fix = Some(fix);
6699 round_record.progressed = progressed;
6700 self.state.reviews.push(round_record);
6701 self.state.save()?;
6702
6703 if matches!(
6716 continuation.outcome,
6717 ContinuationOutcome::Exhausted
6718 | ContinuationOutcome::QuotaLost
6719 | ContinuationOutcome::NoSession
6720 ) {
6721 return self
6722 .stop_reviewing(
6723 "the fixer's adoption report never came back, even after resuming its \
6724 own seat; refusing to start another round against the same worktree \
6725 while that is unresolved",
6726 &shell,
6727 &winner.worktree,
6728 )
6729 .await;
6730 }
6731
6732 let streak = self
6733 .state
6734 .reviews
6735 .iter()
6736 .rev()
6737 .take_while(|r| !r.progressed)
6738 .count();
6739 if streak >= STAGNANT_LIMIT {
6740 return self
6741 .stop_reviewing(
6742 &format!(
6743 "the tree has not moved against base for {streak} round(s) in a row"
6744 ),
6745 &shell,
6746 &winner.worktree,
6747 )
6748 .await;
6749 }
6750 }
6751 Ok(())
6752 }
6753
6754 fn record_contested_handoff(&mut self) {
6788 if self.state.contested_handoff.is_some() {
6789 return;
6790 }
6791 let Some(contested) = self
6792 .state
6793 .reviews
6794 .last()
6795 .and_then(ReviewRound::contested_handoff)
6796 else {
6797 return;
6798 };
6799 self.state.event(
6800 "review",
6801 format!(
6802 "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
6803 wait for the owner's approval",
6804 contested.findings.len(),
6805 contested.rejecters.len()
6806 ),
6807 );
6808 self.state.contested_handoff = Some(contested);
6809 }
6810
6811 async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
6812 let round_idx = self.state.reviews.len() - 1;
6813 let needs_catchup_run = matches!(
6821 self.state.reviews[round_idx].e2e_status(),
6822 E2eStatus::Deferred | E2eStatus::ResourceBlocked
6823 );
6824 if needs_catchup_run {
6825 let round = self.state.reviews[round_idx].round;
6826 let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
6827 let commands = self.state.config.verify.e2e.clone();
6828 let attempted_head = git::rev_parse(worktree, "HEAD").await?;
6829 let cache_dir = self.state.config.cache_dir();
6830 let context = format!(
6831 "round {round}: verification unresolved, catching up before the final decision"
6832 );
6833 let wait = self.repository_wait();
6834 let (outcomes, verify_retried) = with_cache_lease(
6835 &mut self.state,
6836 cache_dir.as_deref(),
6837 "e2e",
6838 "e2e",
6839 worktree,
6840 &attempted_head,
6841 timeout,
6842 &context,
6843 &wait,
6844 |state, budget| {
6845 let shell = shell.to_vec();
6846 let commands = commands.clone();
6847 let context = context.clone();
6848 async move {
6849 run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
6850 .await
6851 }
6852 },
6853 )
6854 .await?;
6855 let last = &mut self.state.reviews[round_idx];
6856 last.e2e = outcomes;
6857 last.verify_retried = verify_retried;
6858 last.verified_head = Some(attempted_head);
6865 last.verified_at = Some(Timestamp::now());
6866 if verify_inconclusive(&last.e2e) {
6867 self.state.save()?;
6874 return Ok(());
6875 }
6876 last.e2e_deferred = false;
6877 }
6878 let last = &self.state.reviews[round_idx];
6879 let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
6880
6881 match last.e2e_status() {
6882 E2eStatus::Failed => {
6883 let red: Vec<String> = last
6884 .e2e
6885 .iter()
6886 .filter(|o| !o.ok())
6887 .map(|o| {
6888 format!(
6889 "`{}` -> {:?}\n{}",
6890 o.command,
6891 o.code,
6892 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
6893 )
6894 })
6895 .collect();
6896 self.state
6897 .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
6898 self.state.status = RunStatus::Blocked;
6899 }
6900 E2eStatus::ResourceBlocked => {
6905 self.state.event(
6906 "review",
6907 format!(
6908 "{why}; e2e could not run (shared build cache unavailable); not \
6909 deciding yet"
6910 ),
6911 );
6912 }
6913 E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
6914 self.state.event(
6915 "review",
6916 format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
6917 );
6918 self.record_contested_handoff();
6919 self.state.status = RunStatus::Gating;
6920 }
6921 }
6922 self.state.save()?;
6923 Ok(())
6924 }
6925
6926 async fn gate(&mut self) -> Result<()> {
6929 if self.state.status == RunStatus::Failed
6941 || self
6942 .state
6943 .base_sync
6944 .as_ref()
6945 .is_some_and(|s| s.conflict.is_some())
6946 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
6947 != Some(RunStatus::Gating)
6948 {
6949 return Ok(());
6950 }
6951 if self.state.gate_ran {
6952 if self.state.gate.iter().any(|outcome| !outcome.ok()) {
6963 self.state.status = RunStatus::Blocked;
6964 self.state.save()?;
6965 }
6966 return Ok(());
6967 }
6968 let Some(winner) = self.state.winner().cloned() else {
6969 return Ok(());
6970 };
6971 self.state.status = RunStatus::Gating;
6972 let mut outcomes = self.run_gate(&winner).await?;
6973 loop {
6974 if verify_inconclusive(&outcomes) {
6985 self.state.save()?;
6986 return Ok(());
6987 }
6988 if outcomes.iter().all(CommandOutcome::ok) {
6989 break;
6990 }
6991 match self.gate_fix_round(&winner, &outcomes).await? {
6992 GateFix::Retry => outcomes = self.run_gate(&winner).await?,
6993 GateFix::Stop => break,
6994 GateFix::Defer => {
6995 self.state.save()?;
6996 return Ok(());
6997 }
6998 }
6999 }
7000 let passed = outcomes.iter().all(CommandOutcome::ok);
7001 self.state.gate = outcomes;
7002 self.state.gate_ran = true;
7003 self.state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7004 if !passed {
7005 self.state.status = RunStatus::Blocked;
7006 let spent = self.state.gate_fixes.len();
7007 self.state.event(
7008 "gate",
7009 if spent == 0 {
7010 "gate failed; not merging".to_owned()
7011 } else {
7012 format!("gate failed after {spent} gate-fix round(s); not merging")
7013 },
7014 );
7015 }
7016 self.state.save()?;
7017 Ok(())
7018 }
7019
7020 async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
7030 let commands = state.config.verify.pre_gate.clone();
7031 if commands.is_empty() {
7032 return;
7033 }
7034 let shell = state.config.shell();
7035 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
7036 let (outcomes, _) = run_commands(
7037 state,
7038 "pre_gate",
7039 "pre_gate",
7040 0,
7041 &shell,
7042 &commands,
7043 &winner.worktree,
7044 timeout,
7045 )
7046 .await;
7047 for o in &outcomes {
7048 if !o.ok() {
7049 tracing::warn!(
7050 "pre_gate `{}` failed ({:?}); the gate decides",
7051 o.command,
7052 o.code
7053 );
7054 }
7055 state.event(
7056 "pre_gate",
7057 format!(
7058 "`{}` -> {}",
7059 o.command,
7060 if o.ok() {
7061 "pass".to_owned()
7062 } else {
7063 format!(
7064 "FAIL ({:?})\n{}",
7065 o.code,
7066 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
7067 )
7068 }
7069 ),
7070 );
7071 }
7072 state.pre_gate = outcomes;
7073 match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
7074 Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
7075 Ok(head) => {
7076 state.event("pre_gate", format!("committed mechanical fixes ({head})"));
7077 state.pre_gate_commit = Some(head);
7078 }
7079 Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
7080 },
7081 Ok(false) => {}
7082 Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
7083 }
7084 if let Err(e) = state.save() {
7085 tracing::warn!("could not persist the pre_gate record: {e:#}");
7086 }
7087 }
7088
7089 async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
7092 let wait = self.repository_wait();
7093 run_landing_gate(&mut self.state, winner, &wait).await
7094 }
7095
7096 async fn gate_fix_round(
7109 &mut self,
7110 winner: &Candidate,
7111 outcomes: &[CommandOutcome],
7112 ) -> Result<GateFix> {
7113 let cap = self.state.config.graph.gate_fix_rounds;
7114 let spent = self.state.gate_fixes.len();
7115 if spent >= cap {
7116 if cap > 0 {
7117 self.state.event(
7118 "gate",
7119 format!("{spent} gate-fix round(s) spent and the gate still fails"),
7120 );
7121 }
7122 return Ok(GateFix::Stop);
7123 }
7124 if !gate_fixable(outcomes) {
7125 self.state.event(
7126 "gate",
7127 "gate failure is not an ordinary non-zero exit with output (timeout, missing \
7128 command or similar); not spending a fix round on it",
7129 );
7130 return Ok(GateFix::Stop);
7131 }
7132 let min_free = self.state.config.disk.min_free_bytes;
7133 if min_free > 0 {
7134 match crate::disk::free_bytes(&winner.worktree) {
7135 Ok(free) if crate::disk::enough_space(free, min_free) => {}
7136 Ok(free) => {
7137 self.state.event(
7138 "gate",
7139 format!(
7140 "only {free} bytes free ({min_free} required by `[disk] \
7141 min_free_bytes`); not spending a fix round on a failure the disk \
7142 may explain"
7143 ),
7144 );
7145 return Ok(GateFix::Stop);
7146 }
7147 Err(e) => {
7148 self.state.event(
7149 "gate",
7150 format!("free disk space could not be measured ({e:#}); no fix round"),
7151 );
7152 return Ok(GateFix::Stop);
7153 }
7154 }
7155 }
7156
7157 let attempt = spent + 1;
7158 let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
7159 let base = self.landing_base();
7160 let fix_prompt = prompt::gate_fix_in(
7161 &self.state.instruction,
7162 &failed,
7163 attempt,
7164 cap,
7165 &self.state.config.graph.language,
7166 &self.state.config.graph.github_language,
7167 );
7168 let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7169 let sessions = self.state.config.graph.sessions;
7170 let artifacts = self.merge_repair_artifacts();
7171 self.state.event(
7172 "gate",
7173 format!("gate failed; gate-fix round {attempt} of {cap}"),
7174 );
7175 let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7176 let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7177 self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
7180 head: before.clone(),
7181 passed: false,
7182 outcomes: Vec::new(),
7183 });
7184 self.state.save()?;
7185 let (_, seat, out) = self
7186 .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
7187 prompt: fix_prompt.clone(),
7188 spec,
7189 seat,
7190 cwd: winner.worktree.clone(),
7191 timeout,
7192 allow_write: true,
7193 sessions,
7194 artifacts: artifacts.clone(),
7195 stem: format!("gate-fix-{attempt}"),
7196 handover: None,
7197 })
7198 .await;
7199 if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7202 return Ok(GateFix::Defer);
7203 }
7204 let mut record = GateFixRecord {
7205 agent: seat.agent.clone(),
7206 failed,
7207 notes: String::new(),
7208 committed: false,
7209 error: None,
7210 };
7211 match out {
7212 AgentOutcome::Ok(o) => {
7213 if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
7216 record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7217 }
7218 }
7219 AgentOutcome::Dropped(_) => {
7220 record.error = Some("the CLI dropped the stream".to_owned());
7221 }
7222 AgentOutcome::Quota(o) => {
7223 self.state.quota.push(QuotaLoss {
7224 seat: seat.key.clone(),
7225 node: "gate-fix".to_owned(),
7226 at: Timestamp::now(),
7227 reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
7228 });
7229 record.error = Some("rate limited (quota); fixer could not run".to_owned());
7230 }
7231 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
7232 }
7233 self.state.seats.insert(seat.key.clone(), seat);
7234 let coordination = self.coordinate(&winner.worktree).await?;
7235 if let Ok(r) = git::rescue_commit(
7236 &winner.worktree,
7237 &format!("magi: gate fix {attempt} (uncommitted work)"),
7238 )
7239 .await
7240 {
7241 self.state.note_withheld("gate-fix", &r.withheld);
7242 }
7243 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7244 record.committed = after != before;
7245 let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
7246 let note = record.error.clone();
7247 self.state.gate_fixes.push(record);
7248 self.state.save()?;
7249 if !changed {
7250 self.state.event(
7251 "gate",
7252 match note {
7253 Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
7254 None => format!("gate-fix round {attempt}: the tree did not change"),
7255 },
7256 );
7257 return Ok(GateFix::Stop);
7258 }
7259 self.state.event(
7260 "gate",
7261 format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
7262 );
7263
7264 drop(coordination);
7265 let wait = self.repository_wait();
7266 check_gate_repair_e2e(&mut self.state, winner, &wait).await
7267 }
7268
7269 async fn guarded_pr_message(
7276 &mut self,
7277 winner: &Candidate,
7278 facts: Option<&BranchFacts>,
7279 posting: bool,
7280 ) -> Result<Option<PrMessage>> {
7281 let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
7282 let head = match final_head.as_deref() {
7283 Some(h) => crate::pr_outcome::FinalHead::At(h),
7284 None => crate::pr_outcome::FinalHead::Unknown,
7285 };
7286 let summarized = if posting {
7288 Some(self.summarize_change(winner, final_head.as_deref()).await)
7289 } else {
7290 None
7291 };
7292 if self.state.resource_block.is_some() {
7297 self.state.event(
7298 "pr-summary",
7299 "a shared resource was busy; nothing was published, resume to summarize again",
7300 );
7301 return Ok(None);
7302 }
7303 if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
7304 anyhow::bail!(
7305 "the final commit moved during public description generation; retry before publishing"
7306 );
7307 }
7308 let note = match (&summarized, &self.state.change_summary) {
7309 (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
7310 (Some(Err(why)), _) => ChangeNote::Unavailable(why),
7311 (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
7312 (None, _) => ChangeNote::NotAsked,
7313 };
7314 let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
7315 if !posting {
7316 let identity = crate::scrub::Identity::current();
7317 return Ok(Some(PrMessage {
7318 title: crate::scrub::scrub(&pr.title, &identity),
7319 body: crate::scrub::scrub(&pr.body, &identity),
7320 }));
7321 }
7322 let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
7325 if let Some(known) = self
7326 .state
7327 .github_text
7328 .clone()
7329 .filter(|g| g.fingerprint == raw_fp)
7330 {
7331 return self.settle_withheld(winner, &pr, known).await;
7332 }
7333 let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
7334 let mut violations =
7335 crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
7336 let identity = crate::scrub::Identity::current();
7337 if (crate::scrub::scrub(&pr.title, &identity) != pr.title
7338 || crate::scrub::scrub(&pr.body, &identity) != pr.body)
7339 && !violations.contains(&crate::github_text::Violation::SensitiveData)
7340 {
7341 violations.push(crate::github_text::Violation::SensitiveData);
7342 }
7343 let has_language = violations
7347 .iter()
7348 .any(|v| *v != crate::github_text::Violation::SensitiveData);
7349 if self.state.config.graph.github_text_guard && !violations.is_empty() {
7350 self.state.event(
7351 "github-text",
7352 if has_language {
7353 format!("description rejected: {violations:?}; requesting one rewrite")
7354 } else {
7355 format!("description rejected: {violations:?}; asking the owner")
7356 },
7357 );
7358 let mut rewritten = false;
7359 let mut rejected_rewrite = None;
7360 if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
7361 {
7362 let output = text;
7363 if let Some(title) = summary_title(&output) {
7364 let raw_body = summary_without_title(&output);
7367 let clean_title = crate::scrub::scrub(&title, &identity);
7368 let mut body = crate::scrub::scrub(&raw_body, &identity);
7369 let redacted = clean_title != title || body != raw_body;
7370 let title = clean_title;
7371 let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
7372 if !complete {
7373 self.state.event(
7374 "github-text",
7375 "description rewrite omitted required findings",
7376 );
7377 }
7378 if !body.trim().is_empty() {
7379 body.push_str("\n\n");
7380 body.push_str(&public_pr_evidence(&self.state, head));
7381 }
7382 let marker = format!("magi:run/{}", self.state.id);
7383 if !body.trim().is_empty() && !body.contains(&marker) {
7384 body.push_str(&format!("\n\n{marker}"));
7385 }
7386 let rewrite = PrMessage {
7387 title: title.clone(),
7388 body: body.clone(),
7389 };
7390 let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
7391 let mut failed = crate::github_text::check_for(
7392 &self.state.config.graph,
7393 &title,
7394 &body,
7395 redecision,
7396 );
7397 if (!crate::github_text::shareable(&title)
7398 || !crate::github_text::shareable(&body))
7399 && !failed.contains(&crate::github_text::Violation::SensitiveData)
7400 {
7401 failed.push(crate::github_text::Violation::SensitiveData);
7402 }
7403 if complete
7404 && !title.trim().is_empty()
7405 && !body.trim().is_empty()
7406 && !failed.is_empty()
7407 {
7408 rejected_rewrite = Some(crate::run::GithubTextRewrite {
7409 title: title.clone(),
7410 body: body.clone(),
7411 failed: failed
7412 .iter()
7413 .map(|v| crate::github_text::category(*v).to_owned())
7414 .collect(),
7415 });
7416 }
7417 if complete
7418 && !title.trim().is_empty()
7419 && !raw_body.trim().is_empty()
7420 && failed.is_empty()
7421 {
7422 decision = redecision;
7423 pr = PrMessage { title, body };
7424 rewritten = true;
7425 self.state
7426 .event("github-text", "description rewrite passed");
7427 if redacted {
7428 self.state.event(
7429 "github-text",
7430 "sensitive data removed from description rewrite",
7431 );
7432 }
7433 }
7434 }
7435 }
7436 if !rewritten {
7437 self.state.event(
7438 "github-text",
7439 "description rewrite unavailable or rejected; asking the owner",
7440 );
7441 if let Some(w) =
7442 crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
7443 {
7444 self.retire_github_text_question("the withheld text changed");
7447 let record = crate::run::GithubTextAsk {
7448 rewrite: rejected_rewrite,
7449 fingerprint: raw_fp,
7450 title: w.title,
7451 body: w.body,
7452 categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
7453 question: None,
7454 asks: 0,
7455 resolved: false,
7456 chosen_title: None,
7457 choice: None,
7458 };
7459 return self.settle_withheld(winner, &pr, record).await;
7460 }
7461 }
7462 }
7463 let (title, body) =
7464 crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
7465 Ok(Some(PrMessage { title, body }))
7466 }
7467
7468 async fn summarize_change(
7479 &mut self,
7480 winner: &Candidate,
7481 head: Option<&str>,
7482 ) -> std::result::Result<(), String> {
7483 let Some(head) = head else {
7484 return Err("the final commit could not be read".to_owned());
7485 };
7486 let remote = self.state.config.merge.remote.clone();
7487 let tracking = format!("{remote}/{}", self.state.base_branch);
7488 if !git::rev_exists(&winner.worktree, &tracking).await {
7489 return Err(format!("the remote base `{tracking}` could not be read"));
7490 }
7491 let base = match git::merge_base(&winner.worktree, &tracking, head).await {
7492 Ok(b) if !b.is_empty() => b,
7493 _ => return Err("the merge base with the remote base could not be read".to_owned()),
7494 };
7495 let evidence = crate::pr_outcome::summary_evidence(
7496 &self.state,
7497 crate::pr_outcome::FinalHead::At(head),
7498 );
7499 let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
7500 if self
7501 .state
7502 .change_summary
7503 .as_ref()
7504 .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
7505 {
7506 return Ok(());
7507 }
7508 if self.state.change_summary.take().is_some() {
7509 self.state.event(
7510 "pr-summary",
7511 "the final diff or evidence changed; the earlier change summary was discarded",
7512 );
7513 }
7514 if let Err(e) = write_artifact(
7516 &self.state,
7517 &format!("public-description-{fingerprint}.md"),
7518 &format!("Final diff: {base}..{head}\n\n{evidence}"),
7519 ) {
7520 tracing::warn!("could not save public description evidence: {e:#}");
7521 }
7522 let chain = match agent::pick_chain(
7523 &self.state.config.agents,
7524 self.state.config.roles.synthesizer.as_ref(),
7525 &agent::installed,
7526 "synthesizer",
7527 ) {
7528 Ok(chain) => chain,
7529 Err(e) => {
7530 tracing::warn!("no agent for the change summary: {e:#}");
7531 self.state.event("pr-summary", "no summary agent available");
7532 return Err("no agent was available to summarize it".to_owned());
7533 }
7534 };
7535 let subjects = git::subjects(&winner.worktree, &base, head)
7536 .await
7537 .unwrap_or_default();
7538 let stat = git::diff_stat(&winner.worktree, &base, head)
7539 .await
7540 .unwrap_or_default();
7541 let stat: String = stat.chars().take(6_000).collect();
7542 let mut prompt = crate::prompt::change_summary_in(
7543 &base,
7544 head,
7545 &subjects,
7546 stat.trim(),
7547 &self.state.config.graph.github_language,
7548 );
7549 prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
7550 let sessions = self.state.config.graph.sessions;
7551 let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
7552 let prompts = self.state.config.prompts.clone();
7553 let cache = self.state.config.cache_dir();
7554 let run = self.state.id.clone();
7555 for (i, spec) in chain.into_iter().enumerate() {
7556 let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
7557 let github_spec = match agent::for_github_text(
7558 &spec,
7559 &self.state.config.graph.github_language,
7560 &winner.worktree,
7561 ) {
7562 Ok(spec) => spec,
7563 Err(e) => {
7564 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7565 continue;
7566 }
7567 };
7568 let job = SeatJob {
7569 spec: github_spec,
7570 seat,
7571 cwd: winner.worktree.clone(),
7572 prompt: prompt.clone(),
7573 timeout,
7574 allow_write: false,
7575 sessions,
7576 artifacts: agent::artifacts_dir(&self.state.dir()),
7577 stem: if i == 0 {
7578 "change-summary".to_owned()
7579 } else {
7580 format!("change-summary-{}", spec.id)
7581 },
7582 handover: None,
7583 };
7584 let ctx = WaveCtx {
7585 run: &run,
7586 node: "pr-summary",
7587 prompts: &prompts,
7588 cache: cache.as_deref(),
7589 round: None,
7590 carry_seats: false,
7591 pauses: Some((&self.pause, &self.interrupt)),
7592 };
7593 let (seat, outcome) =
7594 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7595 self.state.seats.insert(seat.key.clone(), seat);
7596 let text = match outcome {
7597 AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
7598 AgentOutcome::ResourceBlocked(why) => {
7599 if self.state.resource_block.is_none() {
7602 self.state.resource_block = Some(crate::run::ResourceBlock {
7603 node: "pr-summary".to_owned(),
7604 kind: "shared resource".to_owned(),
7605 detail: why,
7606 at: Timestamp::now(),
7607 });
7608 }
7609 return Err("a shared resource was busy".to_owned());
7610 }
7611 _ => continue,
7612 };
7613 let blind = &self.state.config.blind;
7614 let Some(title) = summary_title(&text) else {
7615 continue;
7616 };
7617 let body: String = summary_without_title(&text)
7618 .lines()
7619 .filter(|l| !l.contains("magi:run/"))
7620 .collect::<Vec<_>>()
7621 .join("\n");
7622 let body = blind::sanitize_prose(body.trim(), blind);
7623 let title = blind::sanitize_prose(&title, blind);
7624 if body.trim().is_empty() || title.trim().is_empty() {
7625 continue;
7626 }
7627 if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
7628 continue;
7629 }
7630 if git::rev_parse(&winner.worktree, "HEAD")
7632 .await
7633 .ok()
7634 .as_deref()
7635 != Some(head)
7636 {
7637 return Err("the branch moved while the summary was written".to_owned());
7638 }
7639 if git::merge_base(&winner.worktree, &tracking, head)
7640 .await
7641 .ok()
7642 .as_deref()
7643 != Some(base.as_str())
7644 || crate::pr_outcome::summary_evidence(
7645 &self.state,
7646 crate::pr_outcome::FinalHead::At(head),
7647 ) != evidence
7648 {
7649 return Err("the summary evidence changed while it was written".to_owned());
7650 }
7651 self.state.change_summary = Some(crate::run::ChangeSummary {
7652 head: head.to_owned(),
7653 base,
7654 evidence: fingerprint,
7655 title,
7656 body,
7657 });
7658 self.state.event(
7659 "pr-summary",
7660 format!("change summary written by {} for the final diff", spec.id),
7661 );
7662 if let Err(e) = self.state.save() {
7663 tracing::warn!("could not save the change summary: {e:#}");
7664 }
7665 return Ok(());
7666 }
7667 self.state
7668 .event("pr-summary", "no agent produced a change summary");
7669 Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
7670 }
7671
7672 async fn rewrite_pr_text(
7682 &mut self,
7683 winner: &Candidate,
7684 pr: &PrMessage,
7685 violations: &[crate::github_text::Violation],
7686 ) -> Option<String> {
7687 let prompt = format!(
7688 "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
7689 pr.title, pr.body
7690 );
7691 let sessions = self.state.config.graph.sessions;
7692 let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
7693 let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
7696 let key = format!("impl-{}", winner.label);
7697 if let Some(spec) = self
7698 .state
7699 .config
7700 .agents
7701 .iter()
7702 .find(|a| a.id == winner.agent)
7703 .cloned()
7704 && let Some(existing) = self.state.seats.get(&key)
7705 && existing.agent == spec.id
7706 && agent::has_session(spec.kind, existing, sessions)
7707 {
7708 attempts.push((spec, Some(existing.clone())));
7709 }
7710 match agent::pick_chain(
7711 &self.state.config.agents,
7712 self.state.config.roles.language_judge.as_ref(),
7713 &agent::installed,
7714 "language_judge",
7715 ) {
7716 Ok(chain) => {
7717 for spec in chain {
7718 if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
7719 continue;
7720 }
7721 attempts.push((spec, None));
7722 }
7723 }
7724 Err(e) if attempts.is_empty() => {
7725 tracing::warn!("no agent for the description rewrite: {e:#}");
7726 self.state
7727 .event("github-text", "no rewrite agent available");
7728 return None;
7729 }
7730 Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
7731 }
7732 let prompts = self.state.config.prompts.clone();
7733 let cache = self.state.config.cache_dir();
7734 let run = self.state.id.clone();
7735 for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
7736 let resumed = resumed_seat.is_some();
7737 let seat = resumed_seat.unwrap_or_else(|| {
7738 crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
7739 });
7740 let github_spec = match agent::for_github_text(
7741 &spec,
7742 &self.state.config.graph.github_language,
7743 &winner.worktree,
7744 ) {
7745 Ok(spec) => spec,
7746 Err(e) => {
7747 tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
7748 continue;
7749 }
7750 };
7751 let job = SeatJob {
7752 spec: github_spec,
7753 seat,
7754 cwd: winner.worktree.clone(),
7755 prompt: prompt.clone(),
7756 timeout: retry_budget(timeout_full, resumed),
7757 allow_write: false,
7758 sessions,
7759 artifacts: agent::artifacts_dir(&self.state.dir()),
7760 stem: if i == 0 {
7761 "github-text-rewrite".to_owned()
7762 } else {
7763 format!("github-text-rewrite-{}", spec.id)
7764 },
7765 handover: None,
7766 };
7767 let ctx = WaveCtx {
7768 run: &run,
7769 node: "github-text",
7770 prompts: &prompts,
7771 cache: cache.as_deref(),
7772 round: None,
7773 carry_seats: false,
7774 pauses: Some((&self.pause, &self.interrupt)),
7775 };
7776 let (seat, outcome) =
7777 run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
7778 self.state.seats.insert(seat.key.clone(), seat);
7779 match outcome {
7780 AgentOutcome::Ok(o)
7781 if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
7782 {
7783 return Some(o.text);
7784 }
7785 AgentOutcome::ResourceBlocked(_) => return None,
7786 _ => {}
7787 }
7788 }
7789 None
7790 }
7791
7792 fn retire_github_text_question(&mut self, why: &str) {
7794 let Some(id) = self
7795 .state
7796 .github_text
7797 .as_ref()
7798 .and_then(|g| g.question.clone())
7799 else {
7800 return;
7801 };
7802 let _ = ask::Questions::open().update(&id, |q| {
7803 q.abandon(why);
7804 Ok(())
7805 });
7806 }
7807
7808 async fn settle_withheld(
7812 &mut self,
7813 winner: &Candidate,
7814 pr: &PrMessage,
7815 mut g: crate::run::GithubTextAsk,
7816 ) -> Result<Option<PrMessage>> {
7817 use crate::github_text as gt;
7818 let store = ask::Questions::open();
7819 let timeout = self.state.config.graph.answer_timeout;
7820 if !g.resolved {
7821 let mut standing = g
7822 .question
7823 .as_ref()
7824 .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
7825 if let Some(q) = standing.as_ref()
7826 && q.status.open()
7827 && gt::expired(q, timeout)
7828 {
7829 let why = format!("no answer within {}s of asking", timeout.max(1));
7830 standing = store
7831 .update(&q.id, |q| {
7832 q.abandon(&why);
7833 Ok(())
7834 })
7835 .ok()
7836 .map(|(q, ())| q);
7837 }
7838 let mut ask_again: Option<Option<Vec<String>>> = None;
7839 match standing {
7840 None if g.asks <= 1 => {
7843 g.asks = 0;
7844 ask_again = Some(None);
7845 }
7846 None => {
7847 g.resolved = true;
7848 g.chosen_title = None;
7849 g.choice = Some(GithubTextChoice::Fallback);
7850 }
7851 Some(q) if q.status.open() => {
7852 self.park_on_github_text(&g)?;
7853 return Ok(None);
7854 }
7855 Some(q) => match gt::read_reply(&q) {
7856 gt::Reply::Fallback => {
7857 g.resolved = true;
7858 g.chosen_title = None;
7859 g.choice = Some(GithubTextChoice::Fallback);
7860 }
7861 gt::Reply::Original => {
7862 g.resolved = true;
7863 g.chosen_title = None;
7864 g.choice = Some(GithubTextChoice::Original);
7865 }
7866 gt::Reply::Rewrite => {
7867 g.resolved = true;
7868 g.chosen_title = None;
7869 g.choice = Some(if g.rewrite.is_some() {
7870 GithubTextChoice::Rewrite
7871 } else {
7872 GithubTextChoice::Fallback
7873 });
7874 }
7875 gt::Reply::Title(text) => {
7876 let first = text
7877 .lines()
7878 .map(str::trim)
7879 .find(|l| !l.is_empty())
7880 .unwrap_or("");
7881 let decision = if first.is_empty() {
7882 None
7883 } else {
7884 let candidate = PrMessage {
7885 title: first.to_owned(),
7886 body: String::new(),
7887 };
7888 self.judge_pr_language(&winner.worktree, &candidate).await
7889 };
7890 match gt::vet_title(
7891 &text,
7892 decision,
7893 self.state.config.graph.github_english(),
7894 ) {
7895 Ok(title) => {
7896 g.resolved = true;
7897 g.chosen_title = Some(title);
7898 g.choice = Some(GithubTextChoice::MyTitle);
7899 }
7900 Err(cats) if g.asks < 2 => {
7901 ask_again = Some(Some(
7902 cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
7903 ))
7904 }
7905 Err(_) => {
7906 g.resolved = true;
7907 g.chosen_title = None;
7908 g.choice = Some(GithubTextChoice::Fallback);
7909 }
7910 }
7911 }
7912 },
7913 }
7914 if let Some(rejected) = ask_again {
7915 let w = gt::Withheld {
7918 title: g.title,
7919 body: g.body,
7920 categories: g.categories.clone(),
7921 };
7922 let artifact = match crate::run::write_artifact(
7925 &self.state,
7926 >::artifact_name(>::fingerprint(&pr.title, &pr.body)),
7927 >::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
7928 ) {
7929 Ok(p) => Some(p),
7930 Err(e) => {
7931 tracing::warn!("could not write the withheld-text artifact: {e:#}");
7932 None
7933 }
7934 };
7935 let (detail, comparison) = gt::question_content(
7936 &w,
7937 &pr.title,
7938 &pr.body,
7939 &self.state.id,
7940 rejected.as_deref(),
7941 artifact.as_deref(),
7942 g.rewrite.as_ref(),
7943 );
7944 let mut q = ask::Question::new(
7945 self.state.id.clone(),
7946 gt::ASK_NODE.to_owned(),
7947 gt::ASK_SEAT.to_owned(),
7948 gt::question_summary(&self.state.config.graph.language, &w),
7949 detail,
7950 gt::question_choices(&w, g.rewrite.as_ref()),
7951 );
7952 q.posting_comparison = Some(comparison);
7953 q.answer_timeout = timeout;
7954 g.question = Some(q.id.clone());
7957 g.asks += 1;
7958 self.state.github_text = Some(g.clone());
7959 self.state.event(
7960 "github-text",
7961 format!("asking the owner about the withheld text ({})", q.short()),
7962 );
7963 self.state.save()?;
7964 store
7965 .put_executable(&mut q)
7966 .context("file the posting-gate question")?;
7967 if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
7968 tracing::warn!(
7969 "could not notify about posting-gate question {}: {e:#}",
7970 q.short()
7971 );
7972 }
7973 self.park_on_github_text(&g)?;
7974 return Ok(None);
7975 }
7976 }
7977 let resolved = g.chosen_title.clone();
7978 let original = g.choice == Some(GithubTextChoice::Original);
7979 self.state.event(
7981 "github-text",
7982 if original {
7983 format!(
7984 "posting the original text for: {}",
7985 match (g.title, g.body) {
7986 (true, true) => "title, description",
7987 (true, false) => "title",
7988 _ => "description",
7989 }
7990 )
7991 } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
7992 "posting the saved rewrite title and description".to_owned()
7993 } else if resolved.is_some() {
7994 "posting the owner's replacement title".to_owned()
7995 } else {
7996 "posting the neutral text for the withheld fields".to_owned()
7997 },
7998 );
7999 self.state.github_text = Some(g.clone());
8000 self.state.save()?;
8001 if g.choice == Some(GithubTextChoice::Rewrite)
8002 && let Some(r) = g.rewrite.as_ref()
8003 {
8004 return Ok(Some(PrMessage {
8005 title: r.title.clone(),
8006 body: r.body.clone(),
8007 }));
8008 }
8009 let w = gt::Withheld {
8010 title: g.title,
8011 body: g.body,
8012 categories: g.categories.clone(),
8013 };
8014 let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
8015 if let Some(t) = resolved {
8016 title = t;
8017 }
8018 let english = gt::LanguageDecision {
8020 title_english: true,
8021 body_english: true,
8022 body_complete: true,
8023 };
8024 let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
8025 if original {
8026 if g.title {
8029 title = pr.title.clone();
8030 }
8031 if g.body {
8032 body = pr.body.clone();
8033 }
8034 }
8035 Ok(Some(PrMessage { title, body }))
8036 }
8037
8038 fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
8039 self.state.github_text = Some(g.clone());
8040 self.state.end_pass_parked(ParkCause::Question);
8041 self.state.parked = true;
8042 self.state.event(
8043 "github-text",
8044 "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
8045 );
8046 self.state.save()
8047 }
8048
8049 async fn judge_pr_language(
8052 &mut self,
8053 cwd: &Path,
8054 pr: &PrMessage,
8055 ) -> Option<crate::github_text::LanguageDecision> {
8056 self.state.config.roles.language_judge.as_ref()?;
8057 if !self.state.config.graph.language_gate() {
8058 return None;
8059 }
8060 let decision =
8061 crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
8062 self.state.event(
8063 "github-text",
8064 if decision.is_some() {
8065 "language decided by the language judge"
8066 } else {
8067 "language judge unavailable; using built-in heuristics"
8068 },
8069 );
8070 decision
8071 }
8072
8073 async fn merge(&mut self) -> Result<()> {
8074 if self
8089 .state
8090 .base_sync
8091 .as_ref()
8092 .is_some_and(|s| s.conflict.is_some())
8093 || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8094 != Some(RunStatus::Gating)
8095 || !self.state.gate_allows_progress()
8103 {
8104 return Ok(());
8105 }
8106 if self.state.merge.is_some() && !self.state.pr_publication_pending {
8115 return Ok(());
8116 }
8117 let Some(winner) = self.state.winner().cloned() else {
8118 return Ok(());
8119 };
8120 let repo = self.state.repo.clone();
8121 let base = self.state.base_branch.clone();
8122 let mode = self.state.config.merge.mode;
8123 let style = self.state.config.merge.style;
8124 let facts = if is_review_run(&self.state) {
8125 refresh_reviewed_commits(&mut self.state, &winner.branch).await;
8126 let start = review_base(
8127 &repo,
8128 &self.state.config.merge.remote,
8129 &base,
8130 &self.state.base_commit,
8131 &winner.branch,
8132 )
8133 .await;
8134 branch_facts(&repo, &start, &winner.branch).await
8135 } else {
8136 None
8137 };
8138 let leaked = if is_review_run(&self.state) {
8141 leaked_subjects(&self.state, &winner.branch).await
8142 } else {
8143 Some(Vec::new())
8144 };
8145 let Some(pr) = self
8149 .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
8150 .await?
8151 else {
8152 return Ok(());
8153 };
8154 let message = pr.commit_message();
8155
8156 let outcome = match mode {
8157 MergeMode::None => MergeOutcome {
8158 mode,
8159 ok: true,
8160 detail: manual_merge_command(style, &repo, &winner.branch, &message),
8161 empty: false,
8162 },
8163 MergeMode::Pr | MergeMode::Local
8164 if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
8165 {
8166 MergeOutcome {
8167 mode,
8168 ok: false,
8169 detail: empty_candidate_detail(&self.state, &base),
8170 empty: true,
8171 }
8172 }
8173 MergeMode::Local => {
8174 let on = git::current_branch(&repo).await?;
8175 if on.as_deref() != Some(base.as_str()) {
8176 MergeOutcome {
8177 mode,
8178 ok: false,
8179 detail: format!(
8180 "{} has {} checked out, not the base branch {base}",
8181 repo.display(),
8182 on.unwrap_or_else(|| "a detached HEAD".to_owned())
8183 ),
8184 empty: false,
8185 }
8186 } else if !git::is_clean(&repo).await? {
8187 MergeOutcome {
8188 mode,
8189 ok: false,
8190 detail: format!("{} is dirty; refusing to merge", repo.display()),
8191 empty: false,
8192 }
8193 } else {
8194 let out = match style {
8195 MergeStyle::Merge => {
8196 git::merge_no_ff(&repo, &winner.branch, &message).await?
8197 }
8198 MergeStyle::Squash => {
8199 git::merge_squash(&repo, &winner.branch, &message).await?
8200 }
8201 MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
8202 };
8203 MergeOutcome {
8204 mode,
8205 ok: out.ok(),
8206 detail: if out.ok() { out.stdout } else { out.stderr },
8207 empty: false,
8208 }
8209 }
8210 }
8211 MergeMode::Pr if self.state.task_pr_branch().is_some() => {
8212 self.state.pr_publication_pending = true;
8213 self.state.save()?;
8214 let out =
8215 crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
8216 self.state.pr_publication_pending = out.is_err();
8217 MergeOutcome {
8218 mode,
8219 ok: out.is_ok(),
8220 detail: match out {
8221 Ok(url) => url,
8222 Err(e) => format!("{e:#}"),
8223 },
8224 empty: false,
8225 }
8226 }
8227 MergeMode::Pr => {
8228 let remote = self.state.config.merge.remote.clone();
8229 let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
8230 if !pushed.ok() {
8231 MergeOutcome {
8232 mode,
8233 ok: false,
8234 detail: pushed.stderr,
8235 empty: false,
8236 }
8237 } else {
8238 let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
8244 let out = match pr_merge_plan(found) {
8245 PrPlan::Create => {
8246 gh_pr_create(
8247 &winner.worktree,
8248 &base,
8249 &winner.branch,
8250 &pr.title,
8251 &pr.body,
8252 )
8253 .await
8254 }
8255 PrPlan::Adopt { url, title } => {
8256 self.state
8257 .event("merge", format!("Pr: adopted open pull request {url}"));
8258 if self.state.github_text.as_ref().is_some_and(|g| {
8259 g.resolved
8260 && g.choice == Some(GithubTextChoice::Rewrite)
8261 && g.rewrite.is_some()
8262 }) {
8263 land::set_pr_text_verbatim(
8266 &winner.worktree,
8267 &url,
8268 &pr.title,
8269 &pr.body,
8270 )
8271 .await
8272 .map(|()| url)
8273 } else {
8274 if title != pr.title
8275 && (!is_review_run(&self.state)
8276 || leaked
8277 .as_deref()
8278 .is_some_and(|l| should_retitle(&title, &pr.title, l)))
8279 && let Err(e) =
8280 if self.state.github_text.as_ref().is_some_and(|g| {
8281 g.resolved
8282 && g.title
8283 && g.choice == Some(GithubTextChoice::Original)
8284 }) {
8285 land::set_pr_title_verbatim(
8287 &winner.worktree,
8288 &url,
8289 &pr.title,
8290 )
8291 .await
8292 } else {
8293 land::set_pr_title(
8294 &mut self.state,
8295 &winner.worktree,
8296 &url,
8297 &pr.title,
8298 )
8299 .await
8300 }
8301 {
8302 tracing::warn!("could not refresh title of {url}: {e:#}");
8303 self.state
8304 .event("merge", format!("Pr: title refresh failed: {e:#}"));
8305 }
8306 Ok(url)
8307 }
8308 }
8309 PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
8310 };
8311 match out {
8312 Ok(url) => MergeOutcome {
8313 mode,
8314 ok: true,
8315 detail: url,
8316 empty: false,
8317 },
8318 Err(e) => MergeOutcome {
8319 mode,
8320 ok: false,
8321 detail: e.to_string(),
8322 empty: false,
8323 },
8324 }
8325 }
8326 }
8327 };
8328
8329 self.state.status = match (mode, outcome.ok) {
8330 (MergeMode::None, _) => RunStatus::Ready,
8331 (_, true) if outcome.empty => RunStatus::Ready,
8332 (MergeMode::Pr, true) => RunStatus::Ready,
8333 (_, true) => RunStatus::Merged,
8334 (_, false) => RunStatus::Blocked,
8335 };
8336 self.state.event(
8337 "merge",
8338 format!(
8339 "{:?}: {}",
8340 mode,
8341 outcome.detail.lines().next().unwrap_or("")
8342 ),
8343 );
8344 self.state.merge = Some(outcome);
8345 self.state.save()?;
8346
8347 if self.state.config.graph.land
8353 && mode == MergeMode::Pr
8354 && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
8355 {
8356 self.run_land().await?;
8357 }
8358 self.settle_questions();
8363 Ok(())
8364 }
8365
8366 async fn run_land(&mut self) -> Result<()> {
8377 crate::pr_head::ensure_owner(&self.state)?;
8378 let url = self
8379 .state
8380 .pr
8381 .as_ref()
8382 .map(|pr| pr.url.clone())
8383 .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
8384 .unwrap_or_default();
8385 let url = url.lines().next().unwrap_or("").trim().to_owned();
8386 if !url.starts_with("http") {
8387 return Ok(());
8388 }
8389 loop {
8392 match land::land(&mut self.state, &url).await {
8393 Ok(_)
8394 if self
8395 .state
8396 .merge_repairs
8397 .last()
8398 .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
8399 {
8400 if self.repair_merge_review().await? {
8401 continue;
8402 }
8403 }
8404 Ok(pr) if self.state.parked => {
8405 let _ = pr;
8409 }
8410 Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
8411 self.state.status = RunStatus::Superseded;
8412 self.state.save()?;
8413 }
8414 Ok(pr) => {
8415 self.state.status = match pr.state {
8416 land::PrLifecycle::Merged => RunStatus::Merged,
8417 _ => RunStatus::Blocked,
8418 };
8419 if bump::should_release_bump(self.state.status)
8426 && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
8427 {
8428 self.state
8434 .event("bump", format!("release bump skipped: {e:#}"));
8435 }
8436 if self.state.status == RunStatus::Merged {
8439 crate::followup::after_merge(&mut self.state, &pr.url).await;
8440 }
8441 self.state.save()?;
8442 }
8443 Err(e) => {
8444 if e.is::<crate::repo_lock::ResourceBlocked>() {
8445 return Err(e);
8446 }
8447 self.state.status = RunStatus::Blocked;
8448 self.state.event("land", format!("gave up: {e}"));
8449 self.state.save()?;
8450 }
8451 }
8452 break;
8453 }
8454 Ok(())
8455 }
8456
8457 async fn repair_merge_review(&mut self) -> Result<bool> {
8460 use crate::run::MergeRepairPhase as Phase;
8461 crate::pr_head::ensure_owner(&self.state)?;
8462 let i = self.state.merge_repairs.len() - 1;
8463 let request = self.state.merge_repairs[i].clone();
8464 if request.phase == Phase::Fixing {
8465 let restored = match self.state.winner().cloned() {
8466 Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
8467 None => Err(anyhow::anyhow!("repair has no winning branch")),
8468 };
8469 let why = match restored {
8470 Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
8471 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
8472 Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
8473 };
8474 self.hold_merge_repair(i, why)?;
8475 return Ok(false);
8476 }
8477 let failure = if i >= self.state.config.graph.land_rounds
8478 || self.state.config.graph.review_rounds == 0
8479 {
8480 Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
8481 } else {
8482 None
8483 };
8484 if let Some(why) = failure {
8485 self.hold_merge_repair(i, why)?;
8486 return Ok(false);
8487 }
8488 if request.phase == Phase::Requested {
8489 let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
8490 Ok(pr) => pr,
8491 Err(e) => {
8492 self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
8493 return Ok(false);
8494 }
8495 };
8496 self.state.merge_repairs[i].phase = Phase::Fixing;
8497 self.state.land_approval = None;
8498 self.state.land_armed_head = None;
8499 self.state.save()?;
8500 let evidence = serde_json::to_string_pretty(&request)?;
8503 let reason = format!(
8504 "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
8505 );
8506 let budget = self.state.config.graph.land_rounds;
8507 let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
8508 match result {
8509 Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
8510 Ok(land::Fixed::Failed(why)) => {
8511 self.hold_merge_repair(
8512 i,
8513 format!(
8514 "repair failed: {why}; inspect the branch and run an explicit review"
8515 ),
8516 )?;
8517 return Ok(false);
8518 }
8519 Err(e) => {
8520 self.hold_merge_repair(
8521 i,
8522 format!(
8523 "repair failed: {e:#}; inspect the branch and run an explicit review"
8524 ),
8525 )?;
8526 return Ok(false);
8527 }
8528 }
8529 self.begin_merge_repair_review(i)?;
8533 }
8534 self.review_loop().await?;
8535 self.gate().await?;
8536 if self.resource_halt()? || self.state.parked {
8537 return Ok(false);
8538 }
8539 if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8540 != Some(RunStatus::Gating)
8541 || !self.state.gate_allows_progress()
8542 {
8543 self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
8544 return Ok(false);
8545 }
8546 let winner = self
8549 .state
8550 .winner()
8551 .cloned()
8552 .context("repair has no winning branch")?;
8553 let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8554 if self.state.gate_head.as_deref() != Some(after.as_str())
8555 || !git::is_clean(&winner.worktree).await?
8556 {
8557 self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
8558 return Ok(false);
8559 }
8560 let expected = self
8561 .state
8562 .pr_pushed_head
8563 .as_deref()
8564 .unwrap_or(&request.head)
8565 .to_owned();
8566 let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
8567 let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
8570 let refspec = format!("{after}:refs/heads/{public}");
8571 let push = git::git_raw(
8572 &winner.worktree,
8573 &["push", &lease, &self.state.config.merge.remote, &refspec],
8574 )
8575 .await;
8576 match push {
8577 Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
8578 Ok(out) => {
8579 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
8580 return Ok(false);
8581 }
8582 Err(e) => {
8583 self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
8584 return Ok(false);
8585 }
8586 }
8587 self.state.merge_repairs[i].phase = Phase::Complete;
8588 self.state.status = RunStatus::Landing;
8589 self.state.event(
8590 "repair",
8591 "new review and gate completed; fresh owner merge decision required",
8592 );
8593 self.state.save()?;
8594 Ok(true)
8595 }
8596
8597 fn merge_repair_artifacts(&self) -> PathBuf {
8598 let artifacts = agent::artifacts_dir(&self.state.dir());
8599 match self
8600 .state
8601 .merge_repairs
8602 .last()
8603 .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
8604 {
8605 Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
8606 None => artifacts,
8607 }
8608 }
8609
8610 fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
8611 self.state.reviews.clear();
8612 self.state.gate.clear();
8613 self.state.gate_ran = false;
8614 self.state.gate_head = None;
8615 self.state.gate_fixes.clear();
8616 self.state.gate_repair_e2e = None;
8617 self.state.pre_gate.clear();
8618 self.state.pre_gate_commit = None;
8619 self.state.contested_handoff = None;
8620 self.state.land_reruns = None;
8621 self.state.land_approval = None;
8622 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
8623 self.state.status = RunStatus::Reviewing;
8624 self.state.save()
8625 }
8626
8627 fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
8628 self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
8629 self.state.merge_repairs[index].failure = Some(why.clone());
8630 self.state.status = RunStatus::Blocked;
8631 self.state.event("repair", why);
8632 self.state.save()
8633 }
8634
8635 fn seat(&mut self, key: &str, agent: &str) -> SeatState {
8639 if let Some(existing) = self.state.seats.get(key)
8640 && existing.agent == agent
8641 {
8642 return existing.clone();
8643 }
8644 let fresh = if self.state.seats.contains_key(key) {
8647 handover_seat(key, agent, self.state.next_seat_seed())
8648 } else {
8649 SeatState::new(key, agent, self.state.seed)
8650 };
8651 self.state.seats.insert(key.to_owned(), fresh.clone());
8652 fresh
8653 }
8654
8655 fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
8661 match self.state.seats.get(key) {
8662 Some(s) if s.agent != spec.id => {
8663 self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
8664 }
8665 _ => spec,
8666 }
8667 }
8668
8669 fn view(&self, c: &Candidate) -> CandidateView {
8671 let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
8672 .unwrap_or_default();
8673 let (patch, _) = blind::sanitize_patch(
8674 &format!("candidate {} patch", c.label),
8675 &raw,
8676 &self.state.config.blind,
8677 );
8678 CandidateView {
8679 label: c.label,
8680 branch: c.branch.clone(),
8681 summary: c.summary.clone(),
8682 stat: c.stat.clone(),
8683 patch,
8684 }
8685 }
8686
8687 fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
8689 let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
8690 prompt::judge(
8691 "(see above)",
8692 &views,
8693 self.roles.judges.len(),
8694 base_short,
8695 "en",
8696 )
8697 }
8698
8699 fn vote_prompt_full(
8705 &self,
8706 j: usize,
8707 labels: &[char],
8708 language: &str,
8709 candidates: &[Candidate],
8710 base_short: &str,
8711 ) -> String {
8712 let mut text = format!(
8713 "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
8714 prompt::final_vote(labels, language),
8715 self.state.instruction,
8716 self.candidate_block(candidates, base_short)
8717 );
8718 if let Some(own) = self
8719 .state
8720 .judgements
8721 .get(j)
8722 .filter(|r| !r.ranking.is_empty())
8723 {
8724 let reasons = own
8725 .reasons
8726 .iter()
8727 .map(|(k, v)| format!("- {k}: {v}"))
8728 .collect::<Vec<_>>()
8729 .join("\n");
8730 text.push_str(&format!(
8731 "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
8732 own.ranking.iter().collect::<String>(),
8733 if reasons.is_empty() {
8734 ""
8735 } else {
8736 ", because:\n"
8737 }
8738 ));
8739 }
8740 if !self.state.deliberation.is_empty() {
8741 text.push_str("\n# What was argued before this vote\n");
8742 for t in self.transcript(&[], j) {
8743 text.push_str(&format!(
8744 "\n## {}{}\n\n{}\n",
8745 t.who,
8746 if t.is_self { " (you)" } else { "" },
8747 t.body.trim()
8748 ));
8749 }
8750 }
8751 text
8752 }
8753
8754 fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
8761 let mut turns = Vec::new();
8762 for j in &self.state.judgements {
8763 if j.ranking.is_empty() {
8764 continue;
8765 }
8766 let reasons = j
8767 .reasons
8768 .iter()
8769 .map(|(k, v)| format!("- {k}: {v}"))
8770 .collect::<Vec<_>>()
8771 .join("\n");
8772 turns.push(Turn {
8773 who: format!("Judge {} (opening ranking)", j.judge),
8774 is_self: j.judge == self_idx + 1,
8775 body: format!(
8776 "Ranked {}{}{reasons}",
8777 j.ranking.iter().collect::<String>(),
8778 if reasons.is_empty() {
8779 ""
8780 } else {
8781 ", because:\n"
8782 }
8783 ),
8784 });
8785 }
8786 for t in self
8787 .state
8788 .deliberation
8789 .iter()
8790 .flat_map(|r| r.turns.iter())
8791 .chain(current)
8792 {
8793 turns.push(Turn {
8794 who: format!("Judge {}", t.judge),
8795 is_self: t.judge == self_idx + 1,
8796 body: t.body.clone(),
8797 });
8798 }
8799 turns
8800 }
8801}
8802
8803fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
8805 agent::has_session(spec.kind, seat, sessions)
8806}
8807
8808fn next_untried_in_roster<'a>(
8829 roster: &'a [AgentSpec],
8830 start: usize,
8831 tried: &BTreeSet<String>,
8832) -> Option<&'a AgentSpec> {
8833 roster
8834 .get(start + 1..)?
8835 .iter()
8836 .find(|s| !tried.contains(&s.id))
8837}
8838
8839fn pick_successor<'a>(
8860 roster: &'a [AgentSpec],
8861 start: usize,
8862 tried: &BTreeSet<String>,
8863 carried: Option<&BTreeSet<String>>,
8864 others: &BTreeSet<String>,
8865) -> Option<&'a AgentSpec> {
8866 let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
8867 roster
8868 .get(start + 1..)
8869 .and_then(|tail| {
8870 tail.iter()
8871 .filter(free)
8872 .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
8873 })
8874 .or_else(|| {
8875 carried
8876 .is_some()
8877 .then(|| roster.iter().find(free))
8878 .flatten()
8879 })
8880 .or_else(|| match carried {
8881 Some(c) => next_for_seat(roster, start, tried, c),
8882 None => next_untried_in_roster(roster, start, tried),
8883 })
8884}
8885
8886fn next_for_seat<'a>(
8896 roster: &'a [AgentSpec],
8897 start: usize,
8898 round_tried: &BTreeSet<String>,
8899 carried_failed: &BTreeSet<String>,
8900) -> Option<&'a AgentSpec> {
8901 roster
8902 .get(start + 1..)?
8903 .iter()
8904 .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
8905 .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
8906}
8907
8908fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
8914 let Some(h) = hist.filter(|_| !roster.is_empty()) else {
8915 return spec;
8916 };
8917 let ok = |id: &str| !h.failed.contains(id);
8918 if let Some(last) = h.last_ok.as_deref()
8919 && ok(last)
8920 && let Some(s) = roster.iter().find(|s| s.id == last)
8921 {
8922 return s.clone();
8923 }
8924 if ok(&spec.id) {
8925 return spec;
8926 }
8927 let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
8928 roster
8929 .get(from + 1..)
8930 .into_iter()
8931 .flatten()
8932 .chain(roster.iter())
8933 .find(|s| ok(&s.id))
8934 .cloned()
8935 .unwrap_or(spec)
8936}
8937
8938pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
8942 SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
8943}
8944
8945const TIMED_OUT: &str = "timed out";
8948
8949impl FailClass {
8950 fn of(out: &AgentOutcome) -> Option<Self> {
8952 match out {
8953 AgentOutcome::Ok(_) => None,
8954 AgentOutcome::Quota(_) => Some(Self::Quota),
8955 AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
8956 AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
8957 AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
8958 AgentOutcome::ResourceBlocked(_) => None,
8961 }
8962 }
8963
8964 fn stem_word(&self) -> &'static str {
8966 match self {
8967 Self::Quota => "quota",
8968 _ => "handover",
8969 }
8970 }
8971}
8972
8973fn failure_signature(msg: &str) -> String {
8977 let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
8978 if let Some(code) = line
8981 .strip_prefix("process exited with no usable output (exit code ")
8982 .and_then(|s| s.strip_suffix(')'))
8983 .and_then(|s| s.parse::<i32>().ok())
8984 {
8985 line = format!("exited with some({code}) and no usable output");
8986 } else if matches!(
8987 line.as_str(),
8988 "process exited with no usable output (no exit code reported)"
8989 | "process exited with no usable output (exit code unavailable)"
8990 ) {
8991 line = "exited with none and no usable output".to_owned();
8992 }
8993 let mut out = Vec::new();
8994 for word in line.split_whitespace() {
8995 if word.contains('/') || word.contains('\\') {
8996 out.push("<path>".to_owned());
8997 continue;
8998 }
8999 let mut w = String::new();
9000 let mut in_digits = false;
9001 for c in word.chars() {
9002 if c.is_ascii_digit() {
9003 if !in_digits {
9004 w.push('#');
9005 }
9006 in_digits = true;
9007 } else {
9008 in_digits = false;
9009 w.push(c);
9010 }
9011 }
9012 out.push(w);
9013 }
9014 out.join(" ").chars().take(120).collect()
9015}
9016
9017fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
9024 match cur {
9025 FailClass::Quota | FailClass::Timeout => true,
9026 FailClass::Other(_) => prev != Some(cur),
9027 }
9028}
9029
9030fn fail_reason(out: &AgentOutcome) -> String {
9032 match out {
9033 AgentOutcome::Ok(_) => String::new(),
9034 AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
9035 AgentOutcome::Dropped(o) => format!(
9036 "the CLI dropped the stream ({})",
9037 o.dropped
9038 .as_ref()
9039 .map(|d| d.why.as_str())
9040 .unwrap_or("it ended without delivering its answer")
9041 ),
9042 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
9043 e.lines().next().unwrap_or("").chars().take(160).collect()
9044 }
9045 }
9046}
9047
9048pub(crate) fn record_handover(
9051 state: &mut RunState,
9052 node: &str,
9053 seat: &str,
9054 from: &str,
9055 to: &str,
9056 class: &FailClass,
9057 reason: &str,
9058) {
9059 let message = if *class == FailClass::Quota {
9060 format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
9061 } else {
9062 format!("{seat}: handed over {from} -> {to} ({reason})")
9063 };
9064 state.event(node, message);
9065 state.handovers.push(Handover {
9066 at: Timestamp::now(),
9067 node: node.to_owned(),
9068 seat: seat.to_owned(),
9069 from: from.to_owned(),
9070 to: to.to_owned(),
9071 reason: reason.to_owned(),
9072 });
9073}
9074
9075fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
9089 commands.iter().any(|c| c.exit_code.is_none())
9090}
9091
9092fn verified_noop_claim(
9105 usable: bool,
9106 commands: &[agent::CommandEvidence],
9107 text: &str,
9108) -> Option<String> {
9109 (usable && !has_unconfirmed_command(commands))
9110 .then(|| verdict::verified_noop(text))
9111 .flatten()
9112}
9113
9114fn short(commit: &str) -> String {
9115 commit.chars().take(7).collect()
9116}
9117
9118fn make_executable(path: &Path) -> Result<()> {
9119 #[cfg(unix)]
9120 {
9121 use std::os::unix::fs::PermissionsExt as _;
9122 let mut perms = std::fs::metadata(path)?.permissions();
9123 perms.set_mode(0o755);
9124 std::fs::set_permissions(path, perms)?;
9125 }
9126 #[cfg(not(unix))]
9127 {
9128 let _ = path;
9129 }
9130 Ok(())
9131}
9132
9133struct WaveCtx<'a> {
9140 run: &'a str,
9143 node: &'a str,
9145 prompts: &'a Prompts,
9146 cache: Option<&'a Path>,
9148 round: Option<usize>,
9151 carry_seats: bool,
9155 pauses: Option<(&'a Pause, &'a Pause)>,
9158}
9159
9160fn repository_wait(
9162 state: &RunState,
9163 pauses: Option<(&Pause, &Pause)>,
9164) -> crate::repo_lock::WaitPolicy {
9165 let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
9166 let inherited = crate::repo_lock::WaitPolicy::current();
9167 crate::repo_lock::WaitPolicy::new(
9168 Duration::from_secs(state.config.graph.timeout_resource_wait),
9169 move || {
9170 inherited.cancelled()
9171 || pauses
9172 .as_ref()
9173 .is_some_and(|(a, b)| a.parked() || b.parked())
9174 },
9175 )
9176}
9177
9178fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
9179 if error.is::<crate::repo_lock::ResourceBlocked>() {
9180 state.resource_block = Some(crate::run::ResourceBlock {
9181 node: "repository".into(),
9182 kind: if error.to_string().contains("disk pressure") {
9183 "disk pressure"
9184 } else {
9185 "repository coordination"
9186 }
9187 .into(),
9188 detail: error.to_string(),
9189 at: Timestamp::now(),
9190 });
9191 state.event("resource", error.to_string());
9192 }
9193}
9194
9195async fn coordinate_operation(
9196 state: &mut RunState,
9197 cwd: &Path,
9198 wait: &crate::repo_lock::WaitPolicy,
9199) -> Result<Option<crate::repo_lock::SeatLease>> {
9200 let result =
9201 crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
9202 .await;
9203 if let Err(e) = &result {
9204 record_repository_block(state, e);
9205 }
9206 result
9207}
9208
9209async fn run_one(
9211 job: SeatJob,
9212 sem: Arc<Semaphore>,
9213 ctx: &WaveCtx<'_>,
9214 state: &mut RunState,
9215 attempt: usize,
9216) -> (SeatState, AgentOutcome) {
9217 let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
9218 .await
9219 .pop()
9220 .expect("one job in, one result out");
9221 (seat, out)
9222}
9223
9224async fn wave(
9230 jobs: Vec<SeatJob>,
9231 sem: Arc<Semaphore>,
9232 ctx: &WaveCtx<'_>,
9233 state: &mut RunState,
9234 attempt: usize,
9235) -> Vec<(usize, SeatState, AgentOutcome)> {
9236 let WaveCtx {
9237 run,
9238 node,
9239 prompts,
9240 cache,
9241 round,
9242 carry_seats: _,
9243 pauses,
9244 } = *ctx;
9245 for job in &jobs {
9246 state.seat_started(node, &job.seat.key, job.timeout, attempt);
9247 }
9248 if let Err(e) = state.save() {
9249 tracing::warn!("could not persist in-progress seats: {e:#}");
9254 }
9255 let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
9271 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
9275 let cache_guard = if let Some(cache_dir) = cache {
9276 if jobs_had_a_writer {
9277 let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
9278 let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
9279 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, node, &cancel).await
9280 {
9281 Ok(Some(g)) => Some(g),
9282 Ok(None) => {
9285 state.resource_block = Some(crate::run::ResourceBlock {
9286 node: node.to_owned(),
9287 kind: "shared build cache".to_owned(),
9288 detail: "the run is parking".to_owned(),
9289 at: Timestamp::now(),
9290 });
9291 return block_wave(
9292 jobs,
9293 state,
9294 node,
9295 attempt,
9296 "parked while waiting for the shared build cache",
9297 );
9298 }
9299 Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
9302 Err(e) => {
9303 let why = format!("{e:#}");
9307 state.resource_block = Some(crate::run::ResourceBlock {
9308 node: node.to_owned(),
9309 kind: "shared build cache".to_owned(),
9310 detail: why.clone(),
9311 at: Timestamp::now(),
9312 });
9313 state.event(
9314 "resource",
9315 format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
9316 );
9317 return block_wave(jobs, state, node, attempt, &why);
9318 }
9319 }
9320 } else {
9321 None
9322 }
9323 } else {
9324 None
9325 };
9326 state.seats_spawning(node, attempt);
9329 if let Err(e) = state.save() {
9330 tracing::warn!("could not persist the start of a wave: {e:#}");
9331 }
9332 let slot_limit = resource_limit
9336 + jobs
9337 .iter()
9338 .map(|j| j.timeout)
9339 .max()
9340 .unwrap_or_default()
9341 .saturating_mul(jobs.len() as u32);
9342 let mut set = tokio::task::JoinSet::new();
9343 let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
9346 let overlay = prompts.overlay(node);
9347 let repository_wait = repository_wait(state, pauses);
9348 for (i, mut job) in jobs.into_iter().enumerate() {
9349 job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
9350 if cache.is_some() {
9351 job.prompt.push('\n');
9352 job.prompt
9353 .push_str(&prompt::build_cache_note(node, job.allow_write));
9354 }
9355 let sem = Arc::clone(&sem);
9356 let run = run.to_owned();
9357 let node = node.to_owned();
9358 let attachments = if node == "implement" {
9361 state.attachments.clone()
9362 } else {
9363 Vec::new()
9364 };
9365 let cache = cache
9376 .filter(|_| job.allow_write && cache_guard.is_some())
9377 .map(Path::to_path_buf);
9378 let spawned_tx = spawned_tx.clone();
9379 let repository_wait = repository_wait.clone();
9380 let coordination = state.config.coordination.clone();
9381 let fetched_base = state.fetched_base.clone();
9382 set.spawn(async move {
9383 let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
9387 Ok(p) => p,
9388 Err(_) => {
9389 return (
9390 i,
9391 job.seat,
9392 AgentOutcome::ResourceBlocked(format!(
9393 "no parallel slot became free within {}s",
9394 slot_limit.as_secs()
9395 )),
9396 false,
9397 );
9398 }
9399 };
9400 spawned_tx.send(job.seat.key.clone()).ok();
9401 let mut seat = job.seat;
9402 let out = repository_wait
9403 .scope(agent::invoke(
9404 &job.spec,
9405 &mut seat,
9406 &Invocation {
9407 scope: crate::agent::InvocationScope::Repository,
9408 coordination: coordination.as_ref(),
9409 fetched_base: fetched_base.as_ref(),
9410 cwd: &job.cwd,
9411 prompt: &job.prompt,
9412 timeout: job.timeout,
9413 allow_write: job.allow_write,
9414 unsandboxed: false,
9415 sessions: job.sessions,
9416 artifacts: &job.artifacts,
9417 stem: &job.stem,
9418 run: &run,
9419 node: &node,
9420 cache_dir: cache.as_deref(),
9421 attachments: &attachments,
9422 writable: &[],
9423 },
9424 ))
9425 .await;
9426 let interrupted = out
9427 .as_ref()
9428 .err()
9429 .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
9430 let out = match out {
9431 Ok(o) if o.usable() => AgentOutcome::Ok(o),
9432 Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
9433 Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
9441 Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
9442 Ok(o) => AgentOutcome::Failed(format!(
9443 "process exited with no usable output ({})",
9444 crate::notices::exit_status(o.exit_code)
9445 )),
9446 Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
9447 AgentOutcome::ResourceBlocked(e.to_string())
9448 }
9449 Err(e) => AgentOutcome::Failed(e.to_string()),
9450 };
9451 (i, seat, out, interrupted)
9452 });
9453 }
9454 drop(spawned_tx);
9455 let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
9456 loop {
9457 let joined = tokio::select! {
9458 Some(key) = spawned_rx.recv() => {
9459 state.seat_spawned(&key);
9461 if let Err(e) = state.save() {
9462 tracing::warn!("could not persist a seat's start: {e:#}");
9463 }
9464 continue;
9465 }
9466 joined = set.join_next() => joined,
9467 };
9468 let Some(joined) = joined else {
9469 break;
9470 };
9471 let (i, seat, out, interrupted) = match joined {
9472 Ok(v) => v,
9473 Err(e) => {
9477 tracing::error!("agent task panicked: {e}");
9478 continue;
9479 }
9480 };
9481 state.seat_finished(&seat.key);
9482 if interrupted
9483 && node == "implement"
9484 && let Some(label) = state
9485 .candidates
9486 .iter()
9487 .find(|candidate| seat.key == format!("impl-{}", candidate.label))
9488 .map(|candidate| candidate.label)
9489 {
9490 state.candidate_preservation.insert(
9491 label,
9492 "interrupted implementation awaiting persistence".into(),
9493 );
9494 state.interrupted_implementations.insert(label);
9495 }
9496 if let AgentOutcome::ResourceBlocked(why) = &out
9497 && state.resource_block.is_none()
9498 {
9499 state.resource_block = Some(crate::run::ResourceBlock {
9500 node: node.to_owned(),
9501 kind: if why.contains("disk pressure") {
9502 "disk pressure"
9503 } else if why.contains("repository coordination") {
9504 "repository coordination"
9505 } else {
9506 "parallel slot"
9507 }
9508 .to_owned(),
9509 detail: why.clone(),
9510 at: Timestamp::now(),
9511 });
9512 let activity = if interrupted {
9513 "agent execution interrupted"
9514 } else {
9515 "no agent was started"
9516 };
9517 state.event("resource", format!("{node}: {activity} - {why}"));
9518 }
9519 record_jobs(state, node, round, &seat.key, &out);
9520 if let Err(e) = state.save() {
9521 tracing::warn!("could not persist a seat's completion: {e:#}");
9522 }
9523 if collected.len() <= i {
9524 collected.resize_with(i + 1, || None);
9525 }
9526 collected[i] = Some((i, seat, out));
9527 }
9528 if state
9534 .active
9535 .values()
9536 .any(|a| a.node == node && a.attempt == attempt)
9537 {
9538 state
9539 .active
9540 .retain(|_, a| !(a.node == node && a.attempt == attempt));
9541 if let Err(e) = state.save() {
9542 tracing::warn!("could not persist the end of a wave: {e:#}");
9543 }
9544 }
9545 if let Some(cache_dir) = cache
9552 && jobs_had_a_writer
9553 {
9554 crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
9555 }
9556 if let Some(guard) = cache_guard {
9557 guard.release();
9558 }
9559 collected.into_iter().flatten().collect()
9560}
9561
9562fn record_jobs(
9573 state: &mut RunState,
9574 node: &str,
9575 round: Option<usize>,
9576 seat: &str,
9577 out: &AgentOutcome,
9578) {
9579 let commands: &[agent::CommandEvidence] = match out {
9580 AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
9581 AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
9582 };
9583 let checked_at = Timestamp::now();
9584 for c in commands {
9585 state.jobs.push(JobRecord {
9586 node: node.to_owned(),
9587 round,
9588 seat: seat.to_owned(),
9589 id: c.id.clone(),
9590 description: c.description.clone(),
9591 checked_at,
9592 status: match c.exit_code {
9593 Some(0) => JobStatus::Completed,
9594 Some(_) => JobStatus::Failed,
9595 None => JobStatus::Unknown,
9596 },
9597 exit_code: c.exit_code,
9598 result_summary: c.result_summary.clone(),
9599 source: c.source.clone(),
9600 });
9601 }
9602}
9603
9604fn round_is_clean(
9625 blocking: usize,
9626 e2e_ok: bool,
9627 answered: usize,
9628 expected: usize,
9629 quota_missing: usize,
9630 policy: IncompleteReviewPolicy,
9631) -> bool {
9632 if blocking != 0 || !e2e_ok {
9633 return false;
9634 }
9635 if answered == expected || policy == IncompleteReviewPolicy::Warn {
9636 return true;
9637 }
9638 answered > 0 && expected - answered <= quota_missing
9639}
9640
9641pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
9665 if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
9666 return Some(RunStatus::Gating);
9667 }
9668 let last = reviews.last()?;
9669 let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
9670 if reviews.len() < max_rounds && !stagnant {
9671 return None;
9672 }
9673 if last.incomplete() && last.blocking == 0 {
9674 return Some(RunStatus::Blocked);
9675 }
9676 if last.e2e_status() == E2eStatus::ResourceBlocked {
9677 return None;
9678 }
9679 Some(if last.e2e.iter().all(CommandOutcome::ok) {
9680 RunStatus::Gating
9681 } else {
9682 RunStatus::Blocked
9683 })
9684}
9685
9686pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
9701 if nudged {
9702 (full / 4).max(Duration::from_secs(120)).min(full)
9703 } else {
9704 full
9705 }
9706}
9707
9708#[allow(clippy::too_many_arguments)]
9737async fn ask_json_wave<T>(
9738 jobs: Vec<SeatJob>,
9739 sem: Arc<Semaphore>,
9740 retries: usize,
9741 roster: &[AgentSpec],
9742 ctx: &WaveCtx<'_>,
9743 losses: &mut Vec<QuotaLoss>,
9744 state: &mut RunState,
9745 validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
9746) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9747where
9748 T: serde::de::DeserializeOwned + Send + 'static,
9749{
9750 ask_wave_with(
9751 jobs,
9752 sem,
9753 retries,
9754 roster,
9755 ctx,
9756 losses,
9757 state,
9758 &|text: &str| {
9759 let v = verdict::extract_json::<T>(text)?;
9760 validate(&v)?;
9761 Ok(v)
9762 },
9763 )
9764 .await
9765}
9766
9767#[allow(clippy::too_many_arguments)]
9776async fn ask_wave_with<T>(
9777 jobs: Vec<SeatJob>,
9778 sem: Arc<Semaphore>,
9779 retries: usize,
9780 roster: &[AgentSpec],
9781 ctx: &WaveCtx<'_>,
9782 losses: &mut Vec<QuotaLoss>,
9783 state: &mut RunState,
9784 parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
9785) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
9786where
9787 T: Send + 'static,
9788{
9789 let n = jobs.len();
9790 let originals: Vec<SeatJob> = jobs;
9791 let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
9792 let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
9793 let mut nudges: Vec<usize> = vec![0; n];
9800 let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
9804 let mut tried: Vec<BTreeSet<String>> = specs
9805 .iter()
9806 .map(|s| BTreeSet::from([s.id.clone()]))
9807 .collect();
9808 let starts: Vec<usize> = specs
9809 .iter()
9810 .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
9811 .collect();
9812 let carry = ctx.carry_seats && !roster.is_empty();
9813 let carried: Vec<BTreeSet<String>> = originals
9816 .iter()
9817 .map(|j| {
9818 state
9819 .seat_history
9820 .get(&j.seat.key)
9821 .filter(|_| carry)
9822 .map(|h| h.failed.clone())
9823 .unwrap_or_default()
9824 })
9825 .collect();
9826 let mut prev: Vec<Option<FailClass>> = originals
9827 .iter()
9828 .map(|j| {
9829 state
9830 .seat_history
9831 .get(&j.seat.key)
9832 .filter(|_| carry)
9833 .and_then(|h| h.last_fail.clone())
9834 })
9835 .collect();
9836 let next_agent =
9837 |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
9838 let others: BTreeSet<String> = specs
9839 .iter()
9840 .enumerate()
9841 .filter(|(j, _)| *j != i)
9842 .map(|(_, s)| s.id.clone())
9843 .collect();
9844 pick_successor(
9845 roster,
9846 starts[i],
9847 tried,
9848 carry.then(|| &carried[i]),
9849 &others,
9850 )
9851 .cloned()
9852 };
9853 let mut fresh: Vec<Option<String>> = vec![None; n];
9854 let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
9855 let mut pending: Vec<usize> = (0..n).collect();
9856
9857 let max_rounds = (retries + 1) * roster.len().max(1) + 1;
9862 for round in 0..max_rounds {
9863 if pending.is_empty() {
9864 break;
9865 }
9866 let mut batch = Vec::with_capacity(pending.len());
9867 let mut renudged: Vec<&str> = Vec::new();
9868 for &i in &pending {
9869 let src = &originals[i];
9870 let full: &str = match &src.handover {
9875 Some(h) if specs[i].id != src.spec.id => h,
9876 _ => &src.prompt,
9877 };
9878 let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
9881 (
9882 full.to_owned(),
9883 src.timeout,
9884 format!("{}-{word}-{}", src.stem, specs[i].id),
9885 )
9886 } else if nudges[i] == 0 {
9887 (full.to_owned(), src.timeout, src.stem.clone())
9888 } else {
9889 renudged.push(src.seat.key.as_str());
9890 let why = done[i]
9891 .as_ref()
9892 .and_then(|r| r.as_ref().err().map(ToString::to_string))
9893 .unwrap_or_else(|| "no parsable answer".to_owned());
9894 let nudge = prompt::nudge(&why);
9895 let nudged = has_context(&specs[i], &seats[i], src.sessions);
9896 let prompt = if nudged {
9897 nudge
9898 } else {
9899 format!("{full}\n\n---\n\n{nudge}")
9900 };
9901 (
9902 prompt,
9903 retry_budget(src.timeout, nudged),
9904 format!("{}-retry{}", src.stem, nudges[i]),
9905 )
9906 };
9907 batch.push(SeatJob {
9908 spec: specs[i].clone(),
9909 seat: seats[i].clone(),
9910 cwd: src.cwd.clone(),
9911 prompt,
9912 timeout,
9913 allow_write: src.allow_write,
9914 sessions: src.sessions,
9915 artifacts: src.artifacts.clone(),
9916 stem,
9917 handover: None,
9918 });
9919 }
9920
9921 if !renudged.is_empty() {
9922 state.event(
9923 ctx.node,
9924 format!("retry {round}: re-asking {}", renudged.join(", ")),
9925 );
9926 }
9927 let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
9928 let mut still = Vec::new();
9929 for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
9930 seats[i] = seat;
9931 if let AgentOutcome::ResourceBlocked(why) = &out {
9934 done[i] = Some(Err(anyhow::anyhow!("{why}")));
9935 continue;
9936 }
9937 let class = FailClass::of(&out);
9938 let nudge_first = matches!(out, AgentOutcome::Dropped(_))
9941 && nudges[i] < retries
9942 && !roster.is_empty();
9943 if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
9944 let next = should_hand_over(prev[i].as_ref(), &cur)
9945 .then(|| next_agent(i, &tried[i], &specs))
9946 .flatten();
9947 if carry {
9948 let h = state
9949 .seat_history
9950 .entry(originals[i].seat.key.clone())
9951 .or_default();
9952 h.failed.insert(specs[i].id.clone());
9953 h.last_fail = Some(cur.clone());
9954 if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
9955 h.last_ok = None;
9956 }
9957 if let Err(e) = state.save() {
9960 tracing::warn!("could not persist a seat's failure history: {e:#}");
9961 }
9962 }
9963 if let Some(next) = next {
9964 record_handover(
9965 state,
9966 ctx.node,
9967 &originals[i].seat.key,
9968 &specs[i].id,
9969 &next.id,
9970 &cur,
9971 &fail_reason(&out),
9972 );
9973 tried[i].insert(next.id.clone());
9974 prev[i] = Some(cur.clone());
9975 seats[i] =
9976 handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
9977 specs[i] = next;
9978 fresh[i] = Some(cur.stem_word().to_owned());
9979 nudges[i] = 0;
9980 done[i] = Some(Err(anyhow::anyhow!(
9981 "handed over after: {}",
9982 fail_reason(&out)
9983 )));
9984 still.push(i);
9985 continue;
9986 }
9987 }
9988 if carry
9989 && !nudge_first
9990 && let Some(cur) = class.clone()
9991 {
9992 let h = state
9994 .seat_history
9995 .entry(originals[i].seat.key.clone())
9996 .or_default();
9997 h.failed.insert(specs[i].id.clone());
9998 h.last_fail = Some(cur);
9999 }
10000 let parsed = match out {
10001 AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
10002 AgentOutcome::Quota(o) => {
10003 last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
10004 Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
10005 }
10006 AgentOutcome::Dropped(o) => {
10010 let why = o
10011 .dropped
10012 .as_ref()
10013 .map(|d| d.why.as_str())
10014 .unwrap_or("the CLI ended the stream without delivering its answer");
10015 Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
10016 }
10017 AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10018 Err(anyhow::anyhow!(e))
10019 }
10020 };
10021 let quota = class == Some(FailClass::Quota);
10022 let failed = parsed.is_err();
10023 done[i] = Some(parsed);
10024 if carry && !failed {
10025 let h = state
10026 .seat_history
10027 .entry(originals[i].seat.key.clone())
10028 .or_default();
10029 h.failed.remove(&specs[i].id);
10030 h.last_ok = Some(specs[i].id.clone());
10031 h.last_fail = None;
10032 }
10033 let agent_failure = class.is_some()
10041 && !nudge_first
10042 && !roster.is_empty()
10043 && next_agent(i, &tried[i], &specs).is_some();
10044 if failed && !quota && !agent_failure && nudges[i] < retries {
10045 nudges[i] += 1;
10046 still.push(i);
10047 }
10048 }
10049 pending = still;
10050 }
10051
10052 for (i, q) in last_quota.into_iter().enumerate() {
10055 if let Some(reset) = q {
10056 losses.push(QuotaLoss {
10057 seat: originals[i].seat.key.clone(),
10058 node: ctx.node.to_owned(),
10059 at: Timestamp::now(),
10060 reset,
10061 });
10062 }
10063 }
10064
10065 seats
10066 .into_iter()
10067 .zip(done)
10068 .zip(nudges)
10069 .map(|((seat, res), attempts)| {
10070 (
10071 seat,
10072 res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
10073 attempts,
10074 )
10075 })
10076 .collect()
10077}
10078
10079fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
10081 results
10082 .iter()
10083 .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
10084}
10085
10086fn block_wave(
10090 jobs: Vec<SeatJob>,
10091 state: &mut RunState,
10092 node: &str,
10093 attempt: usize,
10094 why: &str,
10095) -> Vec<(usize, SeatState, AgentOutcome)> {
10096 state.resource_wait = None;
10097 state
10098 .active
10099 .retain(|_, a| !(a.node == node && a.attempt == attempt));
10100 if let Err(e) = state.save() {
10101 tracing::warn!("could not persist a resource-blocked wave: {e:#}");
10102 }
10103 jobs.into_iter()
10104 .enumerate()
10105 .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
10106 .collect()
10107}
10108
10109async fn acquire_cache_lease(
10122 state: &mut RunState,
10123 cache_dir: &Path,
10124 owner: &crate::cache::Owner,
10125 budget: Duration,
10126 context: &str,
10127 cancel: &(dyn Fn() -> bool + Sync),
10128) -> Result<Option<crate::cache::Guard>> {
10129 let home = crate::run::home();
10130 let started = Instant::now();
10131 let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
10132 Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
10133 Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
10134 Err(e) => {
10135 state.event(
10136 "verify",
10137 format!("{context}: could not check the shared build cache: {e:#}"),
10138 );
10139 if let Err(e2) = state.save() {
10140 tracing::warn!("could not persist a cache-check failure: {e2:#}");
10141 }
10142 return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
10143 }
10144 };
10145 state.event(
10146 "verify",
10147 format!(
10148 "{context}: waiting for the shared build cache at {} ({})",
10149 cache_dir.display(),
10150 busy.describe()
10151 ),
10152 );
10153 state.resource_wait = Some(crate::run::ResourceWait {
10156 node: context.to_owned(),
10157 kind: "shared build cache".to_owned(),
10158 holder: busy.describe(),
10159 since: Timestamp::now(),
10160 limit_secs: budget.as_secs(),
10161 });
10162 if let Err(e) = state.save() {
10163 tracing::warn!("could not persist a cache wait: {e:#}");
10164 }
10165 let remaining = budget.saturating_sub(started.elapsed());
10166 let waited = crate::cache::wait_for_unless(
10167 &home,
10168 cache_dir,
10169 owner,
10170 remaining,
10171 Duration::from_secs(5),
10172 cancel,
10173 )
10174 .await;
10175 state.resource_wait = None;
10176 match waited {
10177 Ok(g) => Ok(g),
10178 Err(e) => {
10179 state.event("verify", format!("{context}: {e:#}"));
10180 if let Err(e2) = state.save() {
10181 tracing::warn!("could not persist a cache wait timeout: {e2:#}");
10182 }
10183 Err(e)
10184 }
10185 }
10186}
10187
10188#[derive(Debug)]
10193struct CacheCheckError(String);
10194
10195impl std::fmt::Display for CacheCheckError {
10196 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10197 f.write_str(&self.0)
10198 }
10199}
10200
10201impl std::error::Error for CacheCheckError {}
10202
10203#[allow(clippy::too_many_arguments)]
10224async fn with_cache_lease<'s, F, Fut>(
10225 state: &'s mut RunState,
10226 cache_dir: Option<&Path>,
10227 node: &str,
10228 seat: &str,
10229 worktree: &Path,
10230 head: &str,
10231 budget: Duration,
10232 context: &str,
10233 wait: &crate::repo_lock::WaitPolicy,
10234 body: F,
10235) -> Result<(Vec<CommandOutcome>, bool)>
10236where
10237 F: FnOnce(&'s mut RunState, Duration) -> Fut,
10238 Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
10239{
10240 let started = Instant::now();
10243 let home = crate::run::home();
10244 let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10245 let guard = if let Some(cache_dir) = cache_dir {
10246 let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
10247 match acquire_cache_lease(state, cache_dir, &owner, resource_limit, context, &|| {
10248 wait.cancelled()
10249 })
10250 .await
10251 .and_then(|g| {
10252 g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
10253 }) {
10254 Ok(g) => Some(g),
10255 Err(e) => {
10256 return Ok((
10257 vec![CommandOutcome {
10258 command: "(waiting for the shared build cache)".into(),
10259 code: None,
10260 output_tail: e.to_string(),
10261 duration_ms: started.elapsed().as_millis() as u64,
10262 resource_blocked: true,
10263 }],
10264 false,
10265 ));
10266 }
10267 }
10268 } else {
10269 None
10270 };
10271 let _coordination = coordinate_operation(state, worktree, wait).await?;
10272 if let Some(cache_dir) = cache_dir {
10273 let identity = crate::cache::Identity::new(worktree, head);
10274 if let Err(e) = crate::cache::ensure_fresh(
10275 &home,
10276 cache_dir,
10277 &identity,
10278 &state.config.verify.cargo_manifests(),
10279 ) {
10280 state.event(
10281 "verify",
10282 format!(
10283 "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
10284 worktree.display(),
10285 short(head)
10286 ),
10287 );
10288 if let Some(guard) = guard {
10289 guard.release();
10290 }
10291 return Ok((
10292 vec![CommandOutcome {
10293 command: "(confirming the shared build cache is fresh)".into(),
10294 code: None,
10295 output_tail: e.to_string(),
10296 duration_ms: started.elapsed().as_millis() as u64,
10297 resource_blocked: true,
10298 }],
10299 false,
10300 ));
10301 }
10302 }
10303 let (outcomes, retried, timed_out_pids) = body(state, budget).await;
10304 if !timed_out_pids.is_empty() {
10305 wait_for_timed_out_children_to_die(&timed_out_pids).await;
10306 }
10307 drop(_coordination);
10310 if let Some(guard) = guard {
10311 guard.release();
10312 }
10313 Ok((outcomes, retried))
10314}
10315
10316async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
10328 wait_for_pids_with(
10329 pids,
10330 crate::proc::pid_alive,
10331 LEASE_RELEASE_POLL,
10332 LEASE_RELEASE_MAX_WAIT,
10333 )
10334 .await;
10335}
10336
10337async fn wait_for_pids_with<F: Fn(u32) -> bool>(
10343 pids: &[u32],
10344 alive: F,
10345 poll: Duration,
10346 max_wait: Duration,
10347) {
10348 let deadline = Instant::now() + max_wait;
10349 loop {
10350 if pids.iter().all(|&pid| !alive(pid)) {
10351 return;
10352 }
10353 if Instant::now() >= deadline {
10354 return;
10355 }
10356 tokio::time::sleep(poll).await;
10357 }
10358}
10359
10360fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
10367 outcomes.iter().any(|o| o.resource_blocked)
10368}
10369
10370pub(crate) enum GateFix {
10372 Retry,
10374 Stop,
10377 Defer,
10380}
10381
10382fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
10390 let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
10391 red.peek().is_some()
10392 && red.all(|o| {
10393 !o.resource_blocked
10394 && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
10395 && !o.output_tail.trim().is_empty()
10396 })
10397}
10398
10399fn e2e_outcome_label(o: &CommandOutcome) -> String {
10403 if o.ok() {
10404 return "pass".to_owned();
10405 }
10406 let reason = if o.build_failed() {
10407 format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
10408 } else {
10409 format!("FAIL ({:?})", o.code)
10410 };
10411 format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
10412}
10413
10414async fn run_e2e_with_retry(
10422 state: &mut RunState,
10423 shell: &[String],
10424 commands: &[String],
10425 worktree: &Path,
10426 timeout: Duration,
10427 context: &str,
10428) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
10429 let (mut e2e, mut timed_out_pids) = run_commands(
10430 state, "verify", "e2e", 0, shell, commands, worktree, timeout,
10431 )
10432 .await;
10433 for o in &e2e {
10434 state.event(
10435 "verify",
10436 format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
10437 );
10438 }
10439 let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
10443 if verify_retried {
10444 state.event(
10445 "verify",
10446 format!(
10447 "{context}: verify could not build/link, not a test result — retrying once \
10448 before concluding"
10449 ),
10450 );
10451 let retried = run_commands(
10452 state, "verify", "e2e", 1, shell, commands, worktree, timeout,
10453 )
10454 .await;
10455 e2e = retried.0;
10456 timed_out_pids.extend(retried.1);
10459 for o in &e2e {
10460 state.event(
10461 "verify",
10462 format!(
10463 "{context}: retry `{}` -> {}",
10464 o.command,
10465 e2e_outcome_label(o)
10466 ),
10467 );
10468 }
10469 }
10470 (e2e, verify_retried, timed_out_pids)
10471}
10472
10473#[allow(clippy::too_many_arguments)]
10489async fn run_commands(
10490 state: &mut RunState,
10491 node: &str,
10492 task: &str,
10493 attempt: usize,
10494 shell: &[String],
10495 commands: &[String],
10496 cwd: &Path,
10497 timeout: Duration,
10498) -> (Vec<CommandOutcome>, Vec<u32>) {
10499 if commands.is_empty() {
10500 return (Vec::new(), Vec::new());
10505 }
10506 let mut out = Vec::new();
10507 let mut timed_out_pids = Vec::new();
10508 let total = commands.len();
10509 for (idx, command) in commands.iter().enumerate() {
10510 state.task_command(task, node, attempt, command, idx + 1, total, timeout);
10511 if let Err(e) = state.save() {
10512 tracing::warn!("could not persist an in-progress {task} command: {e:#}");
10513 }
10514 let started = Instant::now();
10515 let mut cmd = tokio::process::Command::new(&shell[0]);
10516 cmd.quiet();
10517 cmd.args(&shell[1..])
10518 .arg(command)
10519 .current_dir(cwd)
10520 .stdin(std::process::Stdio::null())
10521 .stdout(std::process::Stdio::piped())
10522 .stderr(std::process::Stdio::piped())
10523 .kill_on_drop(true);
10524 let mut outputs = Vec::new();
10525 let protected = (|| -> Result<()> {
10526 if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
10527 outputs.push(guard);
10528 }
10529 if let Some(cache) = state.config.cache_dir() {
10530 outputs.push(crate::retention::OutputLease::cache(
10531 &crate::run::home(),
10532 &cache,
10533 false,
10534 )?);
10535 }
10536 Ok(())
10537 })();
10538 if let Err(e) = protected {
10539 out.push(CommandOutcome {
10540 command: command.clone(),
10541 code: None,
10542 output_tail: format!("output ownership unavailable: {e:#}"),
10543 duration_ms: 0,
10544 resource_blocked: true,
10545 });
10546 continue;
10547 }
10548 let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
10549 let (code, body) = match spawned {
10550 Ok(mut running) => {
10551 let pid = running.child().id();
10552 let (stdout, out_reader) = agent::drain(running.child().stdout.take());
10553 let (stderr, err_reader) = agent::drain(running.child().stderr.take());
10554 let mut paths = vec![cwd.to_owned()];
10555 if let Some(cache) = state.config.cache_dir() {
10556 paths.push(cache);
10557 }
10558 let mut pressure = None;
10559 let status = tokio::select! {
10560 status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
10561 reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
10562 pressure = Some(reason); None
10563 }
10564 };
10565 let (code, failure) = match status {
10566 Some(Ok(Ok(status))) => (
10567 status.code(),
10568 status
10569 .code()
10570 .is_none()
10571 .then(|| format!("command terminated: {status}")),
10572 ),
10573 Some(Ok(Err(error))) => {
10574 (None, Some(format!("failed waiting for command: {error}")))
10575 }
10576 Some(Err(_)) => {
10577 if !running.coordinated()
10581 && let Some(pid) = pid
10582 {
10583 timed_out_pids.push(pid);
10584 }
10585 (
10586 None,
10587 Some(format!("timed out after {}s", timeout.as_secs())),
10588 )
10589 }
10590 None => (None, None),
10591 };
10592 running.finish().await;
10593 if let Some(reason) = pressure {
10594 state.resource_block = Some(crate::run::ResourceBlock {
10595 node: node.to_owned(),
10596 kind: "disk pressure".to_owned(),
10597 detail: reason.clone(),
10598 at: Timestamp::now(),
10599 });
10600 out.push(CommandOutcome {
10601 command: command.clone(),
10602 code: None,
10603 output_tail: reason,
10604 duration_ms: started.elapsed().as_millis() as u64,
10605 resource_blocked: true,
10606 });
10607 break;
10608 }
10609 let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
10610 body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
10611 if let Some(failure) = failure {
10612 body.push('\n');
10613 body.push_str(&failure);
10614 }
10615 (code, body)
10616 }
10617 Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
10618 };
10619 out.push(CommandOutcome {
10620 command: command.clone(),
10621 code,
10622 output_tail: tail(&body, OUTPUT_TAIL),
10623 duration_ms: started.elapsed().as_millis() as u64,
10624 resource_blocked: false,
10625 });
10626 }
10627 state.task_finished(task);
10628 if let Err(e) = state.save() {
10629 tracing::warn!("could not persist the end of {task}: {e:#}");
10630 }
10631 (out, timed_out_pids)
10632}
10633
10634fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
10646 let repo = repo.display();
10647 match style {
10648 MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
10649 MergeStyle::Squash => {
10650 let subject = message
10653 .lines()
10654 .next()
10655 .unwrap_or(branch)
10656 .replace(['\\', '"', '$', '`'], "");
10657 format!(
10658 "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
10659 )
10660 }
10661 MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
10662 }
10663}
10664
10665const PR_TITLE_MAX: usize = 240;
10676
10677fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
10688 let line = queue::first_line(text)?;
10689 let chars: Vec<char> = line.chars().collect();
10690 let (kept, mark) = if chars.len() <= max {
10691 (line.to_owned(), "")
10692 } else if let Some(end) = sentence_end(&chars, max) {
10693 (chars[..end].iter().collect::<String>(), "")
10694 } else {
10695 let room = max.saturating_sub(3);
10696 let cut = if chars[room].is_whitespace() {
10699 room
10700 } else {
10701 chars[..room]
10702 .iter()
10703 .rposition(|c| c.is_whitespace())
10704 .unwrap_or(room)
10705 };
10706 let head: String = chars[..cut].iter().collect();
10707 let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
10708 (head.to_owned(), "...")
10709 };
10710 if kept.is_empty() {
10711 return None;
10712 }
10713 if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
10714 return None;
10715 }
10716 if !ascii_only && !kept.chars().any(char::is_alphabetic) {
10717 return None;
10718 }
10719 Some(format!("{kept}{mark}"))
10720}
10721
10722fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
10726 const MIN: usize = 20;
10727 for i in MIN..max.min(chars.len()) {
10728 if !matches!(chars[i], '.' | '!' | '?') {
10729 continue;
10730 }
10731 let Some(&next) = chars.get(i + 1) else {
10732 continue;
10733 };
10734 if !next.is_whitespace() {
10735 continue;
10736 }
10737 let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
10738 if after.is_some_and(|c| c.is_ascii_lowercase()) {
10739 continue;
10740 }
10741 let word: String = chars[..i]
10742 .iter()
10743 .rev()
10744 .take_while(|c| !c.is_whitespace())
10745 .collect::<Vec<_>>()
10746 .into_iter()
10747 .rev()
10748 .collect();
10749 let word = word.to_ascii_lowercase();
10750 if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
10751 continue;
10752 }
10753 let end = chars[..i]
10754 .iter()
10755 .rposition(|c| !c.is_whitespace())
10756 .map_or(i, |p| p + 1);
10757 return Some(end);
10758 }
10759 None
10760}
10761
10762struct PrMessage {
10767 title: String,
10768 body: String,
10769}
10770
10771impl PrMessage {
10772 fn commit_message(&self) -> String {
10776 format!("{}\n\n{}", self.title, self.body)
10777 }
10778}
10779
10780fn title_marker(line: &str) -> Option<&str> {
10782 let line = line.trim();
10783 let head = line.get(..6)?;
10784 head.eq_ignore_ascii_case("title:")
10785 .then(|| line[6..].trim())
10786}
10787
10788fn summary_title(summary: &str) -> Option<String> {
10793 let first = summary.lines().find(|l| !l.trim().is_empty())?;
10794 let raw = title_marker(first)?;
10795 if raw.is_empty() {
10796 return None;
10797 }
10798 let title = queue::title_from(raw, PR_TITLE_MAX);
10799 let lower = title.to_ascii_lowercase();
10800 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10801 return None;
10802 }
10803 Some(title)
10804}
10805
10806const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
10808
10809pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
10811
10812pub(crate) fn is_review_run(state: &RunState) -> bool {
10816 state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
10817}
10818
10819fn review_title(state: &RunState) -> Option<String> {
10824 english_subject(
10825 state.reviewed_commits.as_ref()?.first()?,
10826 state.config.graph.github_english(),
10827 )
10828}
10829
10830fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
10833 let raw = raw.trim();
10834 if raw.is_empty() {
10835 return None;
10836 }
10837 if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
10838 return None;
10839 }
10840 if !ascii_only && !raw.chars().any(char::is_alphabetic) {
10841 return None;
10842 }
10843 let title = queue::title_from(raw, PR_TITLE_MAX);
10844 let lower = title.to_ascii_lowercase();
10845 if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
10846 return None;
10847 }
10848 Some(title)
10849}
10850
10851#[derive(Debug, Clone, PartialEq, Eq)]
10854struct BranchFacts {
10855 commits: Vec<(String, String)>,
10857 stat: String,
10859}
10860
10861const STAT_FILE_LINES: usize = 25;
10863const COMMITS_MAX_CHARS: usize = 20_000;
10865
10866async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
10869 let commits = git::commit_log(repo, base, branch).await.ok()?;
10870 if commits.is_empty() {
10871 return None;
10872 }
10873 let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
10874 let lines: Vec<&str> = stat.lines().collect();
10875 let stat = if lines.len() > STAT_FILE_LINES + 1 {
10876 let omitted = lines.len() - 1 - STAT_FILE_LINES;
10877 let more = format!(" ... {omitted} more file(s)");
10878 let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
10879 kept.push(&more);
10880 kept.push(lines[lines.len() - 1]);
10881 kept.join("\n")
10882 } else {
10883 lines.join("\n")
10884 };
10885 Some(BranchFacts { commits, stat })
10886}
10887
10888fn markdown_safe(text: &str) -> String {
10891 text.replace("</details>", "</details>")
10892 .replace("\x60\x60\x60", "~~~")
10893}
10894
10895fn neutral_title(state: &RunState, winner: char) -> String {
10896 format!(
10897 "chore: land candidate {} of run {}",
10898 winner.to_ascii_uppercase(),
10899 state.id
10900 )
10901}
10902
10903pub(crate) async fn repair_landing_gate(
10906 state: &mut RunState,
10907 winner: &Candidate,
10908 outcomes: &[CommandOutcome],
10909) -> Result<GateFix> {
10910 if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
10911 return Ok(GateFix::Stop);
10912 }
10913 let mut runner = Runner {
10914 roles: state.config.resolve_roles()?,
10915 sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
10916 state: state.clone(),
10917 pause: Pause::new(),
10918 interrupt: Pause::new(),
10919 };
10920 let result = runner.gate_fix_round(winner, outcomes).await;
10921 *state = runner.state;
10922 result
10923}
10924
10925pub(crate) async fn check_gate_repair_e2e(
10928 state: &mut RunState,
10929 winner: &Candidate,
10930 wait: &crate::repo_lock::WaitPolicy,
10931) -> Result<GateFix> {
10932 if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
10933 return Ok(GateFix::Retry);
10934 }
10935 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10936 if state
10937 .gate_repair_e2e
10938 .as_ref()
10939 .is_some_and(|r| r.head == head && r.passed)
10940 {
10941 return Ok(GateFix::Retry);
10942 }
10943 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10944 head: head.clone(),
10945 passed: false,
10946 outcomes: Vec::new(),
10947 });
10948 state.save()?;
10949 let commands = state.config.verify.e2e.clone();
10950 let outcomes = if commands.is_empty() {
10951 Vec::new()
10952 } else {
10953 let shell = state.config.shell();
10954 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
10955 let cache_dir = state.config.cache_dir();
10956 let (outcomes, _) = with_cache_lease(
10957 state,
10958 cache_dir.as_deref(),
10959 "e2e",
10960 "e2e",
10961 &winner.worktree,
10962 &head,
10963 timeout,
10964 "gate repair e2e",
10965 wait,
10966 |state, budget| async move {
10967 run_e2e_with_retry(
10968 state,
10969 &shell,
10970 &commands,
10971 &winner.worktree,
10972 budget,
10973 "gate repair e2e",
10974 )
10975 .await
10976 },
10977 )
10978 .await?;
10979 outcomes
10980 };
10981 let deferred = verify_inconclusive(&outcomes);
10982 let passed = outcomes.iter().all(CommandOutcome::ok)
10983 && git::rev_parse(&winner.worktree, "HEAD").await? == head;
10984 state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
10985 head,
10986 passed,
10987 outcomes,
10988 });
10989 state.save()?;
10990 if deferred {
10991 Ok(GateFix::Defer)
10992 } else if !passed {
10993 state.event(
10994 "gate",
10995 "verify.e2e failed after gate repair; nothing published",
10996 );
10997 Ok(GateFix::Stop)
10998 } else {
10999 Ok(GateFix::Retry)
11000 }
11001}
11002
11003pub(crate) async fn run_landing_gate(
11005 state: &mut RunState,
11006 winner: &Candidate,
11007 wait: &crate::repo_lock::WaitPolicy,
11008) -> Result<Vec<CommandOutcome>> {
11009 let shell = state.config.shell();
11010 let gate_commands = state.config.verify.gate.clone();
11011 let has_pre_gate = !state.config.verify.pre_gate.is_empty();
11012 let outcomes = if gate_commands.is_empty() && !has_pre_gate {
11013 Vec::new()
11014 } else {
11015 let timeout = Duration::from_secs(state.config.graph.verify_timeout());
11016 let cache_dir = state.config.cache_dir();
11017 let head = git::rev_parse(&winner.worktree, "HEAD").await?;
11018 let (outcomes, _) = with_cache_lease(
11019 state,
11020 cache_dir.as_deref(),
11021 "gate",
11022 "gate",
11023 &winner.worktree,
11024 &head,
11025 timeout,
11026 "final gate",
11027 wait,
11028 |state, budget| {
11029 let shell = shell.clone();
11030 let gate_commands = gate_commands.clone();
11031 let winner = winner.clone();
11032 async move {
11033 Runner::run_pre_gate(state, &winner).await;
11034 let (outcomes, timed_out_pids) = run_commands(
11035 state,
11036 "gate",
11037 "gate",
11038 0,
11039 &shell,
11040 &gate_commands,
11041 &winner.worktree,
11042 budget,
11043 )
11044 .await;
11045 (outcomes, false, timed_out_pids)
11046 }
11047 },
11048 )
11049 .await?;
11050 outcomes
11051 };
11052 if outcomes.is_empty() {
11053 state.event("gate", "no gate commands configured; not tested");
11058 }
11059 for o in &outcomes {
11060 state.event(
11061 "gate",
11062 format!(
11063 "`{}` -> {}",
11064 o.command,
11065 if o.ok() {
11066 "pass".to_owned()
11067 } else {
11068 format!(
11069 "FAIL ({:?})\n{}",
11070 o.code,
11071 tail(&o.output_tail, EVENT_OUTPUT_TAIL)
11072 )
11073 }
11074 ),
11075 );
11076 }
11077 Ok(outcomes)
11078}
11079
11080pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
11086 if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
11087 ""
11088 } else {
11089 pr_title
11090 }
11091}
11092
11093pub fn landing_subject_source(state: &RunState) -> String {
11097 if is_review_run(state) {
11098 let winner = state.candidates.first().map_or('A', |c| c.label);
11099 return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
11100 }
11101 state.instruction.clone()
11102}
11103
11104fn summary_without_title(summary: &str) -> String {
11107 let mut lines = summary.trim().lines().peekable();
11108 if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
11109 lines.next();
11110 }
11111 lines.collect::<Vec<_>>().join("\n").trim().to_owned()
11112}
11113
11114#[cfg(test)]
11126fn pr_message(state: &RunState, winner: char) -> PrMessage {
11127 pr_message_with(state, winner, None)
11128}
11129
11130#[cfg(test)]
11133fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
11134 let raw = pr_message_raw(
11135 state,
11136 winner,
11137 facts,
11138 crate::pr_outcome::FinalHead::Unreported,
11139 ChangeNote::NotAsked,
11140 );
11141 let id = crate::scrub::Identity::current();
11142 PrMessage {
11143 title: crate::scrub::scrub(&raw.title, &id),
11144 body: crate::scrub::scrub(&raw.body, &id),
11145 }
11146}
11147
11148#[derive(Debug, Clone, Copy)]
11150enum ChangeNote<'a> {
11151 NotAsked,
11153 Summary(&'a crate::run::ChangeSummary),
11155 Unavailable(&'a str),
11157}
11158
11159fn pr_message_raw(
11160 state: &RunState,
11161 winner: char,
11162 facts: Option<&BranchFacts>,
11163 head: crate::pr_outcome::FinalHead<'_>,
11164 change: ChangeNote<'_>,
11165) -> PrMessage {
11166 if !matches!(change, ChangeNote::NotAsked) {
11167 let (title, mut body) = match change {
11168 ChangeNote::Summary(c) => (
11169 c.title.clone(),
11170 format!("## Summary\n\n{}\n\n", c.body.trim()),
11171 ),
11172 ChangeNote::Unavailable(why) => (
11173 neutral_title(state, winner),
11174 format!(
11175 "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
11176 ),
11177 ),
11178 ChangeNote::NotAsked => unreachable!(),
11179 };
11180 body.push_str(&public_pr_evidence(state, head));
11181 body.push_str(&format!(
11182 "\n---\nmagi:run/{} magi:candidate-{}\n",
11183 state.id,
11184 winner.to_ascii_lowercase()
11185 ));
11186 return PrMessage { title, body };
11187 }
11188 pr_preview(state, winner, facts, head)
11189}
11190
11191fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
11194 let mut body = crate::pr_outcome::review_rounds(state);
11195 body.push_str(&crate::pr_outcome::final_verification(state, head));
11196 let open = state.last_round_findings();
11197 if !open.is_empty() {
11198 body.push_str("## Open review findings\n\n");
11199 for f in open {
11200 body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
11201 }
11202 body.push('\n');
11203 }
11204 if let Some(c) = state.change_summary.as_ref()
11205 && head == crate::pr_outcome::FinalHead::At(&c.head)
11206 {
11207 body.push_str(&format!(
11208 "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
11209 c.base, c.head
11210 ));
11211 }
11212 body
11213}
11214
11215fn pr_preview(
11217 state: &RunState,
11218 winner: char,
11219 facts: Option<&BranchFacts>,
11220 head: crate::pr_outcome::FinalHead<'_>,
11221) -> PrMessage {
11222 let summary = state
11223 .candidates
11224 .iter()
11225 .find(|c| c.label == winner)
11226 .map(|c| c.summary.as_str())
11227 .unwrap_or_default();
11228 let review = is_review_run(state);
11231 let title = if review {
11232 facts
11233 .and_then(|f| {
11234 english_subject(&f.commits.first()?.0, state.config.graph.github_english())
11235 })
11236 .or_else(|| review_title(state))
11237 .or_else(|| {
11238 state
11239 .candidates
11240 .iter()
11241 .find(|c| c.label == winner)
11242 .filter(|c| !c.branch.starts_with("magi/"))
11243 .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
11244 })
11245 .unwrap_or_else(|| neutral_title(state, winner))
11246 } else {
11247 summary_title(summary).unwrap_or_else(|| {
11248 english_title(
11249 &state.instruction,
11250 PR_TITLE_MAX,
11251 state.config.graph.github_english(),
11252 )
11253 .unwrap_or_else(|| neutral_title(state, winner))
11254 })
11255 };
11256
11257 let mut body = String::new();
11258 let what = summary_without_title(summary);
11259 if !what.is_empty() {
11260 body.push_str("## Summary\n\n");
11261 body.push_str(&what);
11262 body.push_str("\n\n");
11263 }
11264
11265 body.push_str(&crate::pr_outcome::review_rounds(state));
11267 body.push_str(&crate::pr_outcome::final_verification(state, head));
11268 let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
11269
11270 let open = state.open_findings();
11271 if !open.is_empty() {
11272 body.push_str("## Open review findings\n\n");
11273 for f in &open {
11274 body.push_str(&format!(
11275 "- `{}` [{:?}] {}\n",
11276 f.id, f.severity, "Context retained in local review records."
11277 ));
11278 }
11279 body.push('\n');
11280 }
11281
11282 if let Some(fix) = fix
11283 && !fix.rejected.is_empty()
11284 {
11285 body.push_str("## Declined by the fixer\n\n");
11286 for r in &fix.rejected {
11287 body.push_str(&format!(
11288 "- `{}`: {}\n",
11289 r.id, "Reason retained in local review records."
11290 ));
11291 }
11292 body.push('\n');
11293 }
11294
11295 if review {
11296 body.push_str("## Commits under review\n\n");
11298 if let Some(facts) = facts {
11299 let mut left = COMMITS_MAX_CHARS;
11300 for (i, (subject, text)) in facts.commits.iter().enumerate() {
11301 let mut entry = format!("- {}\n", markdown_safe(subject));
11302 for l in markdown_safe(text).lines() {
11303 entry.push_str(format!(" {l}\n").trim_end_matches(' '));
11304 }
11305 if left == 0 {
11306 body.push_str(&format!(
11307 "- ... {} more commit(s)\n",
11308 facts.commits.len() - i
11309 ));
11310 break;
11311 }
11312 if entry.len() > left {
11313 let mut end = left;
11316 while !entry.is_char_boundary(end) {
11317 end -= 1;
11318 }
11319 entry.truncate(end);
11320 entry.push_str("\n ... (truncated)\n");
11321 left = 0;
11322 } else {
11323 left -= entry.len();
11324 }
11325 body.push_str(&entry);
11326 }
11327 if !facts.stat.trim().is_empty() {
11328 body.push_str(&format!(
11329 "\n## Diff stat\n\n```\n{}\n```\n",
11330 markdown_safe(facts.stat.trim())
11331 ));
11332 }
11333 } else {
11334 match &state.reviewed_commits {
11335 Some(subjects) => {
11336 for s in subjects {
11337 body.push_str(&format!("- {}\n", s.trim()));
11338 }
11339 }
11340 None => {
11341 let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
11344 body.push_str(rest.trim());
11345 body.push('\n');
11346 }
11347 }
11348 }
11349 } else {
11350 let task = state.instruction.trim();
11351 let task = if task.is_empty() {
11352 "(empty task)"
11353 } else {
11354 task
11355 };
11356 body.push_str(&format!(
11357 "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
11358 task.replace("</details>", "</details>")
11359 ));
11360 }
11361
11362 body.push_str(&format!(
11363 "\n---\nmagi:run/{} magi:candidate-{}\n",
11364 state.id,
11365 winner.to_ascii_lowercase()
11366 ));
11367
11368 PrMessage { title, body }
11369}
11370
11371fn seeded_instruction(state: &RunState) -> String {
11374 let mut inputs = Vec::new();
11375 if let Some(start) = &state.sources.start_from {
11376 inputs.push(format!(
11377 "Start from `{}` at `{}` (including ancestry).",
11378 start.token, start.sha
11379 ));
11380 }
11381 for pick in &state.sources.pick_commits {
11382 inputs.push(format!(
11383 "Pick `{}` at `{}` (single commit).",
11384 pick.token, pick.sha
11385 ));
11386 }
11387 if inputs.is_empty() {
11388 state.instruction.clone()
11389 } else {
11390 format!(
11391 "{}\n\n# Explicit source inputs already prepared\n\n{}",
11392 state.instruction,
11393 inputs.join("\n")
11394 )
11395 }
11396}
11397
11398async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
11402 let base = &state.base_branch;
11403 let mut against = base.clone();
11404 if mode == MergeMode::Pr {
11405 let remote = &state.config.merge.remote;
11409 let tracking = format!("{remote}/{base}");
11410 let fetched = git::fetch(repo, remote, base).await;
11411 if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
11412 against = tracking;
11413 } else {
11414 return false;
11415 }
11416 }
11417 matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
11418}
11419
11420fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
11423 let mut detail = format!(
11424 "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
11425 nothing to open a pull request for"
11426 );
11427 detail.push_str(
11428 "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
11429 );
11430 detail
11431}
11432
11433#[derive(Debug, PartialEq, Eq)]
11436enum PrPlan {
11437 Create,
11438 Adopt { url: String, title: String },
11439 Stop(String),
11440}
11441
11442fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
11445 match found {
11446 Ok(land::OpenPr::None) => PrPlan::Create,
11447 Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
11448 Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
11449 "several open pull requests exist for this branch, not picking one: {}",
11450 urls.join(" ")
11451 )),
11452 Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
11453 }
11454}
11455
11456pub(crate) async fn gh_pr_create(
11458 cwd: &Path,
11459 base: &str,
11460 head: &str,
11461 title: &str,
11462 body: &str,
11463) -> Result<String> {
11464 let out = tokio::process::Command::new("gh")
11465 .args([
11466 "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
11467 ])
11468 .current_dir(cwd)
11469 .quiet()
11470 .stdin(std::process::Stdio::null())
11471 .output()
11472 .await
11473 .context("spawn gh")?;
11474 if out.status.success() {
11475 Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
11476 } else {
11477 bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
11478 }
11479}
11480
11481async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
11483 match std::fs::symlink_metadata(path) {
11484 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
11485 git::worktree_remove_clean(repo, path).await?;
11488 return Ok(());
11489 }
11490 Err(e) => return Err(e.into()),
11491 Ok(_) => {}
11492 }
11493 if !git::is_clean(path).await? {
11494 bail!(
11495 "worktree has uncommitted work; refusing removal: {}",
11496 path.display()
11497 );
11498 }
11499 if !git::worktree_remove_clean(repo, path).await? {
11500 bail!("worktree removal refused: {}", path.display());
11501 }
11502 Ok(())
11503}
11504
11505pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
11514 if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
11515 bail!("refusing fold: {reason}");
11516 }
11517 if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
11518 bail!("candidate preservation incomplete; refusing cleanup");
11519 }
11520 std::fs::create_dir_all(home.join("runs").join(&state.id))?;
11524 let _owner = crate::pr_head::owner_lock(home, &state.id)?;
11525 let _driver = crate::driver::maintenance(home, &state.id)?;
11526 let mut output_guards = Vec::new();
11527 for entry in std::fs::read_dir(state.worktree_root())
11528 .into_iter()
11529 .flatten()
11530 .flatten()
11531 {
11532 if entry.path().join(".git").is_file() {
11533 output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
11534 }
11535 }
11536 let repo = state.repo.clone();
11537 let wait = repository_wait(state, None);
11540 let _coordination = match coordinate_operation(state, &repo, &wait).await {
11541 Ok(lease) => lease,
11542 Err(e) => {
11543 state.save_under(home)?;
11544 return Err(e);
11545 }
11546 };
11547 let _source_outputs = if state.status == RunStatus::Merged
11550 && state.merge.as_ref().is_some_and(|merge| merge.ok)
11551 {
11552 verify_merged_fold_sources(state, drop_winner, home).await?
11553 } else {
11554 Vec::new()
11555 };
11556 let root = state.worktree_root();
11557 let winner = state.tally.as_ref().map(|t| t.winner);
11558 let mut removed = Vec::new();
11559
11560 for i in 0..state.candidates.len() {
11561 let c = state.candidates[i].clone();
11562 let is_winner = Some(c.label) == winner;
11563 if is_winner && !drop_winner {
11564 continue;
11565 }
11566 let existed = c.worktree.exists();
11567 remove_preserved_worktree(&repo, &c.worktree).await?;
11570 if existed {
11571 removed.push(c.worktree.to_string_lossy().into_owned());
11572 }
11573 let handed_over = state.released_branches.contains(&c.branch);
11576 if !handed_over && git::branch_exists(&repo, &c.branch).await? {
11577 if !git::branch_delete(&repo, &c.branch).await? {
11578 bail!("candidate branch removal refused");
11579 }
11580 removed.push(c.branch.clone());
11581 }
11582 state.candidates[i].folded = true;
11583 }
11584
11585 for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
11586 let path = name.path();
11587 let keep = !drop_winner
11588 && winner.is_some_and(|w| {
11589 path.file_name()
11590 .is_some_and(|n| n == format!("cand-{w}").as_str())
11591 });
11592 if keep {
11593 continue;
11594 }
11595 remove_preserved_worktree(&repo, &path).await?;
11596 removed.push(path.to_string_lossy().into_owned());
11597 }
11598
11599 remove_if_empty(&root);
11608
11609 if state.enabled_worktree_config && drop_winner {
11610 git::release_worktree_config(&repo).await.ok();
11614 state.enabled_worktree_config = false;
11615 }
11616 state.save_under(home)?;
11617 Ok(removed)
11618}
11619
11620async fn verify_merged_fold_sources(
11621 state: &RunState,
11622 drop_winner: bool,
11623 home: &Path,
11624) -> Result<Vec<std::fs::File>> {
11625 let winner = state.tally.as_ref().map(|t| t.winner);
11626 let mut heads = BTreeSet::new();
11627 for candidate in &state.candidates {
11628 if Some(candidate.label) == winner && !drop_winner {
11629 continue;
11630 }
11631 anyhow::ensure!(
11632 candidate.worktree.parent() == Some(state.worktree_root().as_path()),
11633 "candidate recovery source is outside the inventoried run root: {}",
11634 candidate.worktree.display()
11635 );
11636 match std::fs::symlink_metadata(&candidate.worktree) {
11637 Ok(_) => {}
11638 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11639 Err(error) => return Err(error.into()),
11640 }
11641 if !state.released_branches.contains(&candidate.branch)
11642 && git::branch_exists(&state.repo, &candidate.branch).await?
11643 {
11644 heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
11645 }
11646 }
11647 let mut cargo_guards = Vec::new();
11648 let entries = match std::fs::read_dir(state.worktree_root()) {
11649 Ok(entries) => Some(entries),
11650 Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
11651 Err(error) => return Err(error.into()),
11652 };
11653 for entry in entries.into_iter().flatten() {
11654 let path = entry?.path();
11655 let metadata = std::fs::symlink_metadata(&path)?;
11656 anyhow::ensure!(
11657 metadata.is_dir() && !metadata.file_type().is_symlink(),
11658 "non-directory or symlink recovery source retained: {}",
11659 path.display()
11660 );
11661 if !drop_winner
11662 && winner.is_some_and(|w| {
11663 path.file_name()
11664 .is_some_and(|name| name == format!("cand-{w}").as_str())
11665 })
11666 {
11667 continue;
11668 }
11669 anyhow::ensure!(
11670 path.join(".git").is_file(),
11671 "unregistered recovery source retained: {}",
11672 path.display()
11673 );
11674 anyhow::ensure!(
11675 git::git(
11676 &path,
11677 &[
11678 "status",
11679 "--porcelain",
11680 "--untracked-files=all",
11681 "--ignored",
11682 "--",
11683 ".",
11684 ":(exclude)target"
11685 ]
11686 )
11687 .await?
11688 .is_empty(),
11689 "source changes or ignored recovery files retained: {}",
11690 path.display()
11691 );
11692 match std::fs::symlink_metadata(path.join("target")) {
11693 Ok(_) => {
11694 cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
11695 }
11696 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
11697 Err(error) => return Err(error.into()),
11698 }
11699 for checkpoint in [
11700 "rebase-merge",
11701 "rebase-apply",
11702 "sequencer",
11703 "CHERRY_PICK_HEAD",
11704 "MERGE_HEAD",
11705 "REBASE_HEAD",
11706 "REVERT_HEAD",
11707 "BISECT_LOG",
11708 "AUTO_MERGE",
11709 "MERGE_MSG",
11710 ] {
11711 let location = git::git(
11712 &path,
11713 &[
11714 "rev-parse",
11715 "--path-format=absolute",
11716 "--git-path",
11717 checkpoint,
11718 ],
11719 )
11720 .await?;
11721 anyhow::ensure!(
11722 matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
11723 if e.kind() == std::io::ErrorKind::NotFound),
11724 "Git recovery checkpoint retained: {checkpoint}"
11725 );
11726 }
11727 heads.insert(git::rev_parse(&path, "HEAD").await?);
11728 }
11729 crate::fold_evidence::preserve(state, home, &heads).await?;
11730
11731 Ok(cargo_guards)
11732}
11733
11734fn remove_if_empty(dir: &Path) {
11745 if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
11746 std::fs::remove_dir(dir).ok();
11747 }
11748}
11749
11750pub fn worst_open(state: &RunState) -> Option<Severity> {
11752 state
11753 .reviews
11754 .last()?
11755 .reviews
11756 .iter()
11757 .flat_map(|r| r.findings.iter())
11758 .map(|f| f.severity)
11759 .max()
11760}
11761
11762async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
11766 if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
11767 git::acquire_worktree_config(repo).await?;
11768 state.enabled_worktree_config = true;
11769 }
11770 Ok(())
11771}
11772
11773#[cfg(test)]
11774mod tests {
11775 #[tokio::test]
11776 async fn launcher_base_fetch_requires_remote_success() {
11777 let tmp = tempfile::tempdir().unwrap();
11778 let repo = tmp.path().join("repo");
11779 let remote = tmp.path().join("remote");
11780 std::fs::create_dir(&repo).unwrap();
11781 std::fs::create_dir(&remote).unwrap();
11782 git::git(&remote, &["init", "--bare", "-b", "main"])
11783 .await
11784 .unwrap();
11785 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
11786 git::git(
11787 &repo,
11788 &[
11789 "-c",
11790 "user.name=Fixture",
11791 "-c",
11792 "user.email=fixture@example.invalid",
11793 "commit",
11794 "--allow-empty",
11795 "-m",
11796 "fixture",
11797 ],
11798 )
11799 .await
11800 .unwrap();
11801 git::git(
11802 &repo,
11803 &["remote", "add", "origin", remote.to_str().unwrap()],
11804 )
11805 .await
11806 .unwrap();
11807 git::git(&repo, &["push", "origin", "main"]).await.unwrap();
11808 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
11809 assert_eq!(
11810 resolve_base(&repo, "main", "origin", None)
11811 .await
11812 .unwrap()
11813 .commit,
11814 head
11815 );
11816 #[cfg(unix)]
11817 {
11818 let policy: crate::config::Coordination = toml::from_str(
11819 "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
11820 ).unwrap();
11821 std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
11822 std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
11823 let held = crate::repo_lock::SeatLease::operation_with(
11824 &repo,
11825 Some(&policy),
11826 &crate::repo_lock::WaitPolicy::default(),
11827 )
11828 .await
11829 .unwrap();
11830 assert!(held.is_some());
11831 let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
11832 tokio::pin!(fetch);
11833 assert!(
11834 tokio::time::timeout(Duration::from_millis(50), &mut fetch)
11835 .await
11836 .is_err()
11837 );
11838 assert!(
11839 !repo.join(".git/FETCH_HEAD").exists(),
11840 "no fetch before admission"
11841 );
11842 drop(held);
11843 let fetched = fetch.await.unwrap();
11844 assert_eq!(fetched.commit, head);
11845 assert_eq!(fetched.remote, "origin");
11846 assert_eq!(fetched.branch, "main");
11847 }
11848 git::git(
11850 &repo,
11851 &[
11852 "remote",
11853 "set-url",
11854 "origin",
11855 tmp.path().join("missing").to_str().unwrap(),
11856 ],
11857 )
11858 .await
11859 .unwrap();
11860 let error = resolve_base(&repo, "main", "origin", None)
11861 .await
11862 .unwrap_err()
11863 .to_string();
11864 assert!(error.contains("cannot read origin/main"));
11865 assert!(error.contains("refusing to branch off"));
11866 }
11867
11868 #[test]
11869 fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
11870 let leaked = vec!["chore(deps): update a crate".to_owned()];
11871 let own = "fix(daemon): apply a chosen action";
11872 assert!(should_retitle("", own, &leaked));
11873 assert!(should_retitle(
11874 &format!("{REVIEW_PROMPT_OPENING} `x`"),
11875 own,
11876 &leaked
11877 ));
11878 assert!(should_retitle(
11879 "chore: land candidate A of run 1",
11880 own,
11881 &leaked
11882 ));
11883 assert!(should_retitle(
11884 "magi: candidate A (uncommitted work)",
11885 own,
11886 &leaked
11887 ));
11888 assert!(should_retitle("chore(deps): update a crate", own, &leaked));
11889 assert!(!should_retitle("feat: renamed by hand", own, &leaked));
11890 assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
11891 }
11892
11893 #[test]
11894 fn pr_merge_plan_creates_adopts_or_stops() {
11895 assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
11896 assert_eq!(
11897 pr_merge_plan(Ok(land::OpenPr::One {
11898 url: "u".into(),
11899 title: "t".into()
11900 })),
11901 PrPlan::Adopt {
11902 url: "u".into(),
11903 title: "t".into()
11904 }
11905 );
11906 let PrPlan::Stop(many) =
11907 pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
11908 else {
11909 panic!("many must stop");
11910 };
11911 assert!(many.contains('a') && many.contains('b'));
11912 let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
11913 panic!("a failed lookup must stop");
11914 };
11915 assert!(err.contains("bad token"));
11916 }
11917
11918 use super::*;
11919 use crate::run::GateStatus;
11920 use std::collections::BTreeMap;
11921 use std::time::Duration;
11922
11923 fn conductor() -> AgentSpec {
11924 AgentSpec {
11925 id: "conductor".to_owned(),
11926 kind: crate::config::AgentKind::Command,
11927 model: None,
11928 command: vec!["true".to_owned()],
11929 extra_args: Vec::new(),
11930 env: BTreeMap::new(),
11931 prompt_delivery: None,
11932 max_turn_secs: None,
11933 }
11934 }
11935
11936 fn spec(id: &str) -> AgentSpec {
11937 AgentSpec {
11938 id: id.to_owned(),
11939 kind: crate::config::AgentKind::Command,
11940 model: None,
11941 command: vec!["true".to_owned()],
11942 extra_args: Vec::new(),
11943 env: BTreeMap::new(),
11944 prompt_delivery: None,
11945 max_turn_secs: None,
11946 }
11947 }
11948
11949 fn ids(xs: &[&str]) -> BTreeSet<String> {
11950 xs.iter().map(|s| (*s).to_owned()).collect()
11951 }
11952
11953 #[test]
11954 fn pick_successor_skips_an_agent_another_seat_holds() {
11955 let roster = [spec("a"), spec("b"), spec("c")];
11957 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11958 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11959 }
11960
11961 #[test]
11962 fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
11963 let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
11965 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11966 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
11967 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
11968 assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
11969 }
11970
11971 #[test]
11972 fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
11973 let roster = [spec("a"), spec("b")];
11974 let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
11975 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11976 let carried = ids(&["b"]);
11977 let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
11978 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11979 }
11980
11981 #[test]
11982 fn pick_successor_returns_none_without_any_untried_successor() {
11983 let roster = [spec("a"), spec("b")];
11984 assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
11985 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
11986 let carried = ids(&["a"]);
11987 assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
11988 }
11989
11990 #[test]
11991 fn pick_successor_rescue_avoids_another_seats_occupant() {
11992 let roster = [spec("a"), spec("b"), spec("c")];
11994 let carried = ids(&["b", "c"]);
11995 let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
11996 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
11997 }
11998
11999 #[test]
12000 fn next_for_seat_prefers_an_agent_that_has_not_failed() {
12001 let roster = [spec("a"), spec("b"), spec("c")];
12002 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
12003 assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
12004 }
12005
12006 #[test]
12007 fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
12008 let roster = [spec("a"), spec("b"), spec("c")];
12009 let failed = ids(&["a", "b", "c"]);
12010 let mut tried = ids(&["b"]);
12012 let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12013 assert_eq!(first.id, "a");
12014 tried.insert(first.id.clone());
12015 let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
12016 assert_eq!(second.id, "c");
12017 tried.insert(second.id.clone());
12018 assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
12019 }
12020
12021 #[test]
12022 fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
12023 let roster = [spec("a"), spec("b")];
12024 let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
12025 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12026 }
12027
12028 #[test]
12029 fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
12030 let roster = [spec("a"), spec("b"), spec("c")];
12031 let h = SeatHistory {
12032 failed: ids(&["a"]),
12033 last_ok: Some("b".to_owned()),
12034 last_fail: None,
12035 };
12036 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
12037 let gone = SeatHistory {
12039 last_ok: Some("zzz".to_owned()),
12040 ..h.clone()
12041 };
12042 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
12043 let failed = SeatHistory {
12044 failed: ids(&["a", "b"]),
12045 last_ok: Some("b".to_owned()),
12046 last_fail: None,
12047 };
12048 assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
12049 }
12050
12051 #[test]
12052 fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
12053 let first = SeatState::new("review-1", "alpha", 7);
12054 let next = handover_seat("review-1", "gamma", 7);
12055 assert_ne!(first.claude_session, next.claude_session);
12056 }
12057
12058 #[test]
12059 fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
12060 let mut state = state_with_summary("x", "y");
12061 let a = handover_seat("review-1", "beta", state.next_seat_seed());
12062 let b = handover_seat("review-1", "beta", state.next_seat_seed());
12063 assert_ne!(a.claude_session, b.claude_session);
12064 }
12065
12066 #[test]
12067 fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
12068 let roster = [spec("a"), spec("b")];
12069 let h = SeatHistory {
12070 failed: ids(&["a", "b"]),
12071 ..SeatHistory::default()
12072 };
12073 assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
12074 assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
12075 assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
12076 }
12077
12078 #[test]
12085 fn failure_signature_ignores_numbers_and_paths() {
12086 assert_eq!(
12087 failure_signature("exited with Some(2) and no usable output"),
12088 failure_signature("exited with Some(137) and no usable output")
12089 );
12090 assert_eq!(
12091 failure_signature("process exited with no usable output (exit code 2)"),
12092 failure_signature("process exited with no usable output (exit code 137)")
12093 );
12094 for (old, new) in [
12095 (
12096 "exited with Some(2) and no usable output",
12097 "process exited with no usable output (exit code 2)",
12098 ),
12099 (
12100 "exited with None and no usable output",
12101 "process exited with no usable output (exit code unavailable)",
12102 ),
12103 ] {
12104 let previous = FailClass::Other(failure_signature(old));
12105 let current = FailClass::Other(failure_signature(new));
12106 assert_eq!(previous, current, "resumption retains the failure class");
12107 assert!(!should_hand_over(Some(&previous), ¤t));
12108 }
12109 assert_eq!(
12110 failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
12111 failure_signature("cannot open /var/x.txt: denied")
12112 );
12113 assert_ne!(failure_signature("boom"), failure_signature("bang"));
12114 }
12115
12116 #[test]
12117 fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
12118 let other = FailClass::Other("x".into());
12119 assert!(should_hand_over(None, &FailClass::Quota));
12120 assert!(should_hand_over(Some(&other), &FailClass::Quota));
12121 assert!(should_hand_over(
12122 Some(&FailClass::Timeout),
12123 &FailClass::Timeout
12124 ));
12125 assert!(should_hand_over(None, &other));
12126 assert!(!should_hand_over(Some(&other), &other));
12127 assert!(should_hand_over(
12128 Some(&other),
12129 &FailClass::Other("y".into())
12130 ));
12131 assert!(should_hand_over(Some(&FailClass::Timeout), &other));
12133 assert!(should_hand_over(Some(&FailClass::Quota), &other));
12134 }
12135
12136 #[test]
12137 fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
12138 let a = SeatState::new("judge-1", "alpha", 7);
12139 let b = handover_seat("judge-1", "beta", 7);
12140 assert_ne!(a.claude_session, b.claude_session);
12141 assert_eq!(b.turns, 0);
12142 }
12143
12144 #[test]
12145 fn a_timeout_is_classified_apart_from_other_failures() {
12146 assert_eq!(
12147 FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
12148 Some(FailClass::Timeout)
12149 );
12150 assert!(matches!(
12151 FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
12152 Some(FailClass::Other(_))
12153 ));
12154 }
12155
12156 #[test]
12157 fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
12158 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12159 let tried = BTreeSet::from(["beta".to_owned()]);
12160 let next = next_untried_in_roster(&roster, 1, &tried);
12163 assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
12164 }
12165
12166 #[test]
12167 fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
12168 let roster = vec![spec("alpha"), spec("beta")];
12169 let tried = BTreeSet::from(["beta".to_owned()]);
12170 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12174 }
12175
12176 #[test]
12177 fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
12178 let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
12179 let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
12180 assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
12184 }
12185
12186 #[test]
12187 fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
12188 let roster = vec![spec("a"), spec("a"), spec("b")];
12189 let tried = BTreeSet::from(["a".to_owned()]);
12190 let next = next_untried_in_roster(&roster, 0, &tried);
12191 assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
12192 }
12193
12194 #[test]
12195 fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
12196 let roster = vec![spec("a"), spec("b")];
12197 let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
12198 assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
12199 }
12200
12201 #[test]
12202 fn remove_if_empty_only_ever_takes_a_bare_directory() {
12203 let dir = tempfile::tempdir().unwrap();
12204 let bay = dir.path().join("ffff");
12205
12206 remove_if_empty(&bay);
12208 assert!(!bay.exists());
12209
12210 std::fs::create_dir_all(bay.join("cand-A")).unwrap();
12213 remove_if_empty(&bay);
12214 assert!(bay.exists(), "non-empty directory must survive");
12215
12216 std::fs::remove_dir(bay.join("cand-A")).unwrap();
12218 remove_if_empty(&bay);
12219 assert!(!bay.exists(), "an empty bay is a leftover, not a record");
12220 }
12221
12222 #[test]
12231 fn a_full_panel_that_found_nothing_is_clean() {
12232 assert!(round_is_clean(
12233 0,
12234 true,
12235 2,
12236 2,
12237 0,
12238 IncompleteReviewPolicy::Block
12239 ));
12240 }
12241
12242 #[test]
12243 fn a_missing_seat_is_never_clean_under_the_default_policy() {
12244 assert!(!round_is_clean(
12245 0,
12246 true,
12247 1,
12248 2,
12249 0,
12250 IncompleteReviewPolicy::Block
12251 ));
12252 }
12253
12254 #[test]
12255 fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
12256 assert!(!round_is_clean(
12257 1,
12258 true,
12259 1,
12260 2,
12261 0,
12262 IncompleteReviewPolicy::Warn
12263 ));
12264 }
12265
12266 #[test]
12267 fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
12268 assert!(round_is_clean(
12269 0,
12270 true,
12271 1,
12272 2,
12273 0,
12274 IncompleteReviewPolicy::Warn
12275 ));
12276 }
12277
12278 #[test]
12279 fn a_full_panel_with_an_open_finding_is_not_clean() {
12280 assert!(!round_is_clean(
12281 1,
12282 true,
12283 2,
12284 2,
12285 0,
12286 IncompleteReviewPolicy::Block
12287 ));
12288 }
12289
12290 #[test]
12291 fn a_full_panel_with_a_red_e2e_is_not_clean() {
12292 assert!(!round_is_clean(
12293 0,
12294 false,
12295 2,
12296 2,
12297 0,
12298 IncompleteReviewPolicy::Block
12299 ));
12300 }
12301
12302 #[test]
12309 fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
12310 assert!(round_is_clean(
12313 0,
12314 true,
12315 1,
12316 2,
12317 1,
12318 IncompleteReviewPolicy::Block
12319 ));
12320 }
12321
12322 #[test]
12323 fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
12324 assert!(!round_is_clean(
12327 0,
12328 true,
12329 1,
12330 2,
12331 0,
12332 IncompleteReviewPolicy::Block
12333 ));
12334 }
12335
12336 #[test]
12337 fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
12338 assert!(!round_is_clean(
12339 1,
12340 true,
12341 1,
12342 2,
12343 1,
12344 IncompleteReviewPolicy::Block
12345 ));
12346 assert!(!round_is_clean(
12347 0,
12348 false,
12349 1,
12350 2,
12351 1,
12352 IncompleteReviewPolicy::Block
12353 ));
12354 }
12355
12356 #[test]
12357 fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
12358 assert!(!round_is_clean(
12362 0,
12363 true,
12364 0,
12365 2,
12366 2,
12367 IncompleteReviewPolicy::Block
12368 ));
12369 }
12370
12371 fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
12372 CommandOutcome {
12373 command: "test".to_owned(),
12374 code,
12375 output_tail: String::new(),
12376 duration_ms: 0,
12377 resource_blocked,
12378 }
12379 }
12380
12381 #[test]
12382 fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
12383 assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
12384 assert!(
12385 !verify_inconclusive(&[outcome(Some(1), false)]),
12386 "an ordinary failure is still evidence about the patch"
12387 );
12388 assert!(verify_inconclusive(&[outcome(None, true)]));
12389 assert!(
12390 verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
12391 "one inconclusive outcome taints the whole batch"
12392 );
12393 assert!(!verify_inconclusive(&[]));
12394 }
12395
12396 #[tokio::test]
12397 async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
12398 let calls = std::sync::atomic::AtomicUsize::new(0);
12402 let started = Instant::now();
12403 wait_for_pids_with(
12404 &[123],
12405 |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
12406 Duration::from_millis(5),
12407 Duration::from_secs(5),
12408 )
12409 .await;
12410 assert!(
12411 calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
12412 "must keep checking rather than deciding on the first answer"
12413 );
12414 assert!(
12415 started.elapsed() < Duration::from_secs(1),
12416 "must return the moment it is confirmed dead, not wait out the ceiling"
12417 );
12418 }
12419
12420 #[tokio::test]
12421 async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
12422 let started = Instant::now();
12423 wait_for_pids_with(
12424 &[123],
12425 |_| true, Duration::from_millis(5),
12427 Duration::from_millis(30),
12428 )
12429 .await;
12430 let elapsed = started.elapsed();
12431 assert!(
12432 elapsed >= Duration::from_millis(30),
12433 "must not give up before its own ceiling: {elapsed:?}"
12434 );
12435 assert!(
12436 elapsed < Duration::from_secs(1),
12437 "must not wait past its own ceiling either: {elapsed:?}"
12438 );
12439 }
12440
12441 #[tokio::test]
12442 async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
12443 let started = Instant::now();
12444 wait_for_pids_with(
12445 &[],
12446 |_| true,
12447 Duration::from_secs(5),
12448 Duration::from_secs(5),
12449 )
12450 .await;
12451 assert!(
12452 started.elapsed() < Duration::from_millis(200),
12453 "an empty pid list has nothing to confirm"
12454 );
12455 }
12456
12457 fn review_round(
12463 clean: bool,
12464 blocking: usize,
12465 answered: usize,
12466 expected: usize,
12467 progressed: bool,
12468 e2e_ok: bool,
12469 ) -> ReviewRound {
12470 ReviewRound {
12471 round: 1,
12472 head: "h".to_owned(),
12473 verified_head: None,
12474 verified_at: None,
12475 reviews: Vec::new(),
12476 e2e: vec![CommandOutcome {
12477 command: "test".to_owned(),
12478 code: Some(if e2e_ok { 0 } else { 1 }),
12479 output_tail: String::new(),
12480 duration_ms: 0,
12481 resource_blocked: false,
12482 }],
12483 verify_retried: false,
12484 e2e_deferred: false,
12485 e2e_defer_reason: None,
12486 fix: None,
12487 blocking,
12488 answered,
12489 expected,
12490 clean,
12491 progressed,
12492 vote_split: false,
12493 reconsideration: Vec::new(),
12494 verdict: None,
12495 }
12496 }
12497
12498 #[test]
12499 fn review_conclusion_is_none_when_nothing_has_run() {
12500 assert_eq!(review_conclusion(&[], 3), None);
12501 }
12502
12503 #[test]
12504 fn review_conclusion_is_none_while_rounds_remain() {
12505 let rounds = vec![review_round(false, 1, 2, 2, true, true)];
12506 assert_eq!(review_conclusion(&rounds, 3), None);
12507 }
12508
12509 #[test]
12510 fn review_conclusion_is_gating_once_a_round_is_clean() {
12511 let rounds = vec![review_round(true, 0, 2, 2, false, true)];
12512 assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
12513 }
12514
12515 #[test]
12516 fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
12517 let rounds = vec![
12518 review_round(false, 1, 2, 2, true, true),
12519 review_round(false, 1, 2, 2, true, true),
12520 ];
12521 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
12522 }
12523
12524 #[test]
12525 fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
12526 let rounds = vec![
12527 review_round(false, 1, 2, 2, true, true),
12528 review_round(false, 1, 2, 2, true, false),
12529 ];
12530 assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
12531 }
12532
12533 #[test]
12534 fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
12535 let mut blocked = review_round(false, 1, 2, 2, true, false);
12542 blocked.e2e[0].resource_blocked = true;
12543 let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
12544 assert_eq!(review_conclusion(&rounds, 2), None);
12545 }
12546
12547 #[test]
12548 fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
12549 let rounds = vec![review_round(false, 0, 1, 2, false, true)];
12551 assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
12552 }
12553
12554 #[test]
12555 fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
12556 let rounds = vec![
12557 review_round(false, 1, 2, 2, false, true),
12558 review_round(false, 1, 2, 2, false, true),
12559 ];
12560 assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
12561 }
12562
12563 fn secs(n: u64) -> Duration {
12564 Duration::from_secs(n)
12565 }
12566
12567 #[tokio::test]
12570 async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
12571 ask_test_home();
12572 let tmp = tempfile::tempdir().unwrap();
12573 let repo = tmp.path().join("repo");
12574 std::fs::create_dir_all(&repo).unwrap();
12575 init_repo(&repo);
12576 let g = |args: &[&str]| {
12577 let out = std::process::Command::new("git")
12578 .args(args)
12579 .current_dir(&repo)
12580 .quiet()
12581 .output()
12582 .unwrap();
12583 assert!(
12584 out.status.success(),
12585 "{}",
12586 String::from_utf8_lossy(&out.stderr)
12587 );
12588 };
12589 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
12590 g(&["checkout", "-b", "historical"]);
12591 std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
12592 g(&["add", "."]);
12593 g(&["commit", "-m", "old alternative"]);
12594 let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
12595 g(&["checkout", "-b", "magi/task-selected", &base]);
12596 std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
12597 g(&["add", "."]);
12598 g(&["commit", "-m", "measurement infrastructure"]);
12599 let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
12600 std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
12601 g(&["add", "."]);
12602 g(&["commit", "-m", "dependent implementation"]);
12603 let head = git::rev_parse(&repo, "HEAD").await.unwrap();
12604 g(&["checkout", "main"]);
12605 std::fs::write(repo.join("unrelated"), "current main change").unwrap();
12606 g(&["add", "."]);
12607 g(&["commit", "-m", "unrelated base change"]);
12608 let current = git::rev_parse(&repo, "HEAD").await.unwrap();
12609 let origin = tmp.path().join("origin.git");
12610 g(&[
12611 "clone",
12612 "--bare",
12613 repo.to_str().unwrap(),
12614 origin.to_str().unwrap(),
12615 ]);
12616 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
12617 g(&["fetch", "origin"]);
12618 let selection = crate::pr_recovery::Selection {
12619 url: "https://example.test/project/pull/1".into(),
12620 branch: "magi/task-selected".into(),
12621 head: head.clone(),
12622 source_head: None,
12623 source_run: None,
12624 source_proof: None,
12625 disposition: Some(crate::pr_recovery::Disposition::Review),
12626 instruction: "preserve measured behavior".into(),
12627 approved_at: jiff::Timestamp::now(),
12628 };
12629 let mut template = runner_at(RunStatus::Reviewing);
12630 template.state.repo = repo.clone();
12631 template.state.base_commit = current.clone();
12632 template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
12633 template.state.origin = Some(crate::run::Origin::queue("selected"));
12634 template.state.pr_head = Some(selection.branch.clone());
12635 template.state.pr_recovery = Some(selection.clone());
12636 let mut runner =
12637 Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
12638 .await
12639 .unwrap();
12640 let wt = runner.state.winner().unwrap().worktree.clone();
12641 assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
12642 assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
12643 assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
12644 assert!(runner.state.seeds.is_empty());
12645 assert_eq!(runner.state.instruction, "preserve measured behavior");
12646 runner.sync_to_base().await.unwrap();
12647 assert_eq!(
12648 std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
12649 "prerequisite"
12650 );
12651 assert_eq!(
12652 std::fs::read_to_string(wt.join("implementation")).unwrap(),
12653 "measured implementation"
12654 );
12655 assert_eq!(
12656 std::fs::read_to_string(wt.join("unrelated")).unwrap(),
12657 "current main change"
12658 );
12659 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12660 assert_eq!(
12661 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12662 head
12663 );
12664
12665 let mut partial = runner_at(RunStatus::Reviewing);
12667 partial.state.repo = repo.clone();
12668 partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
12669 partial.state.config.worktree.setup = vec![crate::config::SetupStep {
12670 run: Some("exit 1".into()),
12671 ..Default::default()
12672 }];
12673 let selection = runner.state.pr_recovery.clone().unwrap();
12674 partial.state.pr_recovery = Some(selection.clone());
12675 partial.state.pr_head = Some(selection.branch.clone());
12676 partial.state.origin = Some(crate::run::Origin::queue("selected"));
12677 let id = partial.state.id.clone();
12678 let error = match Runner::open_selected(
12679 &repo,
12680 partial.state,
12681 partial.roles,
12682 1,
12683 runner.state.base_commit.clone(),
12684 selection,
12685 )
12686 .await
12687 {
12688 Ok(_) => panic!("setup must fail"),
12689 Err(e) => e,
12690 };
12691 assert_eq!(
12692 error
12693 .downcast_ref::<crate::pr_recovery::StartupFailed>()
12694 .unwrap()
12695 .run,
12696 id
12697 );
12698 let failed = RunState::load(&id).unwrap();
12699 assert_eq!(failed.status, RunStatus::Blocked);
12700 assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
12701 assert!(Runner::resume(&id).is_err());
12702 runner.state.status = RunStatus::Ready;
12703 runner.state.driver_exited = true;
12704 runner.state.save().unwrap();
12705 let mut retry = runner_at(RunStatus::Reviewing);
12706 retry.state.repo = repo.clone();
12707 retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
12708 let mut reselected = runner.state.pr_recovery.clone().unwrap();
12709 reselected.approved_at = jiff::Timestamp::now();
12710 retry.state.origin = Some(crate::run::Origin::queue("selected"));
12711 retry.state.pr_head = Some(reselected.branch.clone());
12712 retry.state.pr_recovery = Some(reselected.clone());
12713 crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
12714 .await
12715 .unwrap();
12716 let restarted = Runner::open_selected(
12717 &repo,
12718 retry.state,
12719 retry.roles,
12720 1,
12721 runner.state.base_commit.clone(),
12722 reselected,
12723 )
12724 .await
12725 .unwrap();
12726 assert_eq!(
12727 git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
12728 .await
12729 .unwrap(),
12730 head
12731 );
12732 assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
12733 assert_eq!(
12734 git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
12735 head
12736 );
12737 }
12738
12739 #[tokio::test]
12740 #[cfg(unix)]
12741 async fn fallback_and_interrupted_restoration_require_coordination() {
12742 use std::os::fd::AsRawFd as _;
12743 ask_test_home();
12744 let tmp = tempfile::tempdir().unwrap();
12745 let mut runner = registered_runner(tmp.path()).await;
12746 runner.state.config.graph.timeout_resource_wait = 1;
12747 let winner = runner.state.candidates[0].clone();
12748 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
12749 let leftover = winner.worktree.join("leftover");
12750 std::fs::write(&leftover, "untrusted edits").unwrap();
12751 let next = tmp.path().join("next-fixer");
12752 let mut failed = spec("first");
12753 failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
12754 let mut successor = spec("second");
12755 successor.command = vec![
12756 "sh".into(),
12757 "-c".into(),
12758 format!("touch '{}'; echo done", next.display()),
12759 ];
12760 let attempts = vec![
12761 (failed.clone(), "first".into()),
12762 (successor.clone(), "second".into()),
12763 ];
12764 let lock_path = runner.state.repo.join(".git/updater.lock");
12765 let external = std::fs::File::create(&lock_path).unwrap();
12766 assert_eq!(
12767 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
12768 0
12769 );
12770 let artifacts = tmp.path().join("artifacts");
12771 let build = |spec, seat| SeatJob {
12772 spec,
12773 seat,
12774 cwd: winner.worktree.clone(),
12775 prompt: "fixture".into(),
12776 timeout: Duration::from_secs(10),
12777 allow_write: true,
12778 sessions: false,
12779 artifacts: artifacts.clone(),
12780 stem: "fix".into(),
12781 handover: None,
12782 };
12783 let (_, _, out) = runner
12784 .ask_fixer_among(
12785 attempts.clone(),
12786 "graft",
12787 None,
12788 Some((&winner.worktree, &head)),
12789 build,
12790 )
12791 .await;
12792 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
12793 assert!(leftover.exists(), "contention must prevent clean");
12794 assert!(!next.exists());
12795 assert!(runner.state.handovers.is_empty());
12796 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12798 from_run: "earlier".into(),
12799 generation: 1,
12800 mode: crate::recovery::Mode::VerifyRepair,
12801 reasons: Vec::new(),
12802 reviewed_head: None,
12803 rounds: Vec::new(),
12804 open_findings: Vec::new(),
12805 evidence: None,
12806 criteria_inherited: true,
12807 repair: crate::recovery::Repair::Started {
12808 before: head.clone(),
12809 },
12810 });
12811 assert!(
12812 runner
12813 .recovery_repair(&winner)
12814 .await
12815 .unwrap_err()
12816 .is::<crate::repo_lock::ResourceBlocked>()
12817 );
12818 assert!(matches!(
12819 runner.state.recovery.as_ref().unwrap().repair,
12820 crate::recovery::Repair::Started { .. }
12821 ));
12822 assert!(leftover.exists());
12823 drop(external);
12824 runner.recovery_repair(&winner).await.unwrap();
12825 assert!(!leftover.exists());
12826 assert!(matches!(
12827 runner.state.recovery.as_ref().unwrap().repair,
12828 crate::recovery::Repair::Done { .. }
12829 ));
12830 std::fs::remove_file(&lock_path).unwrap();
12832 std::fs::create_dir(&lock_path).unwrap();
12833 std::fs::write(&leftover, "untrusted edits").unwrap();
12834 let (_, _, out) = runner
12835 .ask_fixer_among(
12836 attempts,
12837 "graft",
12838 None,
12839 Some((&winner.worktree, &head)),
12840 build,
12841 )
12842 .await;
12843 assert!(matches!(out, AgentOutcome::Failed(_)));
12844 assert!(leftover.exists());
12845 assert!(!next.exists());
12846 assert_eq!(
12847 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
12848 head
12849 );
12850 }
12851
12852 #[cfg(unix)]
12853 async fn registered_runner(root: &Path) -> Runner {
12854 ask_test_home();
12855 let repo = root.join("repo");
12856 std::fs::create_dir_all(&repo).unwrap();
12857 init_repo(&repo);
12858 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
12859 let mut runner = runner_at(RunStatus::Prep);
12860 runner.state.repo = repo.clone();
12861 runner.state.config.coordination = crate::config::test_coordination();
12862 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
12863 runner.state.config.graph.worktree_root = Some(root.join("wt"));
12864 runner.state.config.graph.advise = false;
12865 runner.roles.implementers = vec![spec("implement")];
12866 runner.roles.judges = vec![spec("judge")];
12867 runner.prep().await.unwrap();
12868 runner
12869 }
12870
12871 #[tokio::test]
12872 #[cfg(unix)]
12873 async fn parked_implementer_persists_completed_work() {
12874 for registered in [false, true] {
12875 let tmp = tempfile::tempdir().unwrap();
12876 let mut runner = registered_runner(tmp.path()).await;
12877 if !registered {
12878 std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
12879 }
12880 let marker = tmp.path().join("started");
12881 let release = tmp.path().join("release");
12882 let mut implementer = spec("implement");
12883 implementer.command = vec![
12884 "sh".into(),
12885 "-c".into(),
12886 format!(
12887 "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
12888 marker.display(),
12889 release.display()
12890 ),
12891 ];
12892 runner.roles.implementers = vec![implementer.clone()];
12893 runner.state.config.agents = vec![implementer];
12894 let pause = runner.pause.clone();
12895 let (result, ()) = tokio::join!(runner.implement(), async {
12896 tokio::time::timeout(Duration::from_secs(10), async {
12897 while !marker.exists() {
12898 tokio::time::sleep(Duration::from_millis(10)).await;
12899 }
12900 })
12901 .await
12902 .unwrap();
12903 pause.park();
12904 std::fs::write(&release, "go").unwrap();
12905 });
12906 result.unwrap();
12907 let candidate = &runner.state.candidates[0];
12908 assert!(candidate.commits > 0, "{candidate:?}");
12909 assert!(candidate.summary.contains("saved-result"));
12910 assert_eq!(candidate.files, 1);
12911 let before = candidate.commits;
12912 runner.implement().await.unwrap();
12913 assert_eq!(runner.state.candidates[0].commits, before);
12914 }
12915 }
12916
12917 #[tokio::test]
12918 #[cfg(unix)]
12919 async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
12920 let tmp = tempfile::tempdir().unwrap();
12921 let mut runner = registered_runner(tmp.path()).await;
12922 let winner = runner.state.candidates[0].clone();
12923 let cache = tmp.path().join("cache");
12924 runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
12925 runner.state.config.graph.timeout_resource_wait = 0;
12926 let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
12927 let crate::cache::AcquireOutcome::Acquired(held) =
12928 crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
12929 else {
12930 panic!("cache must be free")
12931 };
12932 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
12933 from_run: "earlier".into(),
12934 generation: 1,
12935 mode: crate::recovery::Mode::VerifyRepair,
12936 reasons: Vec::new(),
12937 reviewed_head: None,
12938 rounds: Vec::new(),
12939 open_findings: Vec::new(),
12940 evidence: None,
12941 criteria_inherited: true,
12942 repair: crate::recovery::Repair::NotStarted,
12943 });
12944 runner.recovery_repair(&winner).await.unwrap();
12945 assert!(matches!(
12946 runner.state.recovery.as_ref().unwrap().repair,
12947 crate::recovery::Repair::WaitingForRestore { .. }
12948 ));
12949 drop(held);
12950 runner.state.resource_block = None;
12951 runner.state.config.graph.timeout_resource_wait = 1;
12952 runner.recovery_repair(&winner).await.unwrap();
12953 assert!(matches!(
12954 runner.state.recovery.as_ref().unwrap().repair,
12955 crate::recovery::Repair::Done { .. }
12956 ));
12957 assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
12958 }
12959
12960 #[tokio::test]
12961 #[cfg(unix)]
12962 async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
12963 use std::os::fd::AsRawFd as _;
12964 for graft in [false, true] {
12965 let tmp = tempfile::tempdir().unwrap();
12966 let mut runner = registered_runner(tmp.path()).await;
12967 runner.state.config.graph.timeout_resource_wait = 1;
12968 let winner = runner.state.candidates[0].clone();
12969 let marker = tmp.path().join("fixer-started");
12970 let mut fixer = spec("fixer");
12971 fixer.command = vec![
12972 "sh".into(),
12973 "-c".into(),
12974 format!("echo started >> '{}'; echo done", marker.display()),
12975 ];
12976 runner.roles.fixer = Some(vec![fixer]);
12977 if graft {
12978 let mut loser = winner.clone();
12979 loser.label = 'B';
12980 loser.index = 1;
12981 runner.state.candidates.push(loser);
12982 runner.state.config.graph.graft_runner_up = true;
12983 runner.state.tally = Some(Tally {
12984 first_choice: BTreeMap::new(),
12985 borda: BTreeMap::new(),
12986 winner: winner.label,
12987 rankings: 1,
12988 unanimous_initial: true,
12989 deliberated: false,
12990 changed_votes: 0,
12991 unanimous_final: true,
12992 tie_break: None,
12993 judges: 1,
12994 present: 1,
12995 quorum: 1,
12996 met_quorum: true,
12997 uncontested: None,
12998 });
12999 runner.state.judgements.push(crate::run::Judgement {
13000 judge: 1,
13001 seat: "judge-1".into(),
13002 agent: "judge".into(),
13003 ranking: vec![winner.label, 'B'],
13004 reasons: BTreeMap::new(),
13005 confidence: None,
13006 strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
13007 order: vec![0, 1],
13008 failed: None,
13009 duration_ms: 0,
13010 });
13011 } else {
13012 runner.state.recovery = Some(crate::recovery::RecoveryRecord {
13013 from_run: "earlier".into(),
13014 generation: 1,
13015 mode: crate::recovery::Mode::VerifyRepair,
13016 reasons: Vec::new(),
13017 reviewed_head: None,
13018 rounds: Vec::new(),
13019 open_findings: Vec::new(),
13020 evidence: None,
13021 criteria_inherited: true,
13022 repair: crate::recovery::Repair::NotStarted,
13023 });
13024 }
13025 let reader = crate::repo_lock::SeatLease::for_checkout(
13026 &winner.worktree,
13027 false,
13028 crate::config::test_coordination().as_ref(),
13029 )
13030 .await
13031 .unwrap();
13032 let pause = runner.pause.clone();
13033 let (result, ()) = tokio::join!(
13034 async {
13035 if graft {
13036 runner.graft().await
13037 } else {
13038 runner.recovery_repair(&winner).await
13039 }
13040 },
13041 async {
13042 tokio::time::sleep(Duration::from_millis(100)).await;
13043 pause.park();
13044 }
13045 );
13046 result.unwrap();
13047 assert!(!marker.exists(), "a blocked seat must not spawn");
13048 runner.state = RunState::load(&runner.state.id).unwrap();
13050 if graft {
13051 assert_eq!(
13052 runner.state.graft.as_ref().unwrap().outcome,
13053 GraftOutcome::WaitingForRestore
13054 );
13055 } else {
13056 assert!(matches!(
13057 runner.state.recovery.as_ref().unwrap().repair,
13058 crate::recovery::Repair::WaitingForRestore { .. }
13059 ));
13060 }
13061 drop(reader);
13062 runner.pause = Pause::new();
13063 runner.state.resource_block = None;
13064 let external =
13066 std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13067 assert_eq!(
13068 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13069 0
13070 );
13071 let leftover = winner.worktree.join("leftover");
13072 std::fs::write(&leftover, "earlier fallback edits").unwrap();
13073 let result = if graft {
13074 runner.graft().await
13075 } else {
13076 runner.recovery_repair(&winner).await
13077 };
13078 assert!(
13079 result
13080 .unwrap_err()
13081 .is::<crate::repo_lock::ResourceBlocked>()
13082 );
13083 assert!(leftover.exists());
13084 assert!(!marker.exists());
13085 drop(external);
13086 runner.state.resource_block = None;
13087 let result = if graft {
13088 runner.graft().await
13089 } else {
13090 runner.recovery_repair(&winner).await
13091 };
13092 result.unwrap();
13093 assert!(!leftover.exists(), "restore must precede retry");
13094 assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
13095 if graft {
13096 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13097 } else {
13098 assert!(matches!(
13099 runner.state.recovery.as_ref().unwrap().repair,
13100 crate::recovery::Repair::Done { .. }
13101 ));
13102 }
13103 }
13104 }
13105
13106 #[tokio::test]
13107 #[cfg(unix)]
13108 async fn interrupted_graft_restore_waits_and_preserves_its_record() {
13109 use std::os::fd::AsRawFd as _;
13110 let tmp = tempfile::tempdir().unwrap();
13111 let mut runner = registered_runner(tmp.path()).await;
13112 runner.state.config.graph.timeout_resource_wait = 1;
13113 let winner = runner.state.candidates[0].clone();
13114 let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13115 runner.state.tally = Some(Tally {
13116 first_choice: BTreeMap::new(),
13117 borda: BTreeMap::new(),
13118 winner: winner.label,
13119 rankings: 0,
13120 unanimous_initial: false,
13121 deliberated: false,
13122 changed_votes: 0,
13123 unanimous_final: false,
13124 tie_break: None,
13125 judges: 0,
13126 present: 0,
13127 quorum: 0,
13128 met_quorum: true,
13129 uncontested: Some("fixture".into()),
13130 });
13131 runner.state.graft = Some(GraftRecord {
13132 pre: pre.clone(),
13133 post: None,
13134 lines: None,
13135 limit: None,
13136 agent: None,
13137 outcome: GraftOutcome::Running,
13138 requested: Vec::new(),
13139 adoption: None,
13140 adoption_error: None,
13141 });
13142 std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
13143 git::commit_all(&winner.worktree, "unfinished graft")
13144 .await
13145 .unwrap();
13146 let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13147 let leftover = winner.worktree.join("leftover");
13148 std::fs::write(&leftover, "untrusted edits").unwrap();
13149 let path = runner.state.repo.join(".git/updater.lock");
13150 let external = std::fs::File::create(&path).unwrap();
13151 assert_eq!(
13152 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13153 0
13154 );
13155 assert!(
13156 runner
13157 .graft()
13158 .await
13159 .unwrap_err()
13160 .is::<crate::repo_lock::ResourceBlocked>()
13161 );
13162 assert!(matches!(
13163 runner.state.graft.as_ref().unwrap().outcome,
13164 GraftOutcome::Running
13165 ));
13166 assert_eq!(
13167 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13168 unfinished
13169 );
13170 assert!(leftover.exists());
13171 drop(external);
13172 std::fs::remove_file(&path).unwrap();
13173 std::fs::create_dir(&path).unwrap();
13174 assert!(runner.graft().await.is_err());
13175 assert!(matches!(
13176 runner.state.graft.as_ref().unwrap().outcome,
13177 GraftOutcome::Running
13178 ));
13179 assert_eq!(
13180 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13181 unfinished
13182 );
13183 assert!(leftover.exists());
13184 std::fs::remove_dir(&path).unwrap();
13185 runner.graft().await.unwrap();
13186 assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
13187 assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
13188 assert!(!leftover.exists());
13189 }
13190
13191 #[tokio::test]
13192 #[cfg(unix)]
13193 async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
13194 use std::os::fd::AsRawFd as _;
13195 let tmp = tempfile::tempdir().unwrap();
13196 let mut runner = registered_runner(tmp.path()).await;
13197 runner.state.prep_complete = Some(false);
13198 runner.state.config.graph.timeout_resource_wait = 1;
13199 runner.state.save().unwrap();
13200 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13201 assert_eq!(
13202 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13203 0
13204 );
13205 runner.execute().await.unwrap();
13206 let disk = RunState::load(&runner.state.id).unwrap();
13207 assert!(disk.driver_exited);
13208 assert_eq!(disk.driver_pid, None);
13209 assert_eq!(runner.state.status, RunStatus::Blocked);
13210 assert_eq!(
13211 runner.state.resource_block.as_ref().unwrap().kind,
13212 "repository coordination"
13213 );
13214 assert!(runner.state.prep_failure.is_none());
13215 assert!(runner.state.seats.is_empty());
13216 runner.pause.park();
13217 tokio::time::timeout(Duration::from_secs(10), runner.execute())
13220 .await
13221 .unwrap()
13222 .unwrap();
13223 assert!(runner.state.parked);
13224 assert!(runner.state.prep_failure.is_none());
13225 let disk = RunState::load(&runner.state.id).unwrap();
13226 assert!(disk.driver_exited);
13227 assert_eq!(disk.driver_pid, None);
13228 }
13229
13230 #[tokio::test]
13231 #[cfg(unix)]
13232 async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
13233 use std::os::fd::AsRawFd as _;
13234 let tmp = tempfile::tempdir().unwrap();
13235 let mut runner = registered_runner(tmp.path()).await;
13236 runner.state.prep_complete = Some(false);
13237 runner.state.config.graph.timeout_resource_wait = 30;
13238 runner.state.save().unwrap();
13239 let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
13240 assert_eq!(
13241 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13242 0
13243 );
13244 assert!(
13245 tokio::time::timeout(Duration::from_millis(100), runner.execute())
13246 .await
13247 .is_err()
13248 );
13249 let disk = RunState::load(&runner.state.id).unwrap();
13250 assert!(disk.driver_exited);
13251 assert_eq!(disk.driver_pid, None);
13252 assert!(disk.status.resumable());
13253 assert!(runner.state.driver_exited);
13254 assert_eq!(runner.state.driver_pid, None);
13255 runner.state.save().unwrap();
13256 assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
13257 }
13258
13259 #[tokio::test]
13260 #[cfg(unix)]
13261 async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
13262 let tmp = tempfile::tempdir().unwrap();
13263 let mut runner = registered_runner(tmp.path()).await;
13264 let winner = runner.state.candidates[0].clone();
13265 let cache = tmp.path().join("cache");
13266 let started = tmp.path().join("pre-gate-started");
13267 runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
13268 runner.state.config.verify.gate =
13269 vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
13270 let home = crate::run::home();
13271 let owner =
13272 crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
13273 let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
13274 crate::cache::AcquireOutcome::Acquired(g) => g,
13275 crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
13276 };
13277 let mut gate = Box::pin(runner.run_gate(&winner));
13278 assert!(
13279 tokio::time::timeout(Duration::from_millis(150), &mut gate)
13280 .await
13281 .is_err()
13282 );
13283 assert!(!started.exists(), "pre-gate must wait for the cache too");
13284 let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
13288 let editor = crate::repo_lock::SeatLease::checkout_with(
13289 &winner.worktree,
13290 true,
13291 crate::config::test_coordination().as_ref(),
13292 &wait,
13293 )
13294 .await
13295 .unwrap();
13296 drop(editor);
13297 guard.release();
13298 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13299 .await
13300 .unwrap()
13301 .unwrap();
13302 drop(gate);
13303 assert!(outcomes.iter().all(CommandOutcome::ok));
13304 assert!(started.exists());
13305 }
13306
13307 #[tokio::test]
13308 #[cfg(unix)]
13309 async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
13310 use std::os::fd::AsRawFd as _;
13311
13312 ask_test_home();
13313 let tmp = tempfile::tempdir().unwrap();
13314 let repo = tmp.path().join("repo");
13315 std::fs::create_dir_all(&repo).unwrap();
13316 init_repo(&repo);
13317 let common = repo.join(".git");
13318 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13319 let mut runner = runner_at(RunStatus::Prep);
13320 runner.state.repo = repo.clone();
13321 runner.state.config.coordination = crate::config::test_coordination();
13322 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13323 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13324 runner.state.config.graph.advise = false;
13325 runner.roles.implementers = vec![spec("implement")];
13326 runner.roles.judges = vec![spec("judge")];
13327 runner.prep().await.unwrap();
13328 let winner = runner.state.candidates[0].clone();
13329 std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
13332 let completed = tmp.path().join("seat-completed");
13333 runner.roles.implementers[0].command = vec![
13334 "sh".into(),
13335 "-c".into(),
13336 format!("touch '{}'; echo completed", completed.display()),
13337 ];
13338 let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
13339 assert_eq!(
13340 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13341 0
13342 );
13343 let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13344 let mut implement = Box::pin(runner.implement());
13345 tokio::select! {
13346 result = &mut implement => panic!("rescue must wait: {result:?}"),
13347 result = tokio::time::timeout(Duration::from_secs(10), async {
13348 while !completed.exists() {
13349 tokio::time::sleep(Duration::from_millis(20)).await;
13350 }
13351 }) => result.unwrap(),
13352 }
13353 assert!(
13354 tokio::time::timeout(Duration::from_millis(200), &mut implement)
13355 .await
13356 .is_err()
13357 );
13358 assert_eq!(
13359 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13360 before
13361 );
13362 assert_eq!(
13363 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13364 0
13365 );
13366 tokio::time::timeout(Duration::from_secs(10), &mut implement)
13367 .await
13368 .unwrap()
13369 .unwrap();
13370 drop(implement);
13371 assert_ne!(
13372 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13373 before
13374 );
13375 assert!(git::is_clean(&winner.worktree).await.unwrap());
13376
13377 let started = tmp.path().join("pre-gate-started");
13378 let proceed = tmp.path().join("pre-gate-proceed");
13379 let checked = tmp.path().join("gate-ran");
13380 runner.state.config.verify.pre_gate = vec![format!(
13381 "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13382 started.display(),
13383 proceed.display()
13384 )];
13385 runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
13386 assert_eq!(
13387 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13388 0
13389 );
13390 let mut gate = Box::pin(runner.run_gate(&winner));
13391 assert!(
13392 tokio::time::timeout(Duration::from_millis(200), &mut gate)
13393 .await
13394 .is_err()
13395 );
13396 assert!(!started.exists());
13397 assert!(!checked.exists());
13398 assert!(git::is_clean(&winner.worktree).await.unwrap());
13399 assert_eq!(
13400 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
13401 0
13402 );
13403 tokio::select! {
13404 result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
13405 result = tokio::time::timeout(Duration::from_secs(10), async {
13406 while !started.exists() {
13407 tokio::time::sleep(Duration::from_millis(20)).await;
13408 }
13409 }) => result.unwrap(),
13410 }
13411 assert_eq!(
13412 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13413 -1,
13414 "mechanical edits must hold the real lock"
13415 );
13416 std::fs::write(proceed, "ready").unwrap();
13417 let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
13418 .await
13419 .unwrap()
13420 .unwrap();
13421 drop(gate);
13422 assert!(outcomes.iter().all(CommandOutcome::ok));
13423 assert!(checked.exists());
13424 assert!(runner.state.pre_gate_commit.is_some());
13425 assert!(git::is_clean(&winner.worktree).await.unwrap());
13426 assert_eq!(
13427 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13428 0,
13429 "gate completion must release coordination"
13430 );
13431 drop(outsider);
13432 std::fs::remove_file(common.join("updater.lock")).unwrap();
13433 std::fs::create_dir(common.join("updater.lock")).unwrap();
13434 std::fs::remove_file(&started).unwrap();
13435 std::fs::remove_file(&checked).unwrap();
13436 let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
13437 let error = runner.run_gate(&winner).await.unwrap_err();
13438 assert!(
13439 error
13440 .to_string()
13441 .contains("acquire mandatory repository lock")
13442 );
13443 assert!(
13444 !started.exists(),
13445 "failed acquisition must prevent pre-gate"
13446 );
13447 assert!(!checked.exists(), "failed acquisition must prevent gate");
13448 assert!(git::is_clean(&winner.worktree).await.unwrap());
13449 assert_eq!(
13450 git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
13451 head
13452 );
13453 }
13454
13455 #[tokio::test]
13456 #[cfg(unix)]
13457 async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
13458 use std::os::fd::AsRawFd as _;
13459 use std::time::Duration;
13460
13461 ask_test_home();
13462 let tmp = tempfile::tempdir().unwrap();
13463 let repo = tmp.path().join("repo");
13464 std::fs::create_dir_all(&repo).unwrap();
13465 init_repo(&repo);
13466 let common = repo.join(".git");
13467 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
13468 let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
13469 assert_eq!(
13470 unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13471 0
13472 );
13473 let mut runner = runner_at(RunStatus::Prep);
13474 runner.state.repo = repo.clone();
13475 runner.state.config.coordination = crate::config::test_coordination();
13476 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13477 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13478 runner.state.config.graph.advise = true;
13479 runner.state.config.graph.advisors = 1;
13480 runner.roles.implementers = vec![spec("implement")];
13481 runner.roles.judges = vec![spec("judge")];
13482 let root = runner.state.worktree_root();
13483 let started = tmp.path().join("setup-started");
13484 let proceed = tmp.path().join("setup-proceed");
13485 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13486 run: Some(format!(
13487 "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
13488 started.display(),
13489 proceed.display()
13490 )),
13491 ..Default::default()
13492 }];
13493 let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
13494 let config = git::git(&repo, &["config", "--local", "--list"])
13495 .await
13496 .unwrap();
13497 let mut prep = Box::pin(runner.prep());
13498 assert!(
13499 tokio::time::timeout(Duration::from_millis(200), &mut prep)
13500 .await
13501 .is_err()
13502 );
13503 assert!(
13504 !root.exists(),
13505 "contention must prevent even checkout creation"
13506 );
13507 assert!(!started.exists(), "contention must prevent setup");
13508 assert_eq!(
13509 git::git(&repo, &["branch", "--list"]).await.unwrap(),
13510 branches
13511 );
13512 assert_eq!(
13513 git::git(&repo, &["config", "--local", "--list"])
13514 .await
13515 .unwrap(),
13516 config
13517 );
13518 drop(lock);
13519 tokio::select! {
13520 result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
13521 result = tokio::time::timeout(Duration::from_secs(10), async {
13522 while !started.exists() {
13523 tokio::time::sleep(Duration::from_millis(20)).await;
13524 }
13525 }) => result.unwrap(),
13526 }
13527 let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
13528 assert_eq!(
13529 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13530 -1
13531 );
13532 assert!(
13533 tokio::time::timeout(
13534 Duration::from_millis(100),
13535 crate::repo_lock::SeatLease::for_checkout(
13536 &repo,
13537 true,
13538 crate::config::test_coordination().as_ref()
13539 )
13540 )
13541 .await
13542 .is_err(),
13543 "preparation must also exclude local editors"
13544 );
13545 std::fs::write(proceed, "ready").unwrap();
13546 tokio::time::timeout(Duration::from_secs(10), &mut prep)
13547 .await
13548 .unwrap()
13549 .unwrap();
13550 drop(prep);
13551 assert_eq!(runner.state.prep_complete, Some(true));
13552 assert!(root.join("judge-1").exists());
13553 assert!(root.join("advisor-1").exists());
13554 assert_eq!(
13555 unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
13556 0
13557 );
13558 }
13559
13560 #[tokio::test]
13561 #[cfg(unix)]
13562 async fn preparation_lock_failure_prevents_repository_changes() {
13563 ask_test_home();
13564 let tmp = tempfile::tempdir().unwrap();
13565 let repo = tmp.path().join("repo");
13566 std::fs::create_dir_all(&repo).unwrap();
13567 init_repo(&repo);
13568 std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
13569 std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
13572 let mut runner = runner_at(RunStatus::Prep);
13573 runner.state.repo = repo.clone();
13574 runner.state.config.coordination = crate::config::test_coordination();
13575 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13576 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13577 runner.roles.implementers = vec![spec("implement")];
13578 let error = runner.prep().await.unwrap_err();
13579 assert!(
13580 error
13581 .to_string()
13582 .contains("acquire mandatory repository lock")
13583 );
13584 assert!(!runner.state.worktree_root().exists());
13585 assert!(runner.state.candidates.is_empty());
13586 assert!(!repo.join(".git/config.worktree").exists());
13587 assert_eq!(
13588 git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
13589 "* main"
13590 );
13591 }
13592
13593 #[tokio::test]
13594 async fn preparation_records_creation_intent_before_git_can_fail() {
13595 ask_test_home();
13596 let tmp = tempfile::tempdir().unwrap();
13597 let repo = tmp.path().join("repo");
13598 std::fs::create_dir_all(&repo).unwrap();
13599 init_repo(&repo);
13600 let mut runner = runner_at(RunStatus::Prep);
13601 runner.state.repo = repo.clone();
13602 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13603 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13604 runner.state.config.blind.commit_msg_hook = false;
13605 runner.roles.implementers = vec![spec("one")];
13606 let label = blind::assign_labels(1, runner.state.seed)[0];
13607 let branch = runner.state.branch_for(label);
13608 git::git(&repo, &["branch", &branch]).await.unwrap();
13611 assert!(runner.prep().await.is_err());
13612 let saved = RunState::load(&runner.state.id).unwrap();
13613 assert!(saved.candidates.is_empty());
13614 let event = saved
13615 .events
13616 .iter()
13617 .find(|e| e.node == "prep-intent")
13618 .unwrap();
13619 let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
13620 assert_eq!(intent["branch"], branch);
13621 assert_eq!(intent["start"], runner.state.base_commit);
13622 assert_eq!(
13623 intent["worktree"],
13624 runner
13625 .state
13626 .worktree_root()
13627 .join(format!("cand-{label}"))
13628 .to_string_lossy()
13629 .as_ref()
13630 );
13631 assert_eq!(intent["picks"], serde_json::json!([]));
13632 }
13633
13634 #[tokio::test]
13635 async fn invalid_pinned_sources_create_no_partial_worktrees() {
13636 ask_test_home();
13637 let tmp = tempfile::tempdir().unwrap();
13638 let repo = tmp.path().join("repo");
13639 std::fs::create_dir_all(&repo).unwrap();
13640 init_repo(&repo);
13641 let mut runner = runner_at(RunStatus::Prep);
13642 runner.state.repo = repo.clone();
13643 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13644 runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
13645 runner.state.sources.pick_commits.push(refs::Input {
13646 token: "missing".into(),
13647 sha: "0".repeat(40),
13648 provenance: refs::Provenance::ExplicitCli,
13649 });
13650 assert!(runner.prep().await.is_err());
13651 assert!(runner.state.candidates.is_empty());
13652 assert!(!runner.state.worktree_root().exists());
13653 }
13654
13655 #[tokio::test]
13656 async fn prose_and_legacy_inference_never_import_into_candidates() {
13657 ask_test_home();
13658 let tmp = tempfile::tempdir().unwrap();
13659 let repo = tmp.path().join("repo");
13660 std::fs::create_dir_all(&repo).unwrap();
13661 init_repo(&repo);
13662 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13663 git::git(&repo, &["checkout", "-b", "magi/history/A"])
13664 .await
13665 .unwrap();
13666 std::fs::write(repo.join("unwanted"), "do not import").unwrap();
13667 git::git(&repo, &["add", "."]).await.unwrap();
13668 git::git(&repo, &["commit", "-m", "historical implementation"])
13669 .await
13670 .unwrap();
13671 let source = git::rev_parse(&repo, "HEAD").await.unwrap();
13672 git::git(&repo, &["checkout", "main"]).await.unwrap();
13673 for instruction in [
13674 format!("Example: magi/history/A at {source}"),
13675 format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
13676 format!("Never import magi/history/A or {source}"),
13677 format!("Conversation answer: I mentioned {source} only as background"),
13678 ] {
13679 let mut runner = runner_at(RunStatus::Prep);
13680 runner.state.repo = repo.clone();
13681 runner.state.base_commit = base.clone();
13682 runner.state.instruction = instruction.clone();
13683 runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
13684 runner.state.config.blind.commit_msg_hook = false;
13685 runner.state.seeds = vec![refs::Seed {
13686 token: "magi/history/A".into(),
13687 sha: source.clone(),
13688 kind: refs::SeedKind::Unmerged,
13689 branch: true,
13690 detail: String::new(),
13691 }];
13692 runner.roles.implementers = vec![spec("one")];
13693 assert_eq!(seeded_instruction(&runner.state), instruction);
13694 let mut legacy = serde_json::to_value(&runner.state).unwrap();
13696 legacy.as_object_mut().unwrap().remove("sources");
13697 runner.state = serde_json::from_value(legacy).unwrap();
13698 assert_eq!(runner.state.sources, refs::Sources::default());
13699 runner.state.save().unwrap();
13700 runner.state = RunState::load(&runner.state.id).unwrap();
13701 runner.prep().await.unwrap();
13702 for candidate in &runner.state.candidates {
13703 assert_eq!(
13704 git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
13705 base
13706 );
13707 assert!(!candidate.worktree.join("unwanted").exists());
13708 }
13709 }
13710 }
13711
13712 #[tokio::test]
13713 async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
13714 ask_test_home();
13715 let tmp = tempfile::tempdir().unwrap();
13716 let repo = tmp.path().join("repo");
13717 std::fs::create_dir_all(&repo).unwrap();
13718 init_repo(&repo);
13719 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
13720 git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
13721 std::fs::write(repo.join("referenced"), "required commit").unwrap();
13722 git::git(&repo, &["add", "."]).await.unwrap();
13723 git::git(&repo, &["commit", "-m", "required seed"])
13724 .await
13725 .unwrap();
13726 let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
13727 git::git(&repo, &["checkout", "main"]).await.unwrap();
13728 let mut runner = runner_at(RunStatus::Prep);
13729 runner.state.repo = repo.clone();
13730 runner.state.base_commit = base.clone();
13731 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13732 runner.state.config.graph.advise = false;
13733 runner.state.config.blind.commit_msg_hook = false;
13734 runner.state.sources =
13735 refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
13736 .await
13737 .unwrap();
13738 runner.roles.implementers = vec![spec("one"), spec("two")];
13739 runner.roles.judges = vec![spec("judge")];
13740 let label = blind::assign_labels(2, runner.state.seed)[0];
13741 let branch = runner.state.branch_for(label);
13742 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13743 git::worktree_add_branch(&repo, &worktree, &branch, &base)
13744 .await
13745 .unwrap();
13746 runner.state.candidates.push(Candidate {
13747 index: 0,
13748 label,
13749 agent: "one".into(),
13750 branch,
13751 worktree,
13752 summary: String::new(),
13753 stat: String::new(),
13754 files: 0,
13755 commits: 0,
13756 empty: false,
13757 failed: None,
13758 verified_noop: None,
13759 duration_ms: 0,
13760 folded: false,
13761 });
13762 runner.state.save().unwrap();
13763 runner.state = RunState::load(&runner.state.id).unwrap();
13765 assert_eq!(runner.state.prep_complete, Some(false));
13766 runner.prep().await.unwrap();
13767 assert_eq!(runner.state.prep_complete, Some(true));
13768 assert_eq!(runner.state.candidates.len(), 2);
13769 for candidate in &runner.state.candidates {
13770 assert_eq!(
13771 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13772 git::tree_of(&repo, &seed).await.unwrap()
13773 );
13774 assert_eq!(
13775 std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
13776 "required commit"
13777 );
13778 }
13779 assert!(runner.state.worktree_root().join("judge-1").exists());
13780 let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
13781 runner.prep().await.unwrap();
13782 assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
13783
13784 runner.state.prep_complete = Some(false);
13785 runner.state.config.worktree.setup = vec![crate::config::SetupStep {
13786 run: Some(format!(
13787 "test -f '{}'",
13788 tmp.path().join("dependency-ready").display()
13789 )),
13790 ..Default::default()
13791 }];
13792 assert!(runner.prep().await.is_err());
13793 assert!(runner.state.prep_failure.is_some());
13794 std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
13795 runner.state = RunState::load(&runner.state.id).unwrap();
13796 assert!(
13797 runner.prep().await.is_err(),
13798 "correcting the cause alone must not auto-retry"
13799 );
13800 let failed_at = runner.state.events.last().unwrap().at;
13801 assert!(
13802 runner.retry_preparation(failed_at).is_err(),
13803 "old approvals cannot unlock the hold"
13804 );
13805 let mut task = crate::queue::Task::new(
13806 "retry".into(),
13807 "retry setup".into(),
13808 repo.clone(),
13809 crate::queue::Source::Human,
13810 );
13811 task.start(runner.state.id.clone());
13812 task.hold_machine(Some("setup failed".into()));
13813 task.release_operator("cli", None);
13814 runner
13815 .retry_preparation(task.resumption.as_ref().unwrap().at)
13816 .unwrap();
13817 runner.prep().await.unwrap();
13818 assert!(runner.state.prep_failure.is_none());
13819 assert_eq!(runner.state.prep_complete, Some(true));
13820 assert_eq!(runner.state.candidates.len(), 2);
13821 assert!(runner.state.worktree_root().join("judge-1").exists());
13822 for candidate in &runner.state.candidates {
13823 assert_eq!(
13824 git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
13825 git::tree_of(&repo, &seed).await.unwrap()
13826 );
13827 }
13828 }
13829
13830 async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
13831 let mut heads = Vec::new();
13832 for candidate in candidates {
13833 heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
13834 }
13835 heads
13836 }
13837
13838 async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
13839 ask_test_home();
13840 let tmp = tempfile::tempdir().unwrap();
13841 let repo = tmp.path().join("repo");
13842 std::fs::create_dir(&repo).unwrap();
13843 init_repo(&repo);
13844 let mut runner = runner_at(RunStatus::Reviewing);
13845 runner.state.repo = repo.clone();
13846 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
13847 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
13848 for (index, label) in ['A', 'B'].into_iter().enumerate() {
13849 let branch = format!("candidate-{label}");
13850 let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
13851 git::worktree_add_branch(&repo, &worktree, &branch, "main")
13852 .await
13853 .unwrap();
13854 if label == 'B' {
13855 std::fs::write(worktree.join("change"), "winner").unwrap();
13856 git::rescue_commit(&worktree, "candidate change")
13857 .await
13858 .unwrap();
13859 }
13860 runner.state.candidates.push(Candidate {
13861 index,
13862 label,
13863 agent: "alpha".into(),
13864 branch,
13865 worktree,
13866 summary: "completed reply".into(),
13867 stat: String::new(),
13868 files: usize::from(label == 'B'),
13869 commits: usize::from(label == 'B'),
13870 empty: label == 'A',
13871 failed: None,
13872 verified_noop: None,
13873 duration_ms: 1,
13874 folded: false,
13875 });
13876 }
13877 runner.state.judge_skipped = true;
13878 runner.state.tally = Some(Tally {
13879 first_choice: BTreeMap::new(),
13880 borda: BTreeMap::new(),
13881 winner: 'B',
13882 rankings: 0,
13883 unanimous_initial: false,
13884 deliberated: false,
13885 changed_votes: 0,
13886 unanimous_final: false,
13887 tie_break: None,
13888 judges: 0,
13889 present: 0,
13890 quorum: 0,
13891 met_quorum: true,
13892 uncontested: Some("only B produced a change".into()),
13893 });
13894 runner.state.save().unwrap();
13895 (tmp, runner)
13896 }
13897
13898 #[tokio::test]
13899 #[cfg(unix)]
13900 async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
13901 for (solo, failed_checkpoint) in
13902 [(true, false), (false, false), (true, true), (false, true)]
13903 {
13904 let (_tmp, mut runner) = preservation_resume_fixture().await;
13905 if solo {
13906 runner.state.candidates.retain(|c| c.label == 'A');
13907 }
13908 runner.state.candidates[0].empty = false;
13909 runner.state.candidates[0].summary.clear();
13910 runner.state.tally = None;
13911 runner.state.judge_skipped = false;
13912 runner.state.status = RunStatus::Implementing;
13913 runner.state.config.disk.min_free_bytes = u64::MAX;
13914 let mut implementer = spec("alpha");
13915 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13916 runner.roles.implementers = vec![implementer];
13917 let patch = runner.state.dir().join("artifacts/cand-A.patch");
13918 if failed_checkpoint {
13919 std::fs::create_dir_all(&patch).unwrap();
13920 }
13921 runner.state.save().unwrap();
13922 assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
13923 runner.state = RunState::load(&runner.state.id).unwrap();
13924 assert_eq!(runner.state.status, RunStatus::Blocked);
13925 assert!(runner.state.interrupted_implementations.contains(&'A'));
13926 assert_eq!(runner.state.candidates[0].commits, 0);
13927 assert!(!runner.state.candidates[0].empty);
13928 assert!(runner.state.candidates[0].failed.is_none());
13929 if failed_checkpoint {
13930 assert!(runner.state.candidate_preservation.contains_key(&'A'));
13931 std::fs::remove_dir(patch).unwrap();
13932 }
13933 runner.state.resource_block = None;
13934 runner.state.config.disk.min_free_bytes = 0;
13935 runner.roles.implementers[0].command = vec![
13936 "sh".into(),
13937 "-c".into(),
13938 "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
13939 ];
13940 runner.implement().await.unwrap();
13941 assert!(runner.state.interrupted_implementations.is_empty());
13942 assert!(runner.state.candidate_preservation.is_empty());
13943 assert_eq!(runner.state.candidates[0].commits, 1);
13944 assert!(!runner.state.candidates[0].empty);
13945 assert!(
13946 std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
13947 .unwrap()
13948 .contains("retried empty interrupted seat")
13949 );
13950 if !solo {
13951 assert_eq!(runner.state.candidates[1].commits, 1);
13952 }
13953 }
13954 }
13955
13956 #[tokio::test]
13957 #[cfg(unix)]
13958 async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
13959 for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
13960 let (_tmp, mut runner) = preservation_resume_fixture().await;
13961 runner.state.candidates.retain(|c| c.label == 'A');
13962 runner.state.candidates[0].empty = false;
13963 runner.state.candidates[0].summary.clear();
13964 runner.state.tally = None;
13965 runner.state.judge_skipped = false;
13966 runner.state.status = RunStatus::Implementing;
13967 runner.state.config.disk.min_free_bytes = u64::MAX;
13970 let wt = runner.state.candidates[0].worktree.clone();
13971 std::fs::write(
13972 wt.join("interrupted-source"),
13973 "retain implementation work\n",
13974 )
13975 .unwrap();
13976 if committed {
13977 git::rescue_commit(&wt, "implementation checkpoint")
13978 .await
13979 .unwrap();
13980 }
13981 let lock = git::git(
13982 &wt,
13983 &[
13984 "rev-parse",
13985 "--path-format=absolute",
13986 "--git-path",
13987 "index.lock",
13988 ],
13989 )
13990 .await
13991 .unwrap();
13992 if block_rescue {
13993 std::fs::write(&lock, "fixture-owned lock").unwrap();
13994 }
13995 let mut implementer = spec("alpha");
13996 implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
13997 runner.roles.implementers = vec![implementer];
13998 runner.state.save().unwrap();
13999 let result = runner.implement().await;
14000 assert_eq!(result.is_err(), block_rescue);
14001 assert_eq!(
14002 runner.state.resource_block.as_ref().unwrap().kind,
14003 "disk pressure"
14004 );
14005 assert!(
14006 runner
14007 .state
14008 .events
14009 .iter()
14010 .any(|e| e.message.contains("agent execution interrupted"))
14011 );
14012 assert!(
14013 runner
14014 .state
14015 .dir()
14016 .join("artifacts/candidate-A-recovery/staged.patch")
14017 .exists()
14018 );
14019 if block_rescue {
14020 let saved = RunState::load(&runner.state.id).unwrap();
14021 assert!(saved.candidate_preservation.contains_key(&'A'));
14022 assert!(!saved.candidates[0].empty);
14023 assert_eq!(
14024 std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
14025 "retain implementation work\n"
14026 );
14027 std::fs::remove_file(lock).unwrap();
14028 runner.state = saved;
14029 } else {
14030 assert_eq!(runner.state.status, RunStatus::Blocked);
14031 assert_eq!(runner.state.candidates[0].commits, 1);
14032 }
14033 runner.roles.implementers.clear();
14036 runner.state.config.disk.min_free_bytes = 0;
14037 runner.state.resource_block = None;
14038 runner.implement().await.unwrap();
14039 assert!(runner.state.candidate_preservation.is_empty());
14040 assert_eq!(runner.state.candidates[0].commits, 1);
14041 assert!(!runner.state.candidates[0].empty);
14042 assert!(git::is_clean(&wt).await.unwrap());
14043 assert!(
14044 crate::run::read_artifact(&runner.state, "cand-A.patch")
14045 .unwrap()
14046 .contains("retain implementation work")
14047 );
14048 }
14049 }
14050
14051 #[tokio::test]
14052 async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
14053 let (_tmp, mut runner) = preservation_resume_fixture().await;
14054 runner.implement().await.unwrap();
14056 assert!(runner.state.candidate_preservation.is_empty());
14057 assert!(
14058 !runner
14059 .state
14060 .dir()
14061 .join("artifacts/candidate-A-recovery")
14062 .exists()
14063 );
14064 runner.fold_losers().await.unwrap();
14065 assert!(!runner.state.candidates[0].worktree.exists());
14066 for schema in [14, crate::run::SCHEMA] {
14067 runner.state.schema = schema;
14068 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14069 runner.state.save().unwrap();
14070 runner.state = RunState::load(&runner.state.id).unwrap();
14071 runner.implement().await.unwrap();
14072 assert!(runner.state.candidate_preservation.is_empty());
14073 assert!(runner.state.candidates[0].folded);
14074 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14075 }
14076 }
14077
14078 #[tokio::test]
14079 async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
14080 let (_tmp, mut runner) = preservation_resume_fixture().await;
14081 let wt = runner.state.candidates[0].worktree.clone();
14082 std::fs::write(wt.join("recovered"), "owned edits").unwrap();
14083 runner.state.schema = 14;
14084 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14085 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14086 let tally = serde_json::to_value(&runner.state.tally).unwrap();
14087 for _ in 0..2 {
14088 assert!(runner.implement().await.is_err());
14089 assert_eq!(runner.state.status, RunStatus::Blocked);
14090 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14091 assert!(
14092 runner.state.candidates[0].empty,
14093 "old verdict must remain unusable"
14094 );
14095 assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
14096 assert!(runner.state.judge_skipped);
14097 assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
14098 assert!(
14099 crate::run::read_artifact(&runner.state, "cand-A.patch")
14100 .unwrap()
14101 .contains("owned edits")
14102 );
14103 assert!(wt.exists());
14104 assert_eq!(
14105 git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
14106 .await
14107 .unwrap(),
14108 1
14109 );
14110 runner.state = RunState::load(&runner.state.id).unwrap();
14111 }
14112 }
14113
14114 #[tokio::test]
14115 async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
14116 let (_tmp, mut runner) = preservation_resume_fixture().await;
14117 let wt = runner.state.candidates[0].worktree.clone();
14118 runner.state.candidates[0].failed = Some("CLI failed before commit".into());
14119 std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
14120 let index_lock = PathBuf::from(
14121 git::git(
14122 &wt,
14123 &[
14124 "rev-parse",
14125 "--path-format=absolute",
14126 "--git-path",
14127 "index.lock",
14128 ],
14129 )
14130 .await
14131 .unwrap(),
14132 );
14133 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14134 runner.state.schema = 14;
14135 runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
14136 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14137 assert!(runner.implement().await.is_err());
14138 assert!(runner.after_implement().is_err());
14139 assert!(runner.fold_losers().await.is_err());
14140 assert_eq!(
14141 std::fs::read_to_string(wt.join("recovered")).unwrap(),
14142 "failed seat edits"
14143 );
14144 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14145 runner.state = RunState::load(&runner.state.id).unwrap();
14146 std::fs::remove_file(index_lock).unwrap();
14147 assert!(runner.implement().await.is_err());
14149 assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
14150 assert!(runner.state.candidates[0].empty);
14151 assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
14152 assert!(
14153 crate::run::read_artifact(&runner.state, "cand-A.patch")
14154 .unwrap()
14155 .contains("failed seat edits")
14156 );
14157 assert!(wt.exists());
14158 }
14159
14160 #[tokio::test]
14161 async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
14162 let (_tmp, mut runner) = preservation_resume_fixture().await;
14163 let repo = runner.state.repo.clone();
14164 let common = git::git(
14165 &repo,
14166 &["rev-parse", "--path-format=absolute", "--git-common-dir"],
14167 )
14168 .await
14169 .unwrap();
14170 let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
14171 std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
14172 assert!(runner.fold_losers().await.is_err());
14173 assert!(!runner.state.candidates[0].worktree.exists());
14174 assert!(!runner.state.candidates[0].folded);
14175 assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
14176 runner.state = RunState::load(&runner.state.id).unwrap();
14177 std::fs::remove_file(branch_lock).unwrap();
14178 let judge = runner.state.worktree_root().join("judge-1");
14179 git::worktree_add_detached(&repo, &judge, "main")
14180 .await
14181 .unwrap();
14182 std::fs::write(judge.join("uncommitted"), "retain").unwrap();
14183 runner.roles.judges = vec![spec("judge")];
14184 assert!(runner.fold_losers().await.is_err());
14185 runner.state = RunState::load(&runner.state.id).unwrap();
14186 assert!(
14187 runner.state.candidates[0].folded,
14188 "candidate progress saved before judge failure"
14189 );
14190 assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
14191 assert!(judge.join("uncommitted").exists());
14192 std::fs::remove_file(judge.join("uncommitted")).unwrap();
14193 runner.fold_losers().await.unwrap();
14194 assert!(!judge.exists());
14195 runner.fold_losers().await.unwrap();
14196 }
14197
14198 #[tokio::test]
14199 async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
14200 let (_tmp, mut runner) = preservation_resume_fixture().await;
14201 runner.state.candidates.retain(|c| c.label == 'B');
14202 runner.state.tally = None;
14203 runner.state.judge_skipped = false;
14204 runner.state.prep_complete = Some(true);
14205 runner.state.config.graph.advise = false;
14206 runner.state.config.graph.selected_implementer = Some("alpha".into());
14207 let wt = runner.state.candidates[0].worktree.clone();
14208 std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
14209 runner
14210 .state
14211 .candidate_preservation
14212 .insert('B', "awaiting persistence".into());
14213 let index_lock = git::git(
14214 &wt,
14215 &[
14216 "rev-parse",
14217 "--path-format=absolute",
14218 "--git-path",
14219 "index.lock",
14220 ],
14221 )
14222 .await
14223 .unwrap();
14224 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14225 runner.state.save().unwrap();
14226 assert!(runner.execute_graph().await.is_err());
14227 assert_eq!(runner.state.status, RunStatus::Blocked);
14228 assert!(runner.state.candidate_preservation.contains_key(&'B'));
14229 std::fs::remove_file(index_lock).unwrap();
14230 for _ in 0..2 {
14233 runner.state = RunState::load(&runner.state.id).unwrap();
14234 runner.pause.park();
14235 runner.execute_graph().await.unwrap();
14236 assert_eq!(runner.state.status, RunStatus::Implementing);
14237 assert!(runner.state.candidate_preservation.is_empty());
14238 assert_eq!(runner.state.candidates[0].commits, 2);
14239 assert!(runner.state.tally.is_none());
14240 }
14241 runner.implement().await.unwrap();
14242 assert_eq!(runner.state.status, RunStatus::Judging);
14243 runner.state.status = RunStatus::Blocked;
14245 runner.state.candidates[0].failed = Some("implementation failed".into());
14246 runner.state.save().unwrap();
14247 let error = runner.execute_graph().await.unwrap_err();
14248 assert!(
14249 error
14250 .to_string()
14251 .contains("selected implementation is held")
14252 );
14253 }
14254
14255 #[tokio::test]
14256 async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
14257 let (_tmp, mut runner) = preservation_resume_fixture().await;
14258 let repo = runner.state.repo.clone();
14259 let candidate = runner.state.candidates[0].clone();
14260 let registered = git::worktree_holding(&repo, &candidate.branch)
14263 .await
14264 .unwrap()
14265 .unwrap();
14266 assert_eq!(
14267 registered.canonicalize().unwrap(),
14268 candidate.worktree.canonicalize().unwrap()
14269 );
14270 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14271 assert_eq!(
14272 git::worktree_holding(&repo, &candidate.branch)
14273 .await
14274 .unwrap(),
14275 Some(registered)
14276 );
14277 assert!(
14278 !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
14279 "the stale registration initially keeps the branch checked out"
14280 );
14281 runner.fold_losers().await.unwrap();
14282 assert!(runner.state.candidates[0].folded);
14283 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14284 assert_eq!(
14285 git::worktree_holding(&repo, &candidate.branch)
14286 .await
14287 .unwrap(),
14288 None
14289 );
14290 runner.state = RunState::load(&runner.state.id).unwrap();
14291 runner.fold_losers().await.unwrap();
14292 }
14293
14294 #[tokio::test]
14295 async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
14296 let (_tmp, mut runner) = preservation_resume_fixture().await;
14297 let repo = runner.state.repo.clone();
14298 let candidate = runner.state.candidates[0].clone();
14299 let winner = runner.state.candidates[1].clone();
14300 let registered = git::worktree_holding(&repo, &candidate.branch)
14301 .await
14302 .unwrap()
14303 .unwrap();
14304 assert_eq!(
14305 registered.canonicalize().unwrap(),
14306 candidate.worktree.canonicalize().unwrap()
14307 );
14308 std::fs::remove_dir_all(&candidate.worktree).unwrap();
14309 assert_eq!(
14310 git::worktree_holding(&repo, &candidate.branch)
14311 .await
14312 .unwrap(),
14313 Some(registered)
14314 );
14315 assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
14316
14317 for _ in 0..2 {
14318 fold_run(&mut runner.state, false, &crate::run::home())
14319 .await
14320 .unwrap();
14321 runner.state = RunState::load(&runner.state.id).unwrap();
14322 assert!(runner.state.candidates[0].folded);
14323 assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
14324 assert_eq!(
14325 git::worktree_holding(&repo, &candidate.branch)
14326 .await
14327 .unwrap(),
14328 None
14329 );
14330 assert!(!runner.state.candidates[1].folded);
14331 assert!(winner.worktree.exists());
14332 assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
14333 }
14334 }
14335
14336 #[tokio::test]
14337 async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
14338 ask_test_home();
14339 let tmp = tempfile::tempdir().unwrap();
14340 let repo = tmp.path().join("repo");
14341 std::fs::create_dir(&repo).unwrap();
14342 init_repo(&repo);
14343 let mut runner = runner_at(RunStatus::Implementing);
14344 runner.state.repo = repo.clone();
14345 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14346 runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
14347 for (index, label) in ['A', 'B'].into_iter().enumerate() {
14348 let branch = format!("candidate-{label}");
14349 let wt = runner.state.worktree_root().join(format!("cand-{label}"));
14350 git::worktree_add_branch(&repo, &wt, &branch, "main")
14351 .await
14352 .unwrap();
14353 std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
14354 git::git(&wt, &["add", "README.md"]).await.unwrap();
14355 std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
14356 std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
14357 runner.state.candidates.push(Candidate {
14358 index,
14359 label,
14360 agent: "alpha".to_owned(),
14361 branch,
14362 worktree: wt,
14363 summary: "completed implementation".to_owned(),
14364 stat: String::new(),
14365 files: 0,
14366 commits: 0,
14367 empty: false,
14368 failed: None,
14369 verified_noop: None,
14370 duration_ms: 42,
14371 folded: false,
14372 });
14373 runner
14374 .state
14375 .candidate_preservation
14376 .insert(label, "awaiting persistence".to_owned());
14377 }
14378 let wt = runner.state.candidates[0].worktree.clone();
14379 let index_lock = git::git(
14380 &wt,
14381 &[
14382 "rev-parse",
14383 "--path-format=absolute",
14384 "--git-path",
14385 "index.lock",
14386 ],
14387 )
14388 .await
14389 .unwrap();
14390 std::fs::write(&index_lock, "fixture-owned lock").unwrap();
14391 runner.state.save().unwrap();
14392 assert!(runner.persist_candidates().await.is_err());
14393 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14394 assert!(!runner.state.candidate_preservation.contains_key(&'B'));
14395 assert_eq!(
14396 runner.state.candidates[1].commits, 1,
14397 "other results are still collected"
14398 );
14399 assert!(
14400 !runner.state.candidates[0].empty,
14401 "failed rescue is not no change"
14402 );
14403 assert_eq!(
14404 runner.state.candidates[0].summary,
14405 "completed implementation"
14406 );
14407 assert!(runner.after_implement().is_err());
14408 let home = tmp.path().join("fold-home");
14409 assert!(fold_run(&mut runner.state, true, &home).await.is_err());
14410 assert!(wt.join("new-file").exists());
14411 let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
14412 assert!(recovery.join("staged.patch").exists());
14413 assert!(recovery.join("unstaged.patch").exists());
14414 assert_eq!(
14415 std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
14416 "untracked edits\n"
14417 );
14418 runner.state.begin_pass();
14419 runner.state.end_pass_parked(ParkCause::Upgrade {
14420 from: "1.0".into(),
14421 to: Some("1.1".into()),
14422 });
14423 runner.state.save().unwrap();
14424 runner.state = RunState::load(&runner.state.id).unwrap();
14425 assert!(matches!(
14426 runner.state.passes.last().unwrap().end,
14427 Some(crate::run::PassEnd::Parked {
14428 cause: ParkCause::Upgrade { .. }
14429 })
14430 ));
14431 std::fs::remove_file(index_lock).unwrap();
14432 runner.implement().await.unwrap();
14434 assert!(runner.state.candidate_preservation.is_empty());
14435 assert!(
14436 runner
14437 .state
14438 .candidates
14439 .iter()
14440 .all(|c| c.commits == 1 && !c.empty)
14441 );
14442 assert!(git::is_clean(&wt).await.unwrap());
14443 assert!(
14444 crate::run::read_artifact(&runner.state, "cand-A.patch")
14445 .unwrap()
14446 .contains("later tracked edits")
14447 );
14448 }
14449
14450 #[tokio::test]
14451 async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
14452 ask_test_home();
14453 let tmp = tempfile::tempdir().unwrap();
14454 let repo = tmp.path().join("repo");
14455 std::fs::create_dir(&repo).unwrap();
14456 init_repo(&repo);
14457 let wt = tmp.path().join("candidate");
14458 git::worktree_add_branch(&repo, &wt, "candidate", "main")
14459 .await
14460 .unwrap();
14461 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14462 assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
14463 assert!(wt.join("uncommitted").exists());
14464 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14465 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14466 let mut runner = runner_at(RunStatus::Implementing);
14467 runner.state.repo = repo.clone();
14468 runner.state.base_commit = "missing-base".to_owned();
14469 runner.state.candidates.push(Candidate {
14470 index: 0,
14471 label: 'A',
14472 agent: "alpha".into(),
14473 branch: "candidate".into(),
14474 worktree: wt.clone(),
14475 summary: "completed".into(),
14476 stat: String::new(),
14477 files: 0,
14478 commits: 0,
14479 empty: false,
14480 failed: None,
14481 verified_noop: None,
14482 duration_ms: 1,
14483 folded: false,
14484 });
14485 std::fs::write(wt.join("uncommitted"), "keep").unwrap();
14486 assert!(
14487 fold_run(&mut runner.state, true, &tmp.path().join("home"))
14488 .await
14489 .is_err()
14490 );
14491 assert!(!runner.state.candidates[0].folded);
14492 assert!(git::branch_exists(&repo, "candidate").await.unwrap());
14493 std::fs::remove_file(wt.join("uncommitted")).unwrap();
14494 runner
14495 .state
14496 .candidate_preservation
14497 .insert('A', "pending".into());
14498 assert!(runner.persist_candidates().await.is_err());
14499 assert!(!runner.state.candidates[0].empty);
14500 assert!(!runner.state.candidates[0].folded);
14501 assert!(wt.exists());
14502 runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14503 let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
14504 std::fs::create_dir_all(&patch_path).unwrap();
14505 assert!(
14506 runner.persist_candidates().await.is_err(),
14507 "artifact failure stays pending"
14508 );
14509 assert!(runner.state.candidate_preservation.contains_key(&'A'));
14510 assert!(!runner.state.candidates[0].empty);
14511 std::fs::remove_dir(patch_path).unwrap();
14512 runner.persist_candidates().await.unwrap();
14513 assert!(
14514 runner.state.candidates[0].empty,
14515 "only successful inspection confirms no change"
14516 );
14517 }
14518
14519 fn init_repo(dir: &Path) {
14520 let run = |args: &[&str]| {
14521 let out = std::process::Command::new("git")
14522 .args(args)
14523 .current_dir(dir)
14524 .quiet()
14525 .output()
14526 .expect("spawn git");
14527 assert!(
14528 out.status.success(),
14529 "git {args:?} failed: {}",
14530 String::from_utf8_lossy(&out.stderr)
14531 );
14532 };
14533 run(&["init", "-b", "main"]);
14534 run(&["config", "user.name", "magi test"]);
14535 run(&["config", "user.email", "magi@example.com"]);
14536 std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
14537 run(&["add", "-A"]);
14538 run(&["commit", "-m", "init"]);
14539 }
14540
14541 fn ask_test_home() {
14549 crate::run::pin_test_home();
14550 }
14551
14552 fn runner_at(status: RunStatus) -> Runner {
14555 let mut state = RunState::new(
14556 PathBuf::from("/nonexistent/repo"),
14557 "main".to_owned(),
14558 "deadbeef".to_owned(),
14559 "task".to_owned(),
14560 Config::default(),
14561 );
14562 static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
14565 state.id = format!(
14566 "20000101-000000-{:04x}",
14567 NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
14568 );
14569 state.status = status;
14570 Runner {
14571 state,
14572 roles: ResolvedRoles {
14573 implementers: Vec::new(),
14574 judges: Vec::new(),
14575 reviewers: Vec::new(),
14576 fixer: None,
14577 conductor: conductor(),
14578 implementer_roster: Vec::new(),
14579 judge_roster: Vec::new(),
14580 reviewer_roster: Vec::new(),
14581 },
14582 sem: Arc::new(Semaphore::new(1)),
14583 pause: Pause::new(),
14584 interrupt: Pause::new(),
14585 }
14586 }
14587
14588 #[tokio::test]
14589 async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
14590 crate::run::pin_test_home();
14591 let mut runner = runner_at(RunStatus::Gating);
14592 runner.state.save().unwrap();
14593 let id = runner.state.id.clone();
14594 let passes = runner.state.passes.len();
14595 let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
14596 let mut claiming = Box::pin(runner.claim_driver());
14597 std::future::poll_fn(|cx| {
14598 assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
14599 std::task::Poll::Ready(())
14600 })
14601 .await;
14602 assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
14603 drop(owner);
14604 let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
14605 .await
14606 .unwrap()
14607 .unwrap();
14608 assert_eq!(execution.runner.state.passes.len(), passes + 1);
14609 assert!(execution.runner.state.resource_block.is_none());
14610 assert_eq!(
14611 execution.runner.state.liveness(false),
14612 crate::run::Liveness::Live
14613 );
14614 drop(execution);
14615 assert!(RunState::load(&id).unwrap().driver_exited);
14616 }
14617
14618 #[tokio::test]
14619 async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
14620 crate::run::pin_test_home();
14621 let mut runner = runner_at(RunStatus::Gating);
14622 runner.state.save().unwrap();
14623 std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
14626 assert!(runner.claim_driver().await.is_err());
14627 assert!(runner.state.driver_exited);
14628 assert_eq!(runner.state.driver_pid, None);
14629 let disk = RunState::load(&runner.state.id).unwrap();
14630 assert!(disk.driver_exited);
14631 assert_eq!(disk.driver_pid, None);
14632 assert_eq!(disk.status, RunStatus::Gating);
14633 }
14634
14635 #[tokio::test]
14636 async fn an_early_graph_error_releases_its_driver() {
14637 crate::run::pin_test_home();
14638 let mut runner = runner_at(RunStatus::Blocked);
14639 runner.state.config.graph.selected_implementer = Some("selected".into());
14640 runner.state.save().unwrap();
14641 assert!(runner.execute().await.is_err());
14642 let disk = RunState::load(&runner.state.id).unwrap();
14643 assert!(disk.driver_exited);
14644 assert_eq!(disk.driver_pid, None);
14645 assert_eq!(disk.status, RunStatus::Blocked);
14646 }
14647
14648 fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
14649 crate::run::MergeRepair {
14650 question: "repair-question".into(),
14651 head: "old-head".into(),
14652 pr: "https://github.com/o/r/pull/42".into(),
14653 reviews: vec![review_round(false, 1, 1, 1, false, true)],
14654 gate: vec![outcome(Some(0), false)],
14655 gate_head: Some("old-head".into()),
14656 gate_fixes: Vec::new(),
14657 pre_gate: Vec::new(),
14658 phase,
14659 failure: None,
14660 }
14661 }
14662
14663 #[tokio::test]
14664 async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
14665 crate::run::pin_test_home();
14666 let mut runner = runner_at(RunStatus::Landing);
14667 runner.state.merge_repairs.push(merge_repair_request(
14668 crate::run::MergeRepairPhase::Requested,
14669 ));
14670 runner.state.save().unwrap();
14671 let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
14672 assert!(
14673 runner
14674 .execute()
14675 .await
14676 .unwrap_err()
14677 .to_string()
14678 .contains("already running")
14679 );
14680 assert_eq!(
14681 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14682 crate::run::MergeRepairPhase::Requested
14683 );
14684 drop(claim);
14685 let mut disk = runner.state.clone();
14686 disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
14687 disk.save().unwrap();
14688 assert!(
14689 runner
14690 .execute()
14691 .await
14692 .unwrap_err()
14693 .to_string()
14694 .contains("decision changed")
14695 );
14696 assert_eq!(
14697 RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
14698 crate::run::MergeRepairPhase::Complete
14699 );
14700 }
14701
14702 #[test]
14703 fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
14704 crate::run::pin_test_home();
14705 let mut runner = runner_at(RunStatus::Landing);
14706 let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
14707 runner.state.reviews = request.reviews.clone();
14708 runner.state.gate = request.gate.clone();
14709 runner.state.gate_ran = true;
14710 runner.state.land_approval = Some(crate::run::LandApproval {
14711 question: "old".into(),
14712 head: "old-head".into(),
14713 });
14714 runner.state.merge_repairs.push(request);
14715 runner.begin_merge_repair_review(0).unwrap();
14716 assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
14717 assert!(!runner.state.gate_status().ok());
14718 assert!(runner.state.land_approval.is_none());
14719 let disk = RunState::load(&runner.state.id).unwrap();
14720 assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
14721 assert!(disk.merge_repairs[0].gate[0].ok());
14722 assert_eq!(
14723 disk.merge_repairs[0].phase,
14724 crate::run::MergeRepairPhase::Reviewing
14725 );
14726 assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
14727 assert!(disk.reviews.is_empty());
14728 }
14729
14730 #[tokio::test]
14731 async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
14732 crate::run::pin_test_home();
14733 for phase in [
14734 crate::run::MergeRepairPhase::Fixing,
14735 crate::run::MergeRepairPhase::Requested,
14736 ] {
14737 let mut runner = runner_at(RunStatus::Landing);
14738 runner.state.config.graph.land_rounds = 0;
14739 runner.state.merge_repairs.push(merge_repair_request(phase));
14740 runner.state.save().unwrap();
14741 assert!(!runner.repair_merge_review().await.unwrap());
14742 assert_eq!(runner.state.status, RunStatus::Blocked);
14743 assert_eq!(
14744 runner.state.merge_repairs[0].phase,
14745 crate::run::MergeRepairPhase::Failed
14746 );
14747 let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
14748 assert!(
14749 reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
14750 "interrupted"
14751 } else {
14752 "budget exhausted"
14753 })
14754 );
14755 assert!(runner.state.merge.is_none());
14756 }
14757 }
14758
14759 #[test]
14763 fn park_here_folds_the_interrupt_reason_into_the_park_event() {
14764 crate::run::pin_test_home();
14765 let mut runner = runner_at(RunStatus::Implementing);
14766 let interrupt = Pause::new();
14767 runner.watch_interrupt(interrupt.clone());
14768
14769 interrupt.park_because("task a1b2 asked to run first");
14770
14771 assert!(runner.park_here().expect("park_here"));
14772 assert!(runner.state.parked);
14773 let last = runner.state.events.last().expect("a park event");
14774 assert_eq!(last.node, "park");
14775 assert!(
14776 last.message.contains("task a1b2 asked to run first"),
14777 "expected the interrupt reason in {:?}",
14778 last.message
14779 );
14780 }
14781
14782 #[test]
14785 fn park_here_records_the_structured_cause_and_the_first_one_wins() {
14786 crate::run::pin_test_home();
14787 let mut runner = runner_at(RunStatus::Implementing);
14788 runner.state.begin_pass();
14789 let shutdown = Pause::new();
14790 runner.on_pause(shutdown.clone());
14791 let upgrade = ParkCause::Upgrade {
14792 from: "0.121.0".to_owned(),
14793 to: Some("0.121.1".to_owned()),
14794 };
14795 shutdown.park_for(upgrade.clone());
14796 shutdown.park_for(ParkCause::Operator);
14797
14798 assert!(runner.park_here().expect("park_here"));
14799 assert_eq!(
14800 runner.state.passes[0].end,
14801 Some(crate::run::PassEnd::Parked { cause: upgrade })
14802 );
14803 let last = runner.state.events.last().expect("a park event");
14804 assert!(last.message.contains("0.121.0"), "{}", last.message);
14805 }
14806
14807 #[test]
14815 fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
14816 crate::run::pin_test_home();
14817 let mut runner = runner_at(RunStatus::Implementing);
14818 let shutdown = Pause::new();
14819 runner.on_pause(shutdown.clone());
14820 let interrupt = Pause::new();
14821 runner.watch_interrupt(interrupt.clone());
14822
14823 assert!(!runner.park_here().expect("park_here"));
14825 assert!(!runner.state.parked);
14826
14827 interrupt.park_because("test");
14829 assert!(!shutdown.parked());
14830 assert!(runner.park_here().expect("park_here"));
14831 }
14832
14833 #[tokio::test]
14847 async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
14848 crate::run::pin_test_home();
14849 let mut runner = runner_at(RunStatus::Implementing);
14850 let interrupt = Pause::new();
14851 runner.watch_interrupt(interrupt.clone());
14852
14853 let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
14854 let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
14855
14856 let node = async move {
14860 started_tx.send(()).expect("send started");
14861 finish_rx.await.expect("recv finish");
14862 "node finished"
14863 };
14864
14865 let interrupter = async move {
14866 started_rx.await.expect("recv started");
14867 interrupt.park_because("higher-priority task waiting");
14869 tokio::task::yield_now().await;
14873 finish_tx.send(()).expect("send finish");
14874 };
14875
14876 let (node_result, ()) = tokio::join!(node, interrupter);
14877 assert_eq!(
14878 node_result, "node finished",
14879 "the in-flight call ran to completion"
14880 );
14881
14882 assert!(runner.park_here().expect("park_here"));
14885 assert!(runner.state.parked);
14886 }
14887
14888 #[test]
14894 fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
14895 crate::run::pin_test_home();
14896 let mut runner = runner_at(RunStatus::Judging);
14897 runner.state.config.agents = vec![conductor()];
14901 runner.state.candidates = vec![Candidate {
14902 index: 0,
14903 label: 'A',
14904 agent: "alpha".to_owned(),
14905 branch: "magi/x/A".to_owned(),
14906 worktree: PathBuf::from("/nonexistent/worktree"),
14907 summary: "did the thing".to_owned(),
14908 stat: "1 file changed".to_owned(),
14909 files: 1,
14910 commits: 1,
14911 empty: false,
14912 failed: None,
14913 verified_noop: None,
14914 duration_ms: 1234,
14915 folded: false,
14916 }];
14917 let run_id = runner.state.id.clone();
14918
14919 let interrupt = Pause::new();
14920 runner.watch_interrupt(interrupt.clone());
14921 interrupt.park_because("task c3d4 asked to run first");
14922 assert!(runner.park_here().expect("park_here"));
14923
14924 let resumed = Runner::resume(&run_id).expect("resume");
14925 assert_eq!(resumed.state.candidates.len(), 1);
14926 assert_eq!(resumed.state.candidates[0].summary, "did the thing");
14927 assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
14928 assert_eq!(resumed.state.status, runner.state.status);
14929 assert!(
14930 resumed.state.parked,
14931 "still parked until `execute` actually walks the graph again"
14932 );
14933 assert!(resumed.state.events.iter().any(|e| e.node == "park"));
14934 }
14935
14936 fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
14938 let mut q = ask::Question::new(
14939 run.to_owned(),
14940 "implement".to_owned(),
14941 "impl-A".to_owned(),
14942 "Which storage backend should the cache use?".to_owned(),
14943 String::new(),
14944 vec!["SQLite".to_owned(), "Redis".to_owned()],
14945 );
14946 store.put(&mut q).unwrap();
14947 q
14948 }
14949
14950 #[test]
14951 fn a_failed_runs_open_question_is_abandoned() {
14952 ask_test_home();
14953 let store = ask::Questions::open();
14954 let mut runner = runner_at(RunStatus::Failed);
14955 let run = runner.state.id.clone();
14956 let q = ask_open_question(&store, &run);
14957
14958 runner.settle_questions();
14959
14960 let back = store.get(&q.id).unwrap();
14961 assert!(
14962 !back.status.open(),
14963 "the seat that asked died with the run; nobody is left to read an answer"
14964 );
14965 assert!(
14966 back.detail.contains(&run) && back.detail.contains("failed"),
14967 "the reason names what the run became, not just that it is gone: {}",
14968 back.detail
14969 );
14970 }
14971
14972 #[test]
14973 fn a_merged_runs_open_question_is_abandoned_too() {
14974 ask_test_home();
14975 let store = ask::Questions::open();
14976 for status in [RunStatus::Merged, RunStatus::Ready] {
14979 let mut runner = runner_at(status);
14980 let run = runner.state.id.clone();
14981 let q = ask_open_question(&store, &run);
14982
14983 runner.settle_questions();
14984
14985 let back = store.get(&q.id).unwrap();
14986 assert!(
14987 !back.status.open(),
14988 "{status:?} run's question must not outlive the run"
14989 );
14990 }
14991 }
14992
14993 #[test]
14994 fn a_still_resumable_runs_open_question_is_left_alone() {
14995 ask_test_home();
14996 let store = ask::Questions::open();
14997 for status in [RunStatus::Blocked, RunStatus::Stalled] {
15003 let mut runner = runner_at(status);
15004 let run = runner.state.id.clone();
15005 let q = ask_open_question(&store, &run);
15006
15007 runner.settle_questions();
15008
15009 let back = store.get(&q.id).unwrap();
15010 assert!(
15011 back.status.open(),
15012 "{status:?} is still alive; the question must still be waiting"
15013 );
15014 }
15015 }
15016
15017 #[test]
15018 fn settle_questions_never_touches_an_already_answered_question() {
15019 ask_test_home();
15020 let store = ask::Questions::open();
15021 let mut runner = runner_at(RunStatus::Failed);
15022 let run = runner.state.id.clone();
15023 let mut q = ask_open_question(&store, &run);
15024 q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
15025 .unwrap();
15026 store.put(&mut q).unwrap();
15027
15028 runner.settle_questions();
15033 runner.settle_questions();
15034
15035 let back = store.get(&q.id).unwrap();
15036 assert_eq!(
15037 back.status,
15038 ask::QuestionStatus::Answered,
15039 "a real answer is a decision on record, never overwritten by a sweep"
15040 );
15041 }
15042
15043 #[tokio::test]
15054 async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
15055 crate::run::pin_test_home();
15056 let tmp = tempfile::tempdir().expect("tempdir");
15057 let repo = tmp.path().join("repo");
15058 std::fs::create_dir_all(&repo).unwrap();
15059 init_repo(&repo);
15060
15061 let mut config = Config::default();
15062 config.graph.worktree_root = Some(tmp.path().join("wt"));
15063
15064 let mut state = RunState::new(
15065 repo.clone(),
15066 "main".to_owned(),
15067 "deadbeef".to_owned(),
15068 "task".to_owned(),
15069 config,
15070 );
15071 let root = state.worktree_root();
15072 let wt_a = root.join("cand-A");
15073 let wt_b = root.join("cand-B");
15074 git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
15075 .await
15076 .expect("worktree A");
15077 git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
15078 .await
15079 .expect("worktree B");
15080
15081 state.candidates = vec![
15082 Candidate {
15083 index: 0,
15084 label: 'A',
15085 agent: "alpha".to_owned(),
15086 branch: "magi/x/A".to_owned(),
15087 worktree: wt_a.clone(),
15088 summary: String::new(),
15089 stat: String::new(),
15090 files: 0,
15091 commits: 0,
15092 empty: false,
15093 failed: None,
15094 verified_noop: None,
15095 duration_ms: 0,
15096 folded: false,
15097 },
15098 Candidate {
15099 index: 1,
15100 label: 'B',
15101 agent: "beta".to_owned(),
15102 branch: "magi/x/B".to_owned(),
15103 worktree: wt_b.clone(),
15104 summary: String::new(),
15105 stat: String::new(),
15106 files: 0,
15107 commits: 0,
15108 empty: false,
15109 failed: None,
15110 verified_noop: None,
15111 duration_ms: 0,
15112 folded: false,
15113 },
15114 ];
15115 state.tally = Some(Tally {
15116 first_choice: BTreeMap::from([('A', 1)]),
15117 borda: BTreeMap::new(),
15118 winner: 'A',
15119 rankings: 1,
15120 unanimous_initial: true,
15121 deliberated: false,
15122 changed_votes: 0,
15123 unanimous_final: true,
15124 tie_break: None,
15125 judges: 1,
15126 present: 1,
15127 quorum: 1,
15128 met_quorum: true,
15129 uncontested: None,
15130 });
15131 state.status = RunStatus::Ready;
15132
15133 fold_run(&mut state, false, &crate::run::home())
15134 .await
15135 .expect("fold_run");
15136
15137 assert!(wt_a.exists(), "the unmerged winner's worktree survives");
15138 assert!(
15139 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15140 "the unmerged winner's branch survives"
15141 );
15142 assert!(
15143 !state.candidates[0].folded,
15144 "the winner is not marked folded"
15145 );
15146
15147 assert!(!wt_b.exists(), "the loser's worktree is removed");
15148 assert!(
15149 !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
15150 "the loser's branch is removed"
15151 );
15152 assert!(state.candidates[1].folded, "the loser is marked folded");
15153 }
15154
15155 #[tokio::test]
15156 #[cfg(unix)]
15157 async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
15158 use std::os::fd::AsRawFd as _;
15159 use std::time::Duration;
15160
15161 let tmp = tempfile::tempdir().unwrap();
15162 let repo = tmp.path().join("repo");
15163 std::fs::create_dir_all(&repo).unwrap();
15164 init_repo(&repo);
15165 let common = repo.join(".git");
15166 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15167 let home = tmp.path().join("home");
15168 let mut config = Config::default();
15169 config.coordination = crate::config::test_coordination();
15170 config.graph.worktree_root = Some(tmp.path().join("wt"));
15171 config.graph.timeout_resource_wait = 1;
15172 let mut state = RunState::new(
15173 repo.clone(),
15174 "main".into(),
15175 "deadbeef".into(),
15176 "task".into(),
15177 config,
15178 );
15179 let root = state.worktree_root();
15180 let candidate = root.join("cand-A");
15181 let reviewer = root.join("review-1");
15182 let branch = state.branch_for('A');
15183 git::worktree_add_branch(&repo, &candidate, &branch, "main")
15184 .await
15185 .unwrap();
15186 git::worktree_add_detached(&repo, &reviewer, "main")
15187 .await
15188 .unwrap();
15189 git::acquire_worktree_config(&repo).await.unwrap();
15190 state.enabled_worktree_config = true;
15191 state.status = RunStatus::Ready;
15192 state.candidates.push(Candidate {
15193 index: 0,
15194 label: 'A',
15195 agent: "implement".into(),
15196 branch: branch.clone(),
15197 worktree: candidate.clone(),
15198 summary: String::new(),
15199 stat: String::new(),
15200 files: 0,
15201 commits: 0,
15202 empty: false,
15203 failed: None,
15204 verified_noop: None,
15205 duration_ms: 0,
15206 folded: false,
15207 });
15208 state.save_under(&home).unwrap();
15209 let config_before = std::fs::read(common.join("config")).unwrap();
15210 let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
15211 .await
15212 .unwrap();
15213 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
15216 assert_eq!(
15217 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15218 0
15219 );
15220 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15221 assert!(
15222 tokio::time::timeout(Duration::from_millis(200), &mut fold)
15223 .await
15224 .is_err()
15225 );
15226 assert!(candidate.exists());
15227 assert!(reviewer.exists());
15228 assert!(git::branch_exists(&repo, &branch).await.unwrap());
15229 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15230 assert_eq!(
15231 git::git(&repo, &["worktree", "list", "--porcelain"])
15232 .await
15233 .unwrap(),
15234 worktrees_before
15235 );
15236 let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
15237 .await
15238 .unwrap()
15239 .unwrap_err();
15240 assert!(error.is::<crate::repo_lock::ResourceBlocked>());
15241 drop(fold);
15242 assert!(candidate.exists() && reviewer.exists());
15243 assert!(!state.candidates[0].folded && state.enabled_worktree_config);
15244 assert_eq!(
15245 state.resource_block.as_ref().unwrap().kind,
15246 "repository coordination"
15247 );
15248 assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
15249 drop(external);
15250 let mut fold = Box::pin(fold_run(&mut state, true, &home));
15251 let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
15252 .await
15253 .unwrap()
15254 .unwrap();
15255 drop(fold);
15256 assert!(removed.contains(&branch));
15257 assert!(!root.exists());
15258 assert!(!git::branch_exists(&repo, &branch).await.unwrap());
15259 assert!(state.candidates[0].folded);
15260 assert!(!state.enabled_worktree_config);
15261 assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
15262 let enabled = git::git_raw(
15263 &repo,
15264 &["config", "--local", "--get", "extensions.worktreeConfig"],
15265 )
15266 .await
15267 .unwrap();
15268 assert!(!enabled.ok(), "the last config reference was released");
15269 }
15270
15271 #[tokio::test]
15274 async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
15275 let tmp = tempfile::tempdir().expect("tempdir");
15276 let repo = tmp.path().join("repo");
15277 std::fs::create_dir_all(&repo).unwrap();
15278 init_repo(&repo);
15279 let home = tmp.path().join("home");
15280
15281 let mut config = Config::default();
15282 config.graph.worktree_root = Some(tmp.path().join("wt"));
15283 let mut state = RunState::new(
15284 repo.clone(),
15285 "main".to_owned(),
15286 "deadbeef".to_owned(),
15287 "task".to_owned(),
15288 config,
15289 );
15290 git::git(&repo, &["branch", "magi/x/A", "main"])
15292 .await
15293 .expect("branch");
15294 state.candidates = vec![Candidate {
15295 index: 0,
15296 label: 'A',
15297 agent: "alpha".to_owned(),
15298 branch: "magi/x/A".to_owned(),
15299 worktree: state.worktree_root().join("cand-A"),
15300 summary: String::new(),
15301 stat: String::new(),
15302 files: 0,
15303 commits: 0,
15304 empty: false,
15305 failed: None,
15306 verified_noop: None,
15307 duration_ms: 0,
15308 folded: true,
15309 }];
15310 state.released_to = Some("20260901-000000-new1".to_owned());
15311 state.released_branches = vec!["magi/x/A".to_owned()];
15312
15313 fold_run(&mut state, true, &home).await.expect("fold_run");
15314 state.status = RunStatus::Merged;
15315 state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
15316 state.merge = Some(crate::run::MergeOutcome {
15317 mode: MergeMode::Pr,
15318 ok: true,
15319 detail: "confirmed merged".into(),
15320 empty: false,
15321 });
15322 fold_run(&mut state, true, &home)
15323 .await
15324 .expect("merged fold");
15325
15326 assert!(
15327 git::branch_exists(&repo, "magi/x/A").await.unwrap(),
15328 "the handed-over branch survives a fold"
15329 );
15330 }
15331
15332 #[tokio::test]
15336 async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
15337 let tmp = tempfile::tempdir().expect("tempdir");
15338 let repo = tmp.path().join("repo");
15339 std::fs::create_dir_all(&repo).unwrap();
15340 init_repo(&repo);
15341 let run = |args: &[&str]| {
15342 let out = std::process::Command::new("git")
15343 .quiet()
15344 .args(args)
15345 .current_dir(&repo)
15346 .output()
15347 .expect("spawn git");
15348 assert!(out.status.success(), "git {args:?}");
15349 };
15350 run(&["branch", "magi/x/A"]);
15351 run(&["checkout", "-q", "-b", "magi/x/B"]);
15352 std::fs::write(repo.join("f.txt"), "x\n").unwrap();
15353 run(&["add", "-A"]);
15354 run(&["commit", "-q", "-m", "work"]);
15355 run(&["checkout", "-q", "main"]);
15356
15357 let probe = RunState::new(
15361 repo.clone(),
15362 "main".to_owned(),
15363 "deadbeef".to_owned(),
15364 "task".to_owned(),
15365 Config::default(),
15366 );
15367 assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
15368 let bare = tmp.path().join("origin.git");
15369 let out = std::process::Command::new("git")
15370 .quiet()
15371 .args(["init", "-q", "--bare"])
15372 .arg(&bare)
15373 .output()
15374 .expect("spawn git");
15375 assert!(out.status.success(), "git init --bare");
15376 run(&["remote", "add", "origin", bare.to_str().unwrap()]);
15377 run(&["push", "-q", "origin", "main"]);
15378
15379 let mut state = RunState::new(
15380 repo.clone(),
15381 "main".to_owned(),
15382 "deadbeef".to_owned(),
15383 "task".to_owned(),
15384 Config::default(),
15385 );
15386 state.seeds = vec![refs::Seed {
15387 token: "magi/27b2/A".to_owned(),
15388 kind: refs::SeedKind::Unresolved,
15389 sha: String::new(),
15390 branch: true,
15391 detail: "no branch or commit named magi/27b2/A".to_owned(),
15392 }];
15393
15394 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
15395 assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
15396 assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
15397 let detail = empty_candidate_detail(&state, "main");
15398 assert!(detail.starts_with("empty candidate"), "{detail}");
15399 assert!(!detail.contains("magi/27b2/A"), "{detail}");
15400 assert!(detail.contains("--start-from"), "{detail}");
15401 }
15402
15403 #[tokio::test]
15412 async fn merge_does_not_reattempt_once_a_run_has_concluded() {
15413 let tmp = tempfile::tempdir().expect("tempdir");
15414 let repo = tmp.path().join("repo");
15415 std::fs::create_dir_all(&repo).unwrap();
15416 init_repo(&repo);
15417
15418 let mut config = Config::default();
15419 config.merge.mode = MergeMode::Local;
15420
15421 let mut state = RunState::new(
15422 repo.clone(),
15423 "main".to_owned(),
15424 "deadbeef".to_owned(),
15425 "task".to_owned(),
15426 config,
15427 );
15428 state.candidates = vec![Candidate {
15429 index: 0,
15430 label: 'A',
15431 agent: "alpha".to_owned(),
15432 branch: "does-not-exist".to_owned(),
15433 worktree: repo.clone(),
15434 summary: String::new(),
15435 stat: String::new(),
15436 files: 0,
15437 commits: 0,
15438 empty: false,
15439 failed: None,
15440 verified_noop: None,
15441 duration_ms: 0,
15442 folded: false,
15443 }];
15444 state.tally = Some(Tally {
15445 first_choice: BTreeMap::from([('A', 1)]),
15446 borda: BTreeMap::new(),
15447 winner: 'A',
15448 rankings: 1,
15449 unanimous_initial: true,
15450 deliberated: false,
15451 changed_votes: 0,
15452 unanimous_final: true,
15453 tie_break: None,
15454 judges: 0,
15455 present: 0,
15456 quorum: 0,
15457 met_quorum: true,
15458 uncontested: Some("only candidate A produced a change".to_owned()),
15459 });
15460 state.reviews = vec![ReviewRound {
15461 round: 1,
15462 head: "deadbeef".to_owned(),
15463 verified_head: None,
15464 verified_at: None,
15465 reviews: Vec::new(),
15466 e2e: Vec::new(),
15467 fix: None,
15468 blocking: 0,
15469 answered: 0,
15470 expected: 0,
15471 clean: true,
15472 verify_retried: false,
15473 e2e_deferred: false,
15474 e2e_defer_reason: None,
15475 progressed: false,
15476 vote_split: false,
15477 reconsideration: Vec::new(),
15478 verdict: None,
15479 }];
15480 state.gate = vec![CommandOutcome {
15481 command: "test".to_owned(),
15482 code: Some(0),
15483 output_tail: String::new(),
15484 duration_ms: 0,
15485 resource_blocked: false,
15486 }];
15487 state.gate_ran = true;
15488 state.status = RunStatus::Ready;
15493 state.merge = Some(MergeOutcome {
15494 mode: MergeMode::Local,
15495 ok: false,
15496 detail: "already concluded".to_owned(),
15497 empty: false,
15498 });
15499
15500 let mut runner = Runner {
15501 state,
15502 roles: ResolvedRoles {
15503 implementers: Vec::new(),
15504 judges: Vec::new(),
15505 reviewers: Vec::new(),
15506 fixer: None,
15507 conductor: conductor(),
15508 implementer_roster: Vec::new(),
15509 judge_roster: Vec::new(),
15510 reviewer_roster: Vec::new(),
15511 },
15512 sem: Arc::new(Semaphore::new(1)),
15513 pause: Pause::new(),
15514 interrupt: Pause::new(),
15515 };
15516
15517 runner.merge().await.expect("merge");
15518
15519 assert_eq!(
15520 runner.state.status,
15521 RunStatus::Ready,
15522 "a concluded run's status must not change on reentry"
15523 );
15524 assert_eq!(
15525 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
15526 Some("already concluded"),
15527 "merge must not run again once the node already recorded an outcome"
15528 );
15529 }
15530
15531 #[tokio::test]
15540 async fn merge_refuses_a_gate_that_has_not_actually_run() {
15541 let tmp = tempfile::tempdir().expect("tempdir");
15542 let repo = tmp.path().join("repo");
15543 std::fs::create_dir_all(&repo).unwrap();
15544 init_repo(&repo);
15545
15546 let mut config = Config::default();
15547 config.merge.mode = MergeMode::Local;
15548
15549 let mut state = RunState::new(
15550 repo.clone(),
15551 "main".to_owned(),
15552 "deadbeef".to_owned(),
15553 "task".to_owned(),
15554 config,
15555 );
15556 state.candidates = vec![Candidate {
15557 index: 0,
15558 label: 'A',
15559 agent: "alpha".to_owned(),
15560 branch: "does-not-exist".to_owned(),
15561 worktree: repo.clone(),
15562 summary: String::new(),
15563 stat: String::new(),
15564 files: 0,
15565 commits: 0,
15566 empty: false,
15567 failed: None,
15568 verified_noop: None,
15569 duration_ms: 0,
15570 folded: false,
15571 }];
15572 state.tally = Some(Tally {
15573 first_choice: BTreeMap::from([('A', 1)]),
15574 borda: BTreeMap::new(),
15575 winner: 'A',
15576 rankings: 1,
15577 unanimous_initial: true,
15578 deliberated: false,
15579 changed_votes: 0,
15580 unanimous_final: true,
15581 tie_break: None,
15582 judges: 0,
15583 present: 0,
15584 quorum: 0,
15585 met_quorum: true,
15586 uncontested: Some("only candidate A produced a change".to_owned()),
15587 });
15588 state.reviews = vec![ReviewRound {
15589 round: 1,
15590 head: "deadbeef".to_owned(),
15591 verified_head: None,
15592 verified_at: None,
15593 reviews: Vec::new(),
15594 e2e: Vec::new(),
15595 fix: None,
15596 blocking: 0,
15597 answered: 0,
15598 expected: 0,
15599 clean: true,
15600 verify_retried: false,
15601 e2e_deferred: false,
15602 e2e_defer_reason: None,
15603 progressed: false,
15604 vote_split: false,
15605 reconsideration: Vec::new(),
15606 verdict: None,
15607 }];
15608 state.gate = Vec::new();
15610 state.gate_ran = false;
15611 state.status = RunStatus::Gating;
15612
15613 let mut runner = Runner {
15614 state,
15615 roles: ResolvedRoles {
15616 implementers: Vec::new(),
15617 judges: Vec::new(),
15618 reviewers: Vec::new(),
15619 fixer: None,
15620 conductor: conductor(),
15621 implementer_roster: Vec::new(),
15622 judge_roster: Vec::new(),
15623 reviewer_roster: Vec::new(),
15624 },
15625 sem: Arc::new(Semaphore::new(1)),
15626 pause: Pause::new(),
15627 interrupt: Pause::new(),
15628 };
15629
15630 runner.merge().await.expect("merge");
15631
15632 assert!(
15633 runner.state.merge.is_none(),
15634 "an empty gate must never be read as a passing one: {:?}",
15635 runner.state.merge
15636 );
15637 }
15638
15639 #[tokio::test]
15646 async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
15647 let tmp = tempfile::tempdir().expect("tempdir");
15648 let repo = tmp.path().join("repo");
15649 std::fs::create_dir_all(&repo).unwrap();
15650 init_repo(&repo);
15651
15652 let config = Config::default();
15654
15655 let mut state = RunState::new(
15656 repo.clone(),
15657 "main".to_owned(),
15658 "deadbeef".to_owned(),
15659 "task".to_owned(),
15660 config,
15661 );
15662 state.candidates = vec![Candidate {
15663 index: 0,
15664 label: 'A',
15665 agent: "alpha".to_owned(),
15666 branch: "does-not-exist".to_owned(),
15667 worktree: repo.clone(),
15668 summary: String::new(),
15669 stat: String::new(),
15670 files: 0,
15671 commits: 0,
15672 empty: false,
15673 failed: None,
15674 verified_noop: None,
15675 duration_ms: 0,
15676 folded: false,
15677 }];
15678 state.tally = Some(Tally {
15679 first_choice: BTreeMap::from([('A', 1)]),
15680 borda: BTreeMap::new(),
15681 winner: 'A',
15682 rankings: 1,
15683 unanimous_initial: true,
15684 deliberated: false,
15685 changed_votes: 0,
15686 unanimous_final: true,
15687 tie_break: None,
15688 judges: 0,
15689 present: 0,
15690 quorum: 0,
15691 met_quorum: true,
15692 uncontested: Some("only candidate A produced a change".to_owned()),
15693 });
15694 state.reviews = vec![ReviewRound {
15695 round: 1,
15696 head: "deadbeef".to_owned(),
15697 verified_head: None,
15698 verified_at: None,
15699 reviews: Vec::new(),
15700 e2e: Vec::new(),
15701 fix: None,
15702 blocking: 0,
15703 answered: 0,
15704 expected: 0,
15705 clean: true,
15706 verify_retried: false,
15707 e2e_deferred: false,
15708 e2e_defer_reason: None,
15709 progressed: false,
15710 vote_split: false,
15711 reconsideration: Vec::new(),
15712 verdict: None,
15713 }];
15714
15715 let mut runner = Runner {
15716 state,
15717 roles: ResolvedRoles {
15718 implementers: Vec::new(),
15719 judges: Vec::new(),
15720 reviewers: Vec::new(),
15721 fixer: None,
15722 conductor: conductor(),
15723 implementer_roster: Vec::new(),
15724 judge_roster: Vec::new(),
15725 reviewer_roster: Vec::new(),
15726 },
15727 sem: Arc::new(Semaphore::new(1)),
15728 pause: Pause::new(),
15729 interrupt: Pause::new(),
15730 };
15731
15732 runner.gate().await.expect("gate");
15733 assert!(
15734 runner.state.gate_ran,
15735 "zero configured commands is still a real attempt, not an unrun gate"
15736 );
15737 assert!(runner.state.gate.is_empty());
15738 assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
15739 assert_ne!(
15740 runner.state.status,
15741 RunStatus::Blocked,
15742 "a gate with nothing to check must not read as failed"
15743 );
15744
15745 runner.merge().await.expect("merge");
15746 assert_eq!(
15747 runner.state.status,
15748 RunStatus::Ready,
15749 "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
15750 );
15751 }
15752
15753 #[tokio::test]
15764 async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
15765 crate::run::pin_test_home();
15766 let home = crate::run::home();
15767
15768 let tmp = tempfile::tempdir().expect("tempdir");
15769 let repo = tmp.path().join("repo");
15770 std::fs::create_dir_all(&repo).unwrap();
15771 init_repo(&repo);
15772 let cache_dir = tmp.path().join("target");
15775
15776 let mut config = Config::default();
15777 config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
15778 config.graph.timeout_verify = Some(2);
15781 config.graph.timeout_resource_wait = 2;
15782
15783 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
15784 let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
15785 .expect("no io error acquiring directly")
15786 {
15787 crate::cache::AcquireOutcome::Acquired(g) => g,
15788 crate::cache::AcquireOutcome::Busy(b) => {
15789 panic!("expected the direct acquire to win the lease first: {b:?}")
15790 }
15791 };
15792
15793 let mut state = RunState::new(
15794 repo.clone(),
15795 "main".to_owned(),
15796 "deadbeef".to_owned(),
15797 "task".to_owned(),
15798 config,
15799 );
15800 state.candidates = vec![Candidate {
15801 index: 0,
15802 label: 'A',
15803 agent: "alpha".to_owned(),
15804 branch: "does-not-exist".to_owned(),
15805 worktree: repo.clone(),
15806 summary: String::new(),
15807 stat: String::new(),
15808 files: 0,
15809 commits: 0,
15810 empty: false,
15811 failed: None,
15812 verified_noop: None,
15813 duration_ms: 0,
15814 folded: false,
15815 }];
15816 state.tally = Some(Tally {
15817 first_choice: BTreeMap::from([('A', 1)]),
15818 borda: BTreeMap::new(),
15819 winner: 'A',
15820 rankings: 1,
15821 unanimous_initial: true,
15822 deliberated: false,
15823 changed_votes: 0,
15824 unanimous_final: true,
15825 tie_break: None,
15826 judges: 0,
15827 present: 0,
15828 quorum: 0,
15829 met_quorum: true,
15830 uncontested: Some("only candidate A produced a change".to_owned()),
15831 });
15832 state.reviews = vec![ReviewRound {
15833 round: 1,
15834 head: "deadbeef".to_owned(),
15835 verified_head: None,
15836 verified_at: None,
15837 reviews: Vec::new(),
15838 e2e: Vec::new(),
15839 fix: None,
15840 blocking: 0,
15841 answered: 0,
15842 expected: 0,
15843 clean: true,
15844 verify_retried: false,
15845 e2e_deferred: false,
15846 e2e_defer_reason: None,
15847 progressed: false,
15848 vote_split: false,
15849 reconsideration: Vec::new(),
15850 verdict: None,
15851 }];
15852
15853 let mut runner = Runner {
15854 state,
15855 roles: ResolvedRoles {
15856 implementers: Vec::new(),
15857 judges: Vec::new(),
15858 reviewers: Vec::new(),
15859 fixer: None,
15860 conductor: conductor(),
15861 implementer_roster: Vec::new(),
15862 judge_roster: Vec::new(),
15863 reviewer_roster: Vec::new(),
15864 },
15865 sem: Arc::new(Semaphore::new(1)),
15866 pause: Pause::new(),
15867 interrupt: Pause::new(),
15868 };
15869
15870 let started = std::time::Instant::now();
15871 runner.gate().await.expect("gate");
15872 assert!(
15873 started.elapsed() < Duration::from_secs(1),
15874 "a gate with nothing to run must never wait on a lease it never needed"
15875 );
15876 assert!(
15877 runner.state.gate_ran,
15878 "zero commands is still a real, immediate attempt"
15879 );
15880 assert!(runner.state.gate.is_empty());
15881 assert_ne!(
15882 runner.state.status,
15883 RunStatus::Blocked,
15884 "must not read as resource-blocked on a lease it never asked for"
15885 );
15886 }
15887
15888 #[cfg(unix)]
15889 #[tokio::test]
15890 async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
15891 crate::run::pin_test_home();
15892 let tmp = tempfile::tempdir().unwrap();
15893 let mut config = Config::default();
15894 config.disk.min_free_bytes = 0;
15895 let mut state = RunState::new(
15896 tmp.path().to_owned(),
15897 "main".into(),
15898 "base".into(),
15899 "verification evidence".into(),
15900 config,
15901 );
15902 let (outcomes, pids) = run_commands(
15903 &mut state,
15904 "gate",
15905 "gate",
15906 1,
15907 &["sh".into(), "-c".into()],
15908 &[
15909 "printf before-timeout; exec sleep 30".into(),
15910 "kill -TERM $$".into(),
15911 "exit 7".into(),
15912 "printf success".into(),
15913 ],
15914 tmp.path(),
15915 Duration::from_secs(1),
15916 )
15917 .await;
15918 wait_for_timed_out_children_to_die(&pids).await;
15919 assert_eq!(
15920 pids.len(),
15921 1,
15922 "unleased timeout retains its death-check PID"
15923 );
15924 assert_eq!(outcomes.len(), 4);
15925 assert_eq!(outcomes[0].code, None);
15926 assert!(outcomes[0].output_tail.contains("before-timeout"));
15927 assert!(outcomes[0].output_tail.contains("timed out after 1s"));
15928 assert_eq!(outcomes[1].code, None);
15929 assert!(outcomes[1].output_tail.contains("command terminated:"));
15930 assert!(!outcomes[1].output_tail.contains("timed out"));
15931 assert_eq!(outcomes[2].code, Some(7));
15932 assert!(!outcomes[2].output_tail.contains("timed out"));
15933 assert_eq!(outcomes[3].code, Some(0));
15934 assert_eq!(outcomes[3].output_tail, "success");
15935 assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
15936 }
15937
15938 #[tokio::test]
15948 async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
15949 crate::run::pin_test_home();
15950
15951 let tmp = tempfile::tempdir().expect("tempdir");
15952 let repo = tmp.path().join("repo");
15953 std::fs::create_dir_all(&repo).unwrap();
15954 init_repo(&repo);
15955
15956 let mut config = Config::default();
15957 config.verify.gate = vec![
15958 "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
15959 [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
15960 .to_owned(),
15961 ];
15962
15963 let mut state = RunState::new(
15964 repo.clone(),
15965 "main".to_owned(),
15966 "deadbeef".to_owned(),
15967 "task".to_owned(),
15968 config,
15969 );
15970 let run_id = state.id.clone();
15971 state.candidates = vec![Candidate {
15972 index: 0,
15973 label: 'A',
15974 agent: "alpha".to_owned(),
15975 branch: "does-not-exist".to_owned(),
15976 worktree: repo.clone(),
15977 summary: String::new(),
15978 stat: String::new(),
15979 files: 0,
15980 commits: 0,
15981 empty: false,
15982 failed: None,
15983 verified_noop: None,
15984 duration_ms: 0,
15985 folded: false,
15986 }];
15987 state.tally = Some(Tally {
15988 first_choice: BTreeMap::from([('A', 1)]),
15989 borda: BTreeMap::new(),
15990 winner: 'A',
15991 rankings: 1,
15992 unanimous_initial: true,
15993 deliberated: false,
15994 changed_votes: 0,
15995 unanimous_final: true,
15996 tie_break: None,
15997 judges: 0,
15998 present: 0,
15999 quorum: 0,
16000 met_quorum: true,
16001 uncontested: Some("only candidate A produced a change".to_owned()),
16002 });
16003 state.reviews = vec![ReviewRound {
16004 round: 1,
16005 head: "deadbeef".to_owned(),
16006 verified_head: None,
16007 verified_at: None,
16008 reviews: Vec::new(),
16009 e2e: Vec::new(),
16010 fix: None,
16011 blocking: 0,
16012 answered: 0,
16013 expected: 0,
16014 clean: true,
16015 verify_retried: false,
16016 e2e_deferred: false,
16017 e2e_defer_reason: None,
16018 progressed: false,
16019 vote_split: false,
16020 reconsideration: Vec::new(),
16021 verdict: None,
16022 }];
16023
16024 let mut runner = Runner {
16025 state,
16026 roles: ResolvedRoles {
16027 implementers: Vec::new(),
16028 judges: Vec::new(),
16029 reviewers: Vec::new(),
16030 fixer: None,
16031 conductor: conductor(),
16032 implementer_roster: Vec::new(),
16033 judge_roster: Vec::new(),
16034 reviewer_roster: Vec::new(),
16035 },
16036 sem: Arc::new(Semaphore::new(1)),
16037 pause: Pause::new(),
16038 interrupt: Pause::new(),
16039 };
16040
16041 let started_marker = repo.join("started.marker");
16042 let release_marker = repo.join("release.marker");
16043 let poller = tokio::spawn(async move {
16044 for _ in 0..100 {
16049 if started_marker.exists()
16050 && let Ok(s) = crate::run::RunState::load(&run_id)
16051 && let Some(a) = s.active.get("gate")
16052 {
16053 std::fs::write(&release_marker, b"go").expect("release marker");
16054 return Some(a.clone());
16055 }
16056 tokio::time::sleep(Duration::from_millis(50)).await;
16057 }
16058 None
16059 });
16060
16061 runner.gate().await.expect("gate");
16062 let captured = poller.await.expect("poller task");
16063 let captured = captured.expect(
16064 "the poller never saw a `gate` task entry in run.json while the command was \
16065 still blocked on its own release marker",
16066 );
16067
16068 assert_eq!(captured.task.as_deref(), Some("gate"));
16069 assert_eq!(captured.node, "gate");
16070 assert_eq!(captured.index, Some(1));
16071 assert_eq!(captured.total, Some(1));
16072 assert!(
16073 captured
16074 .command
16075 .as_deref()
16076 .is_some_and(|c| c.contains("started.marker")),
16077 "{captured:?}"
16078 );
16079
16080 assert!(
16081 runner.state.active.is_empty(),
16082 "the entry must be cleared once the command actually finished: {:?}",
16083 runner.state.active
16084 );
16085 assert!(runner.state.gate_ran);
16086 assert!(runner.state.gate.iter().all(CommandOutcome::ok));
16087 }
16088
16089 #[tokio::test]
16092 async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
16093 use crate::verdict::{Finding, ReviewVote, Severity};
16094 crate::run::pin_test_home();
16095 let tmp = tempfile::tempdir().expect("tempdir");
16096 let repo = tmp.path().join("repo");
16097 std::fs::create_dir_all(&repo).unwrap();
16098 init_repo(&repo);
16099
16100 for (severity, vote, expect) in [
16101 (Severity::Major, ReviewVote::Reject, true),
16102 (Severity::Minor, ReviewVote::Reject, false),
16103 (Severity::Major, ReviewVote::Approve, false),
16104 ] {
16105 let mut round = review_round(false, 1, 1, 1, false, true);
16106 round.reviews = vec![ReviewRecord {
16107 reviewer: 1,
16108 agent: "alpha".to_owned(),
16109 summary: String::new(),
16110 findings: vec![Finding {
16111 id: "R1-1-1".to_owned(),
16112 severity,
16113 file: None,
16114 line: None,
16115 title: "t".to_owned(),
16116 detail: String::new(),
16117 }],
16118 vote: Some(vote),
16119 failed: None,
16120 duration_ms: 0,
16121 attempts: 0,
16122 }];
16123 let mut state = RunState::new(
16124 repo.clone(),
16125 "main".to_owned(),
16126 "deadbeef".to_owned(),
16127 "task".to_owned(),
16128 Config::default(),
16129 );
16130 state.reviews = vec![round];
16131 let mut runner = Runner {
16132 state,
16133 roles: ResolvedRoles {
16134 implementers: Vec::new(),
16135 judges: Vec::new(),
16136 reviewers: Vec::new(),
16137 fixer: None,
16138 conductor: conductor(),
16139 implementer_roster: Vec::new(),
16140 judge_roster: Vec::new(),
16141 reviewer_roster: Vec::new(),
16142 },
16143 sem: Arc::new(Semaphore::new(1)),
16144 pause: Pause::new(),
16145 interrupt: Pause::new(),
16146 };
16147 let shell = runner.state.config.shell();
16148 runner
16149 .stop_reviewing("round budget spent", &shell, &repo)
16150 .await
16151 .expect("stop_reviewing");
16152 assert_eq!(runner.state.status, RunStatus::Gating);
16153 assert_eq!(
16154 runner.state.contested_handoff.is_some(),
16155 expect,
16156 "{severity:?} + {vote:?}"
16157 );
16158 }
16159 }
16160
16161 #[tokio::test]
16174 async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
16175 crate::run::pin_test_home();
16176 let home = crate::run::home();
16177
16178 let tmp = tempfile::tempdir().expect("tempdir");
16179 let repo = tmp.path().join("repo");
16180 std::fs::create_dir_all(&repo).unwrap();
16181 init_repo(&repo);
16182 let head = crate::git::rev_parse(&repo, "HEAD")
16183 .await
16184 .expect("rev-parse");
16185 let cache_dir = tmp.path().join("target");
16188
16189 let mut config = Config::default();
16190 config.verify.e2e = vec![format!(
16191 "CARGO_TARGET_DIR='{}' test -f README.md",
16192 cache_dir.display()
16193 )];
16194 config.graph.review_rounds = 1;
16195 config.graph.timeout_verify = Some(2);
16198 config.graph.timeout_resource_wait = 2;
16199
16200 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16201 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16202 .expect("no io error acquiring directly")
16203 {
16204 crate::cache::AcquireOutcome::Acquired(g) => g,
16205 crate::cache::AcquireOutcome::Busy(b) => {
16206 panic!("expected the direct acquire to win the lease first: {b:?}")
16207 }
16208 };
16209
16210 let mut state = RunState::new(
16211 repo.clone(),
16212 "main".to_owned(),
16213 head.clone(),
16214 "task".to_owned(),
16215 config,
16216 );
16217 state.candidates = vec![Candidate {
16218 index: 0,
16219 label: 'A',
16220 agent: "alpha".to_owned(),
16221 branch: "does-not-exist".to_owned(),
16222 worktree: repo.clone(),
16223 summary: String::new(),
16224 stat: String::new(),
16225 files: 0,
16226 commits: 0,
16227 empty: false,
16228 failed: None,
16229 verified_noop: None,
16230 duration_ms: 0,
16231 folded: false,
16232 }];
16233 state.tally = Some(Tally {
16234 first_choice: BTreeMap::from([('A', 1)]),
16235 borda: BTreeMap::new(),
16236 winner: 'A',
16237 rankings: 1,
16238 unanimous_initial: true,
16239 deliberated: false,
16240 changed_votes: 0,
16241 unanimous_final: true,
16242 tie_break: None,
16243 judges: 0,
16244 present: 0,
16245 quorum: 0,
16246 met_quorum: true,
16247 uncontested: Some("only candidate A produced a change".to_owned()),
16248 });
16249 state.reviews = vec![ReviewRound {
16253 round: 1,
16254 head: head.clone(),
16255 verified_head: None,
16256 verified_at: None,
16257 reviews: Vec::new(),
16258 e2e: Vec::new(),
16259 fix: None,
16260 blocking: 1,
16261 answered: 1,
16262 expected: 1,
16263 clean: false,
16264 verify_retried: false,
16265 e2e_deferred: true,
16266 e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
16267 progressed: false,
16268 vote_split: false,
16269 reconsideration: Vec::new(),
16270 verdict: None,
16271 }];
16272
16273 let mut runner = Runner {
16274 state,
16275 roles: ResolvedRoles {
16276 implementers: Vec::new(),
16277 judges: Vec::new(),
16278 reviewers: Vec::new(),
16279 fixer: None,
16280 conductor: conductor(),
16281 implementer_roster: Vec::new(),
16282 judge_roster: Vec::new(),
16283 reviewer_roster: Vec::new(),
16284 },
16285 sem: Arc::new(Semaphore::new(1)),
16286 pause: Pause::new(),
16287 interrupt: Pause::new(),
16288 };
16289
16290 let shell = runner.state.config.shell();
16291 runner
16292 .stop_reviewing("round budget spent", &shell, &repo)
16293 .await
16294 .expect("stop_reviewing");
16295
16296 let last = runner.state.reviews.last().expect("round record");
16297 assert_eq!(
16298 last.e2e_status(),
16299 E2eStatus::ResourceBlocked,
16300 "the shared cache is still held; the attempt must read as blocked, not deferred or \
16301 failed: {last:?}"
16302 );
16303 assert_eq!(
16304 last.verified_head.as_deref(),
16305 Some(head.as_str()),
16306 "which commit this attempt targeted is known even though nothing finished checking \
16307 it"
16308 );
16309 let first_attempt_at = last
16310 .verified_at
16311 .expect("when this attempt ran is known too");
16312 assert_ne!(
16313 runner.state.status,
16314 RunStatus::Blocked,
16315 "contention is evidence about the machine, not the patch — it must not settle the \
16316 run as blocked: {:?}",
16317 runner.state.status
16318 );
16319 assert!(
16320 !runner
16321 .state
16322 .events
16323 .iter()
16324 .any(|e| e.node == "review" && e.message.contains("e2e failed")),
16325 "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
16326 runner.state.events
16327 );
16328
16329 runner
16334 .stop_reviewing("round budget spent", &shell, &repo)
16335 .await
16336 .expect("stop_reviewing retry");
16337 assert_eq!(
16338 runner.state.reviews.len(),
16339 1,
16340 "no new round was started: {:?}",
16341 runner.state.reviews
16342 );
16343 let last = runner.state.reviews.last().expect("round record");
16344 assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
16345 assert!(
16346 last.verified_at.expect("still known") > first_attempt_at,
16347 "a second reentry must be a fresh attempt, not a stale copy of the first"
16348 );
16349 assert_ne!(runner.state.status, RunStatus::Blocked);
16350
16351 held.release();
16352 }
16353
16354 #[tokio::test]
16366 async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
16367 crate::run::pin_test_home();
16368 let home = crate::run::home();
16369
16370 let tmp = tempfile::tempdir().expect("tempdir");
16371 let repo = tmp.path().join("repo");
16372 std::fs::create_dir_all(&repo).unwrap();
16373 init_repo(&repo);
16374 let head = crate::git::rev_parse(&repo, "HEAD")
16375 .await
16376 .expect("rev-parse");
16377 let cache_dir = tmp.path().join("target");
16378
16379 let mut config = Config::default();
16380 config.verify.e2e = vec![format!(
16381 "CARGO_TARGET_DIR='{}' test -f README.md",
16382 cache_dir.display()
16383 )];
16384 config.graph.review_rounds = 1;
16385 config.graph.timeout_verify = Some(2);
16386 config.graph.timeout_resource_wait = 2;
16387
16388 let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
16389 let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
16390 .expect("no io error acquiring directly")
16391 {
16392 crate::cache::AcquireOutcome::Acquired(g) => g,
16393 crate::cache::AcquireOutcome::Busy(b) => {
16394 panic!("expected the direct acquire to win the lease first: {b:?}")
16395 }
16396 };
16397
16398 let mut state = RunState::new(
16399 repo.clone(),
16400 "main".to_owned(),
16401 head.clone(),
16402 "task".to_owned(),
16403 config,
16404 );
16405 state.candidates = vec![Candidate {
16406 index: 0,
16407 label: 'A',
16408 agent: "alpha".to_owned(),
16409 branch: "does-not-exist".to_owned(),
16410 worktree: repo.clone(),
16411 summary: String::new(),
16412 stat: String::new(),
16413 files: 0,
16414 commits: 0,
16415 empty: false,
16416 failed: None,
16417 verified_noop: None,
16418 duration_ms: 0,
16419 folded: false,
16420 }];
16421 state.tally = Some(Tally {
16422 first_choice: BTreeMap::from([('A', 1)]),
16423 borda: BTreeMap::new(),
16424 winner: 'A',
16425 rankings: 1,
16426 unanimous_initial: true,
16427 deliberated: false,
16428 changed_votes: 0,
16429 unanimous_final: true,
16430 tie_break: None,
16431 judges: 0,
16432 present: 0,
16433 quorum: 0,
16434 met_quorum: true,
16435 uncontested: Some("only candidate A produced a change".to_owned()),
16436 });
16437 state.reviews = vec![ReviewRound {
16441 round: 1,
16442 head: head.clone(),
16443 verified_head: Some(head.clone()),
16444 verified_at: Some(jiff::Timestamp::now()),
16445 reviews: Vec::new(),
16446 e2e: vec![CommandOutcome {
16447 command: format!(
16448 "CARGO_TARGET_DIR='{}' test -f README.md",
16449 cache_dir.display()
16450 ),
16451 code: None,
16452 output_tail: "waiting for the shared build cache".to_owned(),
16453 duration_ms: 0,
16454 resource_blocked: true,
16455 }],
16456 fix: None,
16457 blocking: 1,
16458 answered: 1,
16459 expected: 1,
16460 clean: false,
16461 verify_retried: false,
16462 e2e_deferred: false,
16463 e2e_defer_reason: None,
16464 progressed: false,
16465 vote_split: false,
16466 reconsideration: Vec::new(),
16467 verdict: None,
16468 }];
16469
16470 let first_attempt_at = state.reviews[0].verified_at.expect("set above");
16471 let mut runner = Runner {
16472 state,
16473 roles: ResolvedRoles {
16474 implementers: Vec::new(),
16475 judges: Vec::new(),
16476 reviewers: Vec::new(),
16477 fixer: None,
16478 conductor: conductor(),
16479 implementer_roster: Vec::new(),
16480 judge_roster: Vec::new(),
16481 reviewer_roster: Vec::new(),
16482 },
16483 sem: Arc::new(Semaphore::new(1)),
16484 pause: Pause::new(),
16485 interrupt: Pause::new(),
16486 };
16487
16488 runner.review_loop().await.expect("review_loop");
16493
16494 assert_eq!(
16495 runner.state.reviews.len(),
16496 1,
16497 "no new round was started on top of the unresolved one: {:?}",
16498 runner.state.reviews
16499 );
16500 let last = &runner.state.reviews[0];
16501 assert_eq!(
16502 last.e2e_status(),
16503 E2eStatus::ResourceBlocked,
16504 "still contended: {last:?}"
16505 );
16506 assert!(
16507 last.verified_at.expect("still known") > first_attempt_at,
16508 "review_loop must have actually retried the check, not left it exactly as found"
16509 );
16510 assert_ne!(
16511 runner.state.status,
16512 RunStatus::Blocked,
16513 "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
16514 runner.state.status
16515 );
16516
16517 held.release();
16518 }
16519
16520 #[tokio::test]
16521 async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
16522 crate::run::pin_test_home();
16523 for status in [RunStatus::Blocked, RunStatus::Gating] {
16524 let mut runner = runner_at(status);
16525 runner.state.config.graph.land = true;
16526 runner.state.gate_ran = true;
16527 runner.state.merge = Some(MergeOutcome {
16528 mode: MergeMode::Pr,
16529 ok: false,
16530 detail: "the fixer produced no commit".into(),
16531 empty: false,
16532 });
16533 let url = "https://example.invalid/o/r/pull/1";
16534 runner.state.pr = Some(crate::run::PrRecord {
16535 url: url.into(),
16536 number: 1,
16537 state: "open".into(),
16538 checks: "red".into(),
16539 round: 1,
16540 rounds: 1,
16541 red_at_merge: Vec::new(),
16542 });
16543 runner.state.save().unwrap();
16544 runner.execute_graph().await.unwrap();
16548 assert_eq!(runner.state.status, RunStatus::Blocked);
16549 assert!(runner.state.land_sync_pending);
16550 assert!(runner.state.events.iter().any(|event|
16551 event.node == "land" && event.message == format!("watching {url}")));
16552 assert_eq!(
16553 runner.state.merge.as_ref().unwrap().detail,
16554 "the fixer produced no commit"
16555 );
16556 }
16557 }
16558
16559 #[tokio::test]
16560 async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
16561 crate::run::pin_test_home();
16562 let tmp = tempfile::tempdir().expect("tempdir");
16563 let repo = tmp.path().join("repo");
16564 std::fs::create_dir_all(&repo).unwrap();
16565 init_repo(&repo);
16566
16567 let mut config = Config::default();
16568 config.merge.mode = MergeMode::Pr;
16569 config.graph.land = true;
16570 config.graph.land_approval = false;
16571
16572 let mut state = RunState::new(
16573 repo.clone(),
16574 "main".to_owned(),
16575 "deadbeef".to_owned(),
16576 "task".to_owned(),
16577 config,
16578 );
16579 state.candidates = vec![Candidate {
16580 index: 0,
16581 label: 'A',
16582 agent: "alpha".to_owned(),
16583 branch: "does-not-exist".to_owned(),
16584 worktree: repo.clone(),
16585 summary: String::new(),
16586 stat: String::new(),
16587 files: 0,
16588 commits: 0,
16589 empty: false,
16590 failed: None,
16591 verified_noop: None,
16592 duration_ms: 0,
16593 folded: false,
16594 }];
16595 state.tally = Some(Tally {
16596 first_choice: BTreeMap::from([('A', 1)]),
16597 borda: BTreeMap::new(),
16598 winner: 'A',
16599 rankings: 1,
16600 unanimous_initial: true,
16601 deliberated: false,
16602 changed_votes: 0,
16603 unanimous_final: true,
16604 tie_break: None,
16605 judges: 0,
16606 present: 0,
16607 quorum: 0,
16608 met_quorum: true,
16609 uncontested: Some("only candidate A produced a change".to_owned()),
16610 });
16611 state.reviews = vec![ReviewRound {
16612 round: 1,
16613 head: "deadbeef".to_owned(),
16614 verified_head: None,
16615 verified_at: None,
16616 reviews: Vec::new(),
16617 e2e: Vec::new(),
16618 fix: None,
16619 blocking: 0,
16620 answered: 0,
16621 expected: 0,
16622 clean: true,
16623 verify_retried: false,
16624 e2e_deferred: false,
16625 e2e_defer_reason: None,
16626 progressed: false,
16627 vote_split: false,
16628 reconsideration: Vec::new(),
16629 verdict: None,
16630 }];
16631 state.gate = vec![CommandOutcome {
16632 command: "test".to_owned(),
16633 code: Some(0),
16634 output_tail: String::new(),
16635 duration_ms: 0,
16636 resource_blocked: false,
16637 }];
16638 state.gate_ran = true;
16639 state.status = RunStatus::Landing;
16643 state.merge = Some(MergeOutcome {
16644 mode: MergeMode::Pr,
16645 ok: true,
16646 detail: "https://example.invalid/x/y/pull/1".to_owned(),
16647 empty: false,
16648 });
16649
16650 ask_test_home();
16654 state.save().expect("save landing run");
16657 let store = ask::Questions::open();
16658 let q = ask_open_question(&store, &state.id);
16659
16660 let mut runner = Runner {
16661 state,
16662 roles: ResolvedRoles {
16663 implementers: Vec::new(),
16664 judges: Vec::new(),
16665 reviewers: Vec::new(),
16666 fixer: None,
16667 conductor: conductor(),
16668 implementer_roster: Vec::new(),
16669 judge_roster: Vec::new(),
16670 reviewer_roster: Vec::new(),
16671 },
16672 sem: Arc::new(Semaphore::new(1)),
16673 pause: Pause::new(),
16674 interrupt: Pause::new(),
16675 };
16676
16677 runner.execute().await.expect("execute");
16682
16683 assert_eq!(
16684 runner.state.merge.as_ref().map(|m| m.detail.as_str()),
16685 Some("https://example.invalid/x/y/pull/1"),
16686 "reentry must not push again or open a second pull request over the \
16687 one `land` is already watching"
16688 );
16689 assert_ne!(
16690 runner.state.status,
16691 RunStatus::Landing,
16692 "land could not actually reach the fake pull request, so it must \
16693 have given up rather than left the run silently parked forever"
16694 );
16695 assert_eq!(runner.state.status, RunStatus::Blocked);
16699 assert!(
16700 store.get(&q.id).unwrap().status.open(),
16701 "Blocked is still alive; settle_questions must have been a no-op here"
16702 );
16703 }
16704
16705 fn state_with_round(round: ReviewRound) -> RunState {
16706 let mut s = RunState::new(
16707 PathBuf::from("/repo"),
16708 "main".to_owned(),
16709 "abc1234".to_owned(),
16710 "add retries".to_owned(),
16711 Config::default(),
16712 );
16713 s.reviews = vec![round];
16714 s
16715 }
16716
16717 fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
16718 crate::verdict::Finding {
16719 id: id.to_owned(),
16720 severity,
16721 file: None,
16722 line: None,
16723 title: title.to_owned(),
16724 detail: String::new(),
16725 }
16726 }
16727
16728 #[test]
16729 fn pr_body_names_open_findings_and_declined_ones() {
16730 let round = ReviewRound {
16731 round: 2,
16732 head: "deadbee".to_owned(),
16733 verified_head: None,
16734 verified_at: None,
16735 reviews: vec![ReviewRecord {
16736 attempts: 0,
16737 reviewer: 1,
16738 agent: "alpha".to_owned(),
16739 summary: String::new(),
16740 findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
16741 vote: None,
16742 failed: None,
16743 duration_ms: 0,
16744 }],
16745 e2e: vec![CommandOutcome {
16746 command: "cargo test".to_owned(),
16747 code: Some(0),
16748 output_tail: String::new(),
16749 duration_ms: 0,
16750 resource_blocked: false,
16751 }],
16752 verify_retried: false,
16753 e2e_deferred: false,
16754 e2e_defer_reason: None,
16755 fix: Some(FixRecord {
16756 agent: "alpha".to_owned(),
16757 addressed: Vec::new(),
16758 rejected: vec![crate::verdict::Rejection {
16759 id: "R1-1-1".to_owned(),
16760 why: "not reachable from any caller".to_owned(),
16761 }],
16762 notes: String::new(),
16763 committed: true,
16764 failed: None,
16765 duration_ms: 0,
16766 continuation: None,
16767 }),
16768 blocking: 0,
16769 answered: 1,
16770 expected: 1,
16771 clean: false,
16772 progressed: true,
16773 vote_split: false,
16774 reconsideration: Vec::new(),
16775 verdict: None,
16776 };
16777 let state = state_with_round(round);
16778 let body = pr_message(&state, 'A').body;
16779
16780 assert!(body.contains("add retries"), "the task must still be there");
16781 assert!(body.contains("R2-1-1"), "{body}");
16782 assert!(!body.contains("unused import"), "{body}");
16783 assert!(body.contains("R1-1-1"), "the declined finding: {body}");
16784 assert!(
16785 !body.contains("not reachable from any caller"),
16786 "the reason it was declined: {body}"
16787 );
16788 }
16789
16790 #[test]
16791 fn pr_body_says_nothing_extra_when_the_round_was_clean() {
16792 let round = ReviewRound {
16793 round: 1,
16794 head: "deadbee".to_owned(),
16795 verified_head: None,
16796 verified_at: None,
16797 reviews: vec![ReviewRecord {
16798 attempts: 0,
16799 reviewer: 1,
16800 agent: "alpha".to_owned(),
16801 summary: String::new(),
16802 findings: Vec::new(),
16803 vote: None,
16804 failed: None,
16805 duration_ms: 0,
16806 }],
16807 e2e: Vec::new(),
16808 verify_retried: false,
16809 e2e_deferred: false,
16810 e2e_defer_reason: None,
16811 fix: None,
16812 blocking: 0,
16813 answered: 1,
16814 expected: 1,
16815 clean: true,
16816 progressed: false,
16817 vote_split: false,
16818 reconsideration: Vec::new(),
16819 verdict: None,
16820 };
16821 let state = state_with_round(round);
16822 let body = pr_message(&state, 'A').body;
16823 assert!(!body.contains("Open review findings"), "{body}");
16824 assert!(!body.contains("Declined"), "{body}");
16825 }
16826
16827 fn state_with_summary(instruction: &str, summary: &str) -> RunState {
16828 let mut state = RunState::new(
16829 PathBuf::from("/repo"),
16830 "main".to_owned(),
16831 "abc1234".to_owned(),
16832 instruction.to_owned(),
16833 Config::default(),
16834 );
16835 state.candidates.push(Candidate {
16836 index: 0,
16837 label: 'A',
16838 agent: "alpha".to_owned(),
16839 branch: "magi/x/A".to_owned(),
16840 worktree: PathBuf::from("/wt"),
16841 summary: summary.to_owned(),
16842 stat: String::new(),
16843 files: 1,
16844 commits: 1,
16845 empty: false,
16846 failed: None,
16847 verified_noop: None,
16848 folded: false,
16849 duration_ms: 0,
16850 });
16851 state
16852 }
16853
16854 fn review_state(subjects: &[&str]) -> RunState {
16855 let mut state = state_with_summary(
16856 "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
16857 "",
16858 );
16859 state.candidates[0].agent = EXISTING_BRANCH.to_owned();
16860 state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
16861 state
16862 }
16863
16864 #[tokio::test]
16867 async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
16868 ask_test_home();
16869 let tmp = tempfile::tempdir().unwrap();
16870 let repo = tmp.path().join("repo");
16871 std::fs::create_dir_all(&repo).unwrap();
16872 init_repo(&repo);
16873 let origin = tmp.path().join("origin.git");
16874 let g = |dir: &Path, args: &[&str]| {
16875 let out = std::process::Command::new("git")
16876 .args(args)
16877 .current_dir(dir)
16878 .quiet()
16879 .output()
16880 .expect("spawn git");
16881 assert!(
16882 out.status.success(),
16883 "git {args:?}: {}",
16884 String::from_utf8_lossy(&out.stderr)
16885 );
16886 };
16887 g(
16888 tmp.path(),
16889 &[
16890 "clone",
16891 "--bare",
16892 "-q",
16893 repo.to_str().unwrap(),
16894 origin.to_str().unwrap(),
16895 ],
16896 );
16897 g(
16898 &repo,
16899 &["remote", "add", "origin", origin.to_str().unwrap()],
16900 );
16901 let c1 = git::rev_parse(&repo, "main").await.unwrap();
16902
16903 std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
16905 g(&repo, &["add", "-A"]);
16906 g(
16907 &repo,
16908 &["commit", "-q", "-m", "chore(deps): update a crate"],
16909 );
16910 g(&repo, &["push", "-q", "origin", "main"]);
16911 g(&repo, &["checkout", "-q", "-b", "feat/own"]);
16912 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16913 g(&repo, &["add", "-A"]);
16914 g(
16915 &repo,
16916 &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
16917 );
16918 g(&repo, &["checkout", "-q", "main"]);
16919
16920 let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
16921 assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
16922 let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
16924 assert_eq!(fallback, c1);
16925
16926 let mut state = review_state(&[
16927 "chore(deps): update a crate",
16928 "fix(daemon): apply a chosen action",
16929 ]);
16930 state.repo = repo.clone();
16931 state.base_branch = "main".to_owned();
16932 state.base_commit = c1;
16933 refresh_reviewed_commits(&mut state, "feat/own").await;
16934 assert_eq!(
16935 state.reviewed_commits,
16936 Some(vec!["fix(daemon): apply a chosen action".to_owned()])
16937 );
16938 assert_eq!(
16939 review_title(&state).as_deref(),
16940 Some("fix(daemon): apply a chosen action")
16941 );
16942 assert_eq!(
16943 leaked_subjects(&state, "feat/own").await,
16944 Some(vec!["chore(deps): update a crate".to_owned()])
16945 );
16946 state.config.merge.remote = "nowhere".to_owned();
16949 assert_eq!(leaked_subjects(&state, "feat/own").await, None);
16950 }
16951
16952 struct SummaryFixture {
16956 _tmp: tempfile::TempDir,
16957 repo: PathBuf,
16958 reply: PathBuf,
16959 runner: Runner,
16960 }
16961
16962 fn summary_fixture() -> SummaryFixture {
16963 ask_test_home();
16964 let tmp = tempfile::tempdir().unwrap();
16965 let repo = tmp.path().join("repo");
16966 std::fs::create_dir_all(&repo).unwrap();
16967 init_repo(&repo);
16968 let origin = tmp.path().join("origin.git");
16969 let g = |args: &[&str]| {
16970 let out = std::process::Command::new("git")
16971 .args(args)
16972 .current_dir(&repo)
16973 .quiet()
16974 .output()
16975 .expect("spawn git");
16976 assert!(
16977 out.status.success(),
16978 "git {args:?}: {}",
16979 String::from_utf8_lossy(&out.stderr)
16980 );
16981 };
16982 g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
16983 g(&["remote", "add", "origin", origin.to_str().unwrap()]);
16984 g(&["push", "-q", "origin", "main"]);
16985 g(&["checkout", "-q", "-b", "feat/own"]);
16986 std::fs::write(repo.join("own.txt"), "own\n").unwrap();
16987 g(&["add", "-A"]);
16988 g(&["commit", "-q", "-m", "feat(own): add the own file"]);
16989 let reply = tmp.path().join("reply.txt");
16990 std::fs::write(
16991 &reply,
16992 "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
16993 )
16994 .unwrap();
16995 let mut runner = runner_at(RunStatus::Gating);
16996 let mut state = review_state(&["feat(own): add the own file"]);
16997 state.repo = repo.clone();
16998 state.base_branch = "main".to_owned();
16999 state.candidates[0].branch = "feat/own".to_owned();
17000 state.candidates[0].worktree = repo.clone();
17001 let mut author = spec("alpha");
17002 author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
17003 author
17004 .env
17005 .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
17006 state.config.agents = vec![author];
17007 runner.state = state;
17008 SummaryFixture {
17009 _tmp: tmp,
17010 repo,
17011 reply,
17012 runner,
17013 }
17014 }
17015
17016 async fn posted(f: &mut SummaryFixture) -> PrMessage {
17017 let winner = f.runner.state.candidates[0].clone();
17018 f.runner
17019 .guarded_pr_message(&winner, None, true)
17020 .await
17021 .unwrap()
17022 .expect("an English description never parks")
17023 }
17024
17025 fn clean_round_with_minor_findings() -> ReviewRound {
17026 let mut round = passing_round(2, None);
17027 round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
17028 round.reviews = vec![ReviewRecord {
17029 attempts: 0,
17030 reviewer: 1,
17031 agent: "alpha".into(),
17032 summary: "内部レビュー".into(),
17033 findings: vec![
17034 finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
17035 finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
17036 ],
17037 vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
17038 failed: None,
17039 duration_ms: 0,
17040 }];
17041 round
17042 }
17043
17044 #[tokio::test]
17045 async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
17046 for review_only in [false, true] {
17047 let mut f = summary_fixture();
17048 if !review_only {
17049 f.runner.state.candidates[0].agent = "alpha".into();
17050 }
17051 f.runner.state.reviews = vec![clean_round_with_minor_findings()];
17052 assert!(f.runner.state.reviews[0].clean);
17053 assert!(f.runner.state.reviews[0].fix.is_none());
17054 assert!(f.runner.state.open_findings().is_empty());
17055 assert_eq!(
17056 crate::pr_outcome::required_finding_ids(&f.runner.state),
17057 ["R2-1-1", "R2-1-2"]
17058 );
17059 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17060 let evidence = crate::pr_outcome::summary_evidence(
17061 &f.runner.state,
17062 crate::pr_outcome::FinalHead::At(&head),
17063 );
17064 assert!(
17065 evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
17066 );
17067 let omitted = posted(&mut f).await;
17069 assert!(omitted.body.contains("No description"));
17070 assert!(omitted.body.contains("`R2-1-1` [Minor]"));
17071 assert!(omitted.body.contains("`R2-1-2` [Minor]"));
17072 assert!(f.runner.state.change_summary.is_none());
17073 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
17074 let described = posted(&mut f).await;
17075 assert!(described.body.contains("improve documentation for callers"));
17076 assert!(described.body.contains("remaining edge case"));
17077 assert!(described.body.contains("## Open review findings"));
17078 assert!(!described.body.contains("未修正"));
17079 }
17080 }
17081
17082 async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
17083 let mut f = summary_fixture();
17084 if !review_only {
17085 f.runner.state.candidates[0].agent = "alpha".into();
17086 }
17087 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17088 let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
17089 fixed
17090 .fix
17091 .as_mut()
17092 .unwrap()
17093 .rejected
17094 .push(crate::verdict::Rejection {
17095 id: "R1-1-1".into(),
17096 why: "呼び出されないコードのため辞退".into(),
17097 });
17098 let mut clean = clean_round_with_minor_findings();
17099 clean.verified_head = Some(head.clone());
17100 f.runner.state.reviews = vec![fixed, clean];
17101 ok_gate(&mut f.runner.state, Some(&head));
17102 std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
17103 f.runner.state.config.agents.push(scripted("beta", reply));
17104 f.runner.state.config.roles.language_judge =
17105 Some(crate::config::AgentChoice::One("beta".into()));
17106 f
17107 }
17108
17109 #[tokio::test]
17110 async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
17111 for review_only in [false, true] {
17112 let mut f = rewrite_fixture_with_findings(review_only,
17113 "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
17114 let m = posted(&mut f).await;
17115 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17116 let expected =
17117 public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
17118 assert!(m.body.contains(&expected));
17119 assert!(m.body.contains("## Review rounds"));
17120 assert!(
17121 m.body
17122 .contains("Gate (`verify.gate`): passed on the final commit")
17123 );
17124 assert!(
17125 m.body
17126 .contains("Tests (`verify.e2e`): passed on the final commit")
17127 );
17128 let c = f.runner.state.change_summary.as_ref().unwrap();
17129 assert!(
17130 m.body
17131 .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
17132 );
17133 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17134 assert!(crate::pr_outcome::covers_required_findings(
17135 &f.runner.state,
17136 &m.body
17137 ));
17138 assert!(m.body.contains("documentation remains incomplete"));
17139 assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
17140 assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
17141 assert!(f.runner.state.github_text.is_none());
17142 }
17143 }
17144
17145 #[tokio::test]
17146 async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
17147 for review_only in [false, true] {
17148 for reply in [
17149 "TITLE: fix: public description\nA short English description.",
17150 "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
17151 ] {
17152 let mut f = rewrite_fixture_with_findings(review_only, reply).await;
17153 let winner = f.runner.state.candidates[0].clone();
17154 let posted = f
17155 .runner
17156 .guarded_pr_message(&winner, None, true)
17157 .await
17158 .unwrap();
17159 assert!(posted.is_none());
17160 assert!(f.runner.state.parked);
17161 let withheld = f.runner.state.github_text.as_ref().unwrap();
17162 assert!(
17163 withheld.rewrite.is_none(),
17164 "an incomplete rewrite must not be selectable"
17165 );
17166 let q = ask::Questions::open()
17167 .get(withheld.question.as_ref().unwrap())
17168 .unwrap();
17169 assert!(
17170 !q.choices
17171 .contains(&crate::github_text::USE_REWRITE.to_owned())
17172 );
17173 assert!(
17174 q.detail.contains("R1-1-1")
17175 && q.detail.contains("R2-1-1")
17176 && q.detail.contains("R2-1-2")
17177 );
17178 }
17179 }
17180 }
17181
17182 #[tokio::test]
17183 async fn japanese_reports_stay_local_in_both_public_run_modes() {
17184 for review_only in [false, true] {
17185 let mut f = summary_fixture();
17186 if !review_only {
17187 f.runner.state.candidates[0].agent = "alpha".into();
17188 }
17189 f.runner.state.instruction = "内部タスクの説明".into();
17190 f.runner.state.candidates[0].summary = "内部の候補報告".into();
17191 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17192 f.runner.state.reviews = vec![
17193 round_with_notes(1, Some("コミットできません。テストは実行できません。")),
17194 round_with_notes(2, Some("二回目の内部修正報告です。")),
17195 passing_round(3, Some(&head)),
17196 ];
17197 f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
17198 ok_gate(&mut f.runner.state, Some(&head));
17199 let captured = f.repo.join("public-prompt.txt");
17200 let author = &mut f.runner.state.config.agents[0];
17201 author.command = vec![
17202 "sh".into(),
17203 "-c".into(),
17204 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17205 ];
17206 author.env.insert(
17207 "PROMPT_FILE".into(),
17208 captured.to_string_lossy().into_owned(),
17209 );
17210 let m = posted(&mut f).await;
17211 assert!(m.body.contains("Adds `own.txt`"));
17212 assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
17213 assert!(m.body.contains("R1-1-1"));
17214 assert!(
17215 m.body
17216 .contains("Gate (`verify.gate`): passed on the final commit")
17217 );
17218 assert!(
17219 m.body
17220 .contains("Tests (`verify.e2e`): passed on the final commit")
17221 );
17222 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17223 assert!(!m.body.contains("内部") && !m.body.contains("できません"));
17224 let prompt = std::fs::read_to_string(captured).unwrap();
17225 assert!(prompt.contains(&format!(
17226 "git diff {} {head}",
17227 f.runner.state.change_summary.as_ref().unwrap().base
17228 )));
17229 assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
17230 assert_eq!(
17231 f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
17232 "二回目の内部修正報告です。"
17233 );
17234 }
17235 }
17236
17237 #[tokio::test]
17238 async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
17239 for review_only in [false, true] {
17240 let mut f = summary_fixture();
17241 if !review_only {
17242 f.runner.state.candidates[0].agent = "alpha".into();
17243 }
17244 f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
17245 f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
17246 std::fs::write(&f.reply, "no usable description").unwrap();
17247 let failed = posted(&mut f).await;
17248 assert!(
17249 failed
17250 .body
17251 .contains("No description of what this change does could be generated")
17252 );
17253 assert!(failed.body.is_ascii());
17254 assert!(f.runner.state.github_text.is_none());
17255 assert!(f.runner.state.change_summary.is_none());
17256 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17257 let evidence = crate::pr_outcome::summary_evidence(
17258 &f.runner.state,
17259 crate::pr_outcome::FinalHead::At(&head),
17260 );
17261 let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
17262 assert!(
17263 crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
17264 .exists()
17265 );
17266 f.runner.state =
17267 serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
17268 std::fs::write(
17269 &f.reply,
17270 "TITLE: feat(own): add the own file\nAdds the committed own file.",
17271 )
17272 .unwrap();
17273 let retried = posted(&mut f).await;
17274 assert!(retried.body.contains("Adds the committed own file."));
17275 assert_eq!(
17276 f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
17277 "日本語の修正報告"
17278 );
17279 }
17280 }
17281
17282 #[tokio::test]
17283 async fn same_head_verification_and_findings_invalidate_public_prose() {
17284 let mut f = summary_fixture();
17285 posted(&mut f).await;
17286 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17287 let original = f
17288 .runner
17289 .state
17290 .change_summary
17291 .as_ref()
17292 .unwrap()
17293 .evidence
17294 .clone();
17295 ok_gate(&mut f.runner.state, Some(&head));
17296 std::fs::write(
17297 &f.reply,
17298 "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
17299 )
17300 .unwrap();
17301 let verified = posted(&mut f).await;
17302 assert!(verified.body.contains("now verified"));
17303 assert_ne!(
17304 f.runner.state.change_summary.as_ref().unwrap().evidence,
17305 original
17306 );
17307 let mut round = round_with_notes(1, Some("内部報告"));
17308 round.reviews.push(ReviewRecord {
17309 attempts: 0,
17310 reviewer: 1,
17311 agent: "alpha".into(),
17312 summary: "内部レビュー".into(),
17313 findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
17314 vote: None,
17315 failed: None,
17316 duration_ms: 0,
17317 });
17318 round
17319 .fix
17320 .as_mut()
17321 .unwrap()
17322 .rejected
17323 .push(crate::verdict::Rejection {
17324 id: "R1-1-1".into(),
17325 why: "呼び出し元から到達できない".into(),
17326 });
17327 f.runner.state.reviews = vec![round];
17328 let omitted = posted(&mut f).await;
17330 assert!(omitted.body.contains("No description"));
17331 assert!(omitted.body.contains("R1-1-1"));
17332 assert!(!omitted.body.contains("到達"));
17333 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
17334 let described = posted(&mut f).await;
17335 assert!(described.body.contains("unreachable from callers"));
17336 assert!(described.body.contains("R1-1-1"));
17337 assert!(!described.body.contains("到達"));
17338 let evidence = f
17339 .runner
17340 .state
17341 .change_summary
17342 .as_ref()
17343 .unwrap()
17344 .evidence
17345 .clone();
17346 f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
17347 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
17348 assert!(posted(&mut f).await.body.contains("rationale updated"));
17349 assert_ne!(
17350 f.runner.state.change_summary.as_ref().unwrap().evidence,
17351 evidence
17352 );
17353 f.runner
17355 .state
17356 .change_summary
17357 .as_mut()
17358 .unwrap()
17359 .evidence
17360 .clear();
17361 std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
17362 assert!(posted(&mut f).await.body.contains("current specification"));
17363 }
17364
17365 #[tokio::test]
17366 async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
17367 let mut f = summary_fixture();
17368 let m = posted(&mut f).await;
17369 assert_eq!(m.title, "feat(own): add the own file");
17370 assert!(
17371 m.body
17372 .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
17373 "{}",
17374 m.body
17375 );
17376 assert!(!m.body.contains("## Review fixes"), "{}", m.body);
17377 assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
17378 assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
17379 let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
17380 let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
17381 let kept = f.runner.state.change_summary.as_ref().unwrap();
17382 assert_eq!(
17383 (kept.head.as_str(), kept.base.as_str()),
17384 (head.as_str(), base.as_str())
17385 );
17386 assert!(!f.runner.state.seats["change-summary"].key.is_empty());
17387 }
17388
17389 #[tokio::test]
17390 async fn review_only_summary_receives_the_configured_github_language() {
17391 let mut f = summary_fixture();
17392 f.runner.state.config.graph.github_language = "ja".into();
17393 let captured = f.repo.join("summary-prompt.txt");
17394 let author = &mut f.runner.state.config.agents[0];
17395 author.command = vec![
17396 "sh".into(),
17397 "-c".into(),
17398 "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
17399 ];
17400 author.env.insert(
17401 "PROMPT_FILE".into(),
17402 captured.to_string_lossy().into_owned(),
17403 );
17404 std::fs::write(
17405 &f.reply,
17406 "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
17407 )
17408 .unwrap();
17409 let m = posted(&mut f).await;
17410 let prompt = std::fs::read_to_string(captured).unwrap();
17411 assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
17412 assert!(!prompt.contains("in English"));
17413 assert_eq!(m.title, "feat(own): ファイルを追加");
17414 assert!(m.body.contains("own.txt を追加します。"));
17415 }
17416
17417 #[tokio::test]
17418 async fn review_fixes_come_after_the_whole_change_summary() {
17419 let mut f = summary_fixture();
17420 f.runner.state.reviews = vec![
17421 round_with_notes(1, Some("handled the PATH case")),
17422 round_with_notes(2, None),
17423 ];
17424 let m = posted(&mut f).await;
17425 let summary = m.body.find("## Summary").expect("summary");
17426 let fixes = m.body.find("## Review rounds").expect("review rounds");
17427 assert!(summary < fixes, "{}", m.body);
17428 assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
17429 assert!(m.body.contains("## Final verification"), "{}", m.body);
17430 }
17431
17432 #[tokio::test]
17433 async fn a_changed_final_diff_invalidates_the_earlier_summary() {
17434 let mut f = summary_fixture();
17435 let first = posted(&mut f).await;
17436 let first_head = f.runner.state.change_summary.clone().unwrap().head;
17437 std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
17439 let again = posted(&mut f).await;
17440 assert_eq!((again.title, again.body), (first.title, first.body));
17441
17442 let g = |args: &[&str]| {
17443 let out = std::process::Command::new("git")
17444 .args(args)
17445 .current_dir(&f.repo)
17446 .quiet()
17447 .output()
17448 .expect("spawn git");
17449 assert!(out.status.success(), "git {args:?}");
17450 };
17451 std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
17452 g(&["add", "-A"]);
17453 g(&["commit", "-q", "-m", "fix(own): handle more"]);
17454 let second = posted(&mut f).await;
17455 let kept = f.runner.state.change_summary.clone().unwrap();
17456 assert_ne!(kept.head, first_head);
17457 assert!(second.body.contains("Not read yet."), "{}", second.body);
17458 assert!(!second.body.contains("so the first summary is read"));
17459 assert_eq!(second.title, "feat(own): other");
17460 }
17461
17462 #[tokio::test]
17463 async fn an_unavailable_summary_is_stated_and_never_invented() {
17464 let mut f = summary_fixture();
17465 std::fs::write(&f.reply, "no title line here").unwrap();
17466 let m = posted(&mut f).await;
17467 assert!(
17468 m.body
17469 .contains("No description of what this change does could be generated"),
17470 "{}",
17471 m.body
17472 );
17473 assert!(m.body.contains("internal reports remain in local records"));
17474 assert!(f.runner.state.change_summary.is_none());
17475 assert!(!m.body.contains("## Commits under review"), "{}", m.body);
17476 let winner = f.runner.state.candidates[0].clone();
17478 let preview = f
17479 .runner
17480 .guarded_pr_message(&winner, None, false)
17481 .await
17482 .unwrap()
17483 .unwrap();
17484 assert!(!preview.body.contains("## Summary"), "{}", preview.body);
17485 }
17486
17487 #[test]
17488 fn pr_message_review_single_commit_uses_its_subject() {
17489 let state = review_state(&["feat(nats): per-role user"]);
17490 let m = pr_message(&state, 'A');
17491 assert_eq!(m.title, "feat(nats): per-role user");
17492 assert!(!m.body.contains("Review the work already"), "{}", m.body);
17493 assert!(m.body.contains("## Commits under review"), "{}", m.body);
17494 }
17495
17496 #[test]
17497 fn pr_message_review_multi_commit_takes_the_oldest() {
17498 let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
17499 let m = pr_message(&state, 'A');
17500 assert_eq!(m.title, "feat: the change");
17501 for s in ["feat: the change", "fix: typo", "fix: again"] {
17502 assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
17503 }
17504 }
17505
17506 #[test]
17507 fn pr_message_review_without_a_usable_first_subject_is_neutral() {
17508 for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
17509 let mut state = review_state(&[first, "fix: later fixup"]);
17510 state.candidates[0].branch = "機能/ログイン".to_owned();
17511 let m = pr_message(&state, 'A');
17512 assert!(
17513 m.title.starts_with("chore: land candidate A of run"),
17514 "{}",
17515 m.title
17516 );
17517 }
17518 }
17519
17520 fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
17521 BranchFacts {
17522 commits: commits
17523 .iter()
17524 .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
17525 .collect(),
17526 stat: stat.to_owned(),
17527 }
17528 }
17529
17530 fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
17531 let mut r = review_round(true, 0, 1, 1, true, true);
17532 r.round = round;
17533 r.fix = notes.map(|n| FixRecord {
17534 agent: "fixer".to_owned(),
17535 addressed: Vec::new(),
17536 rejected: Vec::new(),
17537 notes: n.to_owned(),
17538 committed: true,
17539 failed: None,
17540 duration_ms: 0,
17541 continuation: None,
17542 });
17543 r
17544 }
17545
17546 #[test]
17547 fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
17548 let state = review_state(&["ignored"]);
17549 let f = facts(
17550 &[
17551 (
17552 "fix(login): resolve PATH on macOS",
17553 "Login shells skip rc files.",
17554 ),
17555 ("fix: address review", ""),
17556 ],
17557 " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
17558 );
17559 let m = pr_message_with(&state, 'A', Some(&f));
17560 assert_eq!(m.title, "fix(login): resolve PATH on macOS");
17561 assert!(
17562 m.body
17563 .contains("- fix(login): resolve PATH on macOS\n Login shells skip rc files.\n"),
17564 "{}",
17565 m.body
17566 );
17567 assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
17568 assert!(m.body.contains("## Diff stat"), "{}", m.body);
17569 assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
17570 for banned in [
17571 "Review the work already",
17572 "no task statement",
17573 "Original task",
17574 ] {
17575 assert!(!m.body.contains(banned), "{banned}: {}", m.body);
17576 }
17577 assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
17578 }
17579
17580 #[test]
17581 fn pr_message_review_truncates_a_huge_first_commit_body() {
17582 let state = review_state(&["ignored"]);
17583 let f = facts(
17584 &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
17585 "s",
17586 );
17587 let m = pr_message_with(&state, 'A', Some(&f));
17588 assert!(m.body.len() < 30_000, "{}", m.body.len());
17589 assert!(m.body.contains("(truncated)"), "{}", m.body.len());
17590 assert!(
17591 m.body.contains("- ... 1 more commit(s)"),
17592 "{}",
17593 m.body.len()
17594 );
17595 assert!(m.body.ends_with("magi:candidate-a\n"));
17596 }
17597
17598 #[test]
17599 fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
17600 let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
17601 assert_eq!(m.title, "feat: x");
17602 assert!(m.body.contains("- fix: y\n"), "{}", m.body);
17603 assert!(!m.body.contains("Diff stat"), "{}", m.body);
17604 assert!(!m.body.contains("no task statement"), "{}", m.body);
17605 }
17606
17607 #[test]
17608 fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
17609 let mut state = review_state(&["日本語の件名"]);
17610 state.candidates[0].branch = "fix/macos-login-path".to_owned();
17611 assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
17612 state.candidates[0].branch = "機能/ログイン".to_owned();
17613 assert!(
17614 pr_message(&state, 'A')
17615 .title
17616 .starts_with("chore: land candidate A")
17617 );
17618 }
17619
17620 #[test]
17621 fn pr_message_review_fixes_survive_a_clean_final_round() {
17622 let mut state = review_state(&["feat: x"]);
17623 state.reviews = vec![
17624 round_with_notes(1, Some("handled the PATH case")),
17625 round_with_notes(2, None),
17626 ];
17627 let body = pr_message(&state, 'A').body;
17628 assert!(
17629 body.contains("## Review rounds") && !body.contains("handled the PATH case"),
17630 "{body}"
17631 );
17632 assert!(body.contains("### Round 1"), "{body}");
17633
17634 state.reviews = vec![
17635 round_with_notes(1, Some("first fix")),
17636 round_with_notes(2, Some("")),
17637 round_with_notes(3, Some("second fix")),
17638 round_with_notes(4, None),
17639 ];
17640 let body = pr_message(&state, 'A').body;
17641 assert!(
17642 body.contains("### Round 1") && !body.contains("first fix"),
17643 "{body}"
17644 );
17645 assert!(
17646 body.contains("### Round 3") && !body.contains("second fix"),
17647 "{body}"
17648 );
17649 assert!(body.contains("### Round 2"), "{body}");
17650 }
17651
17652 const FINAL: &str = "abcdef1234567890";
17653
17654 fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
17655 let mut r = round_with_notes(round, None);
17656 r.verified_head = verified.map(str::to_owned);
17657 r
17658 }
17659
17660 fn ok_gate(state: &mut RunState, head: Option<&str>) {
17661 state.gate = vec![CommandOutcome {
17662 command: "gate".to_owned(),
17663 code: Some(0),
17664 output_tail: String::new(),
17665 duration_ms: 0,
17666 resource_blocked: false,
17667 }];
17668 state.gate_ran = true;
17669 state.gate_head = head.map(str::to_owned);
17670 }
17671
17672 fn body_at(state: &RunState) -> String {
17673 pr_message_raw(
17674 state,
17675 'A',
17676 None,
17677 crate::pr_outcome::FinalHead::At(FINAL),
17678 ChangeNote::NotAsked,
17679 )
17680 .body
17681 }
17682
17683 const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
17684
17685 #[test]
17686 fn rescued_commit_and_later_verification_override_the_fixers_limits() {
17687 let mut state = review_state(&["feat: x"]);
17688 state.reviews = vec![
17689 round_with_notes(1, Some(SANDBOXED)),
17690 passing_round(2, Some(FINAL)),
17691 ];
17692 ok_gate(&mut state, Some(FINAL));
17693 let body = body_at(&state);
17694 assert!(!body.contains("Handled the PATH case."), "{body}");
17695 assert!(!body.contains("could not commit"), "{body}");
17696 assert!(!body.contains("could not run"), "{body}");
17697 assert!(
17698 body.contains("The fix is committed on the branch"),
17699 "{body}"
17700 );
17701 assert!(!body.contains("sandbox"), "{body}");
17702 assert!(
17703 body.contains("Tests (`verify.e2e`): passed on the final commit"),
17704 "{body}"
17705 );
17706 assert!(
17707 body.contains("Gate (`verify.gate`): passed on the final commit"),
17708 "{body}"
17709 );
17710 assert!(
17712 state.reviews[0]
17713 .fix
17714 .as_ref()
17715 .unwrap()
17716 .notes
17717 .contains("could not commit")
17718 );
17719 }
17720
17721 #[test]
17722 fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
17723 let mut state = review_state(&["feat: x"]);
17724 let mut r1 = round_with_notes(1, Some(SANDBOXED));
17725 r1.fix.as_mut().unwrap().committed = false;
17726 let mut r2 = passing_round(2, Some(FINAL));
17727 r2.e2e[0].code = Some(1);
17728 state.reviews = vec![r1, r2];
17729 let body = body_at(&state);
17730 assert!(!body.contains("could not commit"), "{body}");
17731 assert!(!body.contains("could not run"), "{body}");
17732 assert!(body.contains("This round produced no commit"), "{body}");
17733 assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
17734 assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
17735 assert!(!body.contains("passed"), "{body}");
17736 }
17737
17738 #[test]
17739 fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
17740 let mut state = review_state(&["feat: x"]);
17741 state.reviews = vec![
17742 round_with_notes(1, Some(SANDBOXED)),
17743 passing_round(2, Some("0123456789abcdef")),
17744 ];
17745 ok_gate(&mut state, Some("0123456789abcdef"));
17746 let body = body_at(&state);
17747 assert!(!body.contains("HTTP tests could not run"), "{body}");
17748 assert!(!body.contains("could not commit"), "{body}");
17750 assert!(
17751 body.contains("passed on an earlier commit (`0123456`)"),
17752 "{body}"
17753 );
17754 assert!(!body.contains("passed on the final commit"), "{body}");
17755
17756 ok_gate(&mut state, None);
17758 let body = body_at(&state);
17759 assert!(
17760 body.contains("passed on a commit that was not recorded"),
17761 "{body}"
17762 );
17763 }
17764
17765 #[test]
17766 fn pr_message_implementation_run_keeps_its_shape_and_marker() {
17767 let state = state_with_summary(
17768 "add retries to the client",
17769 "TITLE: feat: retries\n\nDid it.",
17770 );
17771 let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
17772 assert_eq!(m.title, "feat: retries");
17773 assert!(
17774 m.body.contains("<summary>Original task</summary>"),
17775 "{}",
17776 m.body
17777 );
17778 assert!(!m.body.contains("Commits under review"), "{}", m.body);
17779 assert!(
17780 m.body
17781 .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
17782 "{}",
17783 m.body
17784 );
17785 }
17786
17787 #[test]
17788 fn pr_message_review_bounds_a_long_english_subject() {
17789 let long = format!("feat: {}", "word ".repeat(100));
17790 let m = pr_message(&review_state(&[&long]), 'A');
17791 assert!(m.title.starts_with("feat: word"), "{}", m.title);
17792 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17793 }
17794
17795 #[test]
17796 fn pr_message_implementation_run_is_unchanged_by_review_support() {
17797 let state = state_with_summary("add retries\n\ndetails", "- did some things");
17798 let m = pr_message(&state, 'A');
17799 assert_eq!(m.title, "add retries");
17800 assert!(m.body.contains("<summary>Original task</summary>"));
17801 assert!(!m.body.contains("Commits under review"));
17802 assert_eq!(landing_subject_source(&state), state.instruction);
17803 }
17804
17805 #[test]
17806 fn review_run_squash_subject_is_the_change_not_the_prompt() {
17807 let state = review_state(&["feat: the change", "fix: typo"]);
17808 let source = landing_subject_source(&state);
17809 assert_eq!(land::merge_subject("", &source), "feat: the change");
17810 assert_eq!(
17811 land::merge_subject("magi: candidate A (uncommitted work)", &source),
17812 "feat: the change"
17813 );
17814 assert_eq!(
17816 land::merge_subject("feat: renamed by hand", &source),
17817 "feat: renamed by hand"
17818 );
17819 let blank = review_state(&["日本語"]);
17820 assert!(
17821 land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
17822 );
17823 }
17824
17825 #[test]
17826 fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
17827 let state = review_state(&["feat: the change"]);
17828 let source = landing_subject_source(&state);
17829 let old = "Review the work already on branch `magi/x/A`. There is no task statement";
17830 assert_eq!(
17831 land::merge_subject(landing_title(&state, old), &source),
17832 "feat: the change"
17833 );
17834 assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
17835 let task = state_with_summary("add retries", "");
17836 assert_eq!(landing_title(&task, old), old);
17837 }
17838
17839 #[test]
17840 fn pr_message_describes_the_change_not_the_task() {
17841 let state = state_with_summary(
17842 "今回やってほしいこと: results projector を直す",
17843 "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
17844 );
17845 let m = pr_message(&state, 'A');
17846 assert_eq!(m.title, "fix(web): batch the runs list reads");
17847 assert!(
17848 m.body.starts_with("## Summary\n\n- reads run.json once"),
17849 "{}",
17850 m.body
17851 );
17852 assert!(!m.body.contains("TITLE:"), "{}", m.body);
17853 let task_at = m.body.find("今回やってほしいこと").unwrap();
17854 let details_at = m.body.find("<details>").unwrap();
17855 assert!(
17856 details_at < task_at,
17857 "the task lives inside <details>: {}",
17858 m.body
17859 );
17860 assert!(m.body.contains(&format!("magi:run/{}", state.id)));
17861 assert!(m.body.contains("magi:candidate-a"));
17862 }
17863
17864 #[test]
17865 fn pr_message_falls_back_to_the_task_without_a_title_line() {
17866 let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
17867 let m = pr_message(&state, 'A');
17868 assert_eq!(m.title, "add retries");
17869 assert!(
17870 m.body.contains("## Summary\n\n- did some things"),
17871 "{}",
17872 m.body
17873 );
17874
17875 let none = RunState::new(
17876 PathBuf::from("/repo"),
17877 "main".to_owned(),
17878 "abc1234".to_owned(),
17879 "add retries".to_owned(),
17880 Config::default(),
17881 );
17882 let m = pr_message(&none, 'A');
17883 assert_eq!(m.title, "add retries");
17884 assert!(!m.body.contains("## Summary"), "{}", m.body);
17885 }
17886
17887 #[test]
17888 fn pr_message_refuses_the_candidate_commit_subject() {
17889 for bad in [
17890 "TITLE: magi: candidate A (uncommitted work)",
17891 "TITLE: chore: stuff (uncommitted work)",
17892 "TITLE: ",
17893 ] {
17894 let state = state_with_summary("add retries", bad);
17895 assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
17896 }
17897 }
17898
17899 #[test]
17900 fn pr_message_bounds_a_very_long_task_and_title() {
17901 let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
17902 let state = state_with_summary(&long, "- nothing");
17903 let m = pr_message(&state, 'A');
17904 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17905 assert!(!m.title.contains('\n'));
17906
17907 let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
17908 let m = pr_message(&state, 'A');
17909 assert!(m.title.starts_with("feat: "));
17910 assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
17911 assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
17912 }
17913
17914 fn long_title_of(instruction: &str) -> String {
17915 pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
17916 }
17917
17918 #[test]
17919 fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
17920 let first = "Make the landing path keep a readable title for long tasks";
17921 let line = format!(
17922 "{first}. {}",
17923 "And then keep going with more words ".repeat(20)
17924 );
17925 let t = long_title_of(&line);
17926 assert_eq!(t, first);
17927 assert!(!t.starts_with("chore: land"));
17928 }
17929
17930 #[test]
17931 fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
17932 let line = "word ".repeat(200);
17933 let t = long_title_of(&line);
17934 assert!(t.ends_with("word..."), "{t}");
17935 assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
17936 }
17937
17938 #[test]
17939 fn pr_message_long_non_english_or_letterless_line_is_neutral() {
17940 for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
17941 assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
17942 }
17943 }
17944
17945 #[test]
17946 fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
17947 let mut state = state_with_summary("再試行を追加する", "- nothing");
17948 assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
17949 state.config.graph.github_language = "ja".into();
17950 assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
17951 }
17952
17953 #[test]
17954 fn pr_message_title_limit_is_exact() {
17955 let at = "a".repeat(PR_TITLE_MAX);
17956 assert_eq!(long_title_of(&at), at);
17957 let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
17958 assert!(over.ends_with("..."), "{over}");
17959 assert_eq!(over.chars().count(), PR_TITLE_MAX);
17960 }
17961
17962 #[test]
17963 fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
17964 let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
17965 let t = long_title_of(&line);
17966 assert!(t.ends_with("..."), "{t}");
17967 assert!(t.is_ascii(), "{t}");
17968 }
17969
17970 #[test]
17971 fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
17972 let line = format!(
17973 "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
17974 "plus more filler words ".repeat(20)
17975 );
17976 let t = long_title_of(&line);
17977 assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
17978 }
17979
17980 #[test]
17981 fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
17982 let line = format!("\n\n# {}", "title words ".repeat(40));
17983 let state = state_with_summary(&line, "- nothing");
17984 let m = pr_message(&state, 'A');
17985 assert!(m.title.starts_with("title words"), "{}", m.title);
17986 assert_eq!(
17987 land::merge_subject(&m.title, &landing_subject_source(&state)),
17988 m.title
17989 );
17990 assert_eq!(
17992 land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
17993 "feat: renamed by hand"
17994 );
17995 }
17996
17997 #[tokio::test]
17998 async fn github_text_rewrite_is_bounded_and_falls_back() {
17999 crate::run::pin_test_home();
18000 for (reply, accepted) in [
18001 (
18002 "TITLE: fix: retries\nAdd retries for failed requests.",
18003 true,
18004 ),
18005 (
18006 "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
18007 false,
18008 ),
18009 (
18010 "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
18011 true,
18012 ),
18013 (
18014 "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
18015 true,
18016 ),
18017 (
18018 "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
18019 true,
18020 ),
18021 (
18022 "TITLE: fix: retries\n日本語の説明です。 token=secret",
18023 false,
18024 ),
18025 ("TITLE: fix: retries\n", false),
18026 ] {
18027 let dir = tempfile::tempdir().unwrap();
18028 let mut runner = runner_at(RunStatus::Gating);
18029 runner.state = state_with_summary(
18030 "add retries",
18031 "TITLE: fix: retries\n日本語の説明を書きます。",
18032 );
18033 runner.state.repo = dir.path().to_owned();
18034 runner.state.candidates[0].worktree = dir.path().to_owned();
18035 let mut author = spec("alpha");
18036 author.command = vec![
18037 "sh".into(),
18038 "-c".into(),
18039 "printf '%s' \"$REWRITE_REPLY\"".into(),
18040 ];
18041 author.env.insert("REWRITE_REPLY".into(), reply.into());
18042 runner.state.config.agents = vec![author];
18043 seed_public_summary(&mut runner);
18044 let winner = runner.state.candidates[0].clone();
18045 let first = runner
18046 .guarded_pr_message(&winner, None, true)
18047 .await
18048 .unwrap();
18049 let message = if accepted {
18050 first.expect("a passing rewrite never asks")
18051 } else {
18052 assert!(first.is_none());
18054 assert!(runner.state.parked);
18055 let store = ask::Questions::open();
18056 let asked: Vec<_> = store
18057 .list()
18058 .into_iter()
18059 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18060 .collect();
18061 assert_eq!(asked.len(), 1);
18062 assert!(!asked[0].detail.contains("secret"));
18063 let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
18064 assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
18065 assert_eq!(
18066 asked[0]
18067 .choices
18068 .contains(&crate::github_text::USE_REWRITE.to_owned()),
18069 rewrite.is_some()
18070 );
18071 store
18074 .update(&asked[0].id, |q| {
18075 q.abandon("test");
18076 Ok(())
18077 })
18078 .unwrap();
18079 let again = runner
18080 .guarded_pr_message(&winner, None, true)
18081 .await
18082 .unwrap()
18083 .expect("an abandoned question falls back");
18084 let n = store
18085 .list()
18086 .into_iter()
18087 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18088 .count();
18089 assert_eq!(n, 1, "asked once per rejected text");
18090 again
18091 };
18092 assert!(crate::github_text::check(&message.title, &message.body).is_empty());
18093 assert!(!message.body.contains("secret"));
18094 assert!(!message.body.contains("prefix"));
18095 assert!(!message.title.contains("secret"));
18096 assert_eq!(
18097 message.body.contains("Add retries for failed requests."),
18098 accepted
18099 );
18100 assert_eq!(
18101 runner.state.seats["github-text-rewrite"].turns, 1,
18102 "only one rewrite invocation"
18103 );
18104 assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
18105 if accepted {
18106 assert!(runner.state.github_text.is_none());
18107 assert!(!runner.state.parked);
18108 assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
18109 assert!(message.body.contains("## Final verification"));
18110 assert!(message.body.contains("Evidence: final diff"));
18111 assert!(
18112 message
18113 .body
18114 .contains(&format!("magi:run/{}", runner.state.id))
18115 );
18116 }
18117 if !accepted {
18118 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18119 assert!(
18120 message
18121 .body
18122 .contains(&format!("magi:run/{}", runner.state.id))
18123 );
18124 }
18125 }
18126 }
18127
18128 fn scripted(id: &str, reply: &str) -> AgentSpec {
18129 let mut a = spec(id);
18130 a.command = vec![
18131 "sh".into(),
18132 "-c".into(),
18133 "printf '%s' \"$REWRITE_REPLY\"".into(),
18134 ];
18135 a.env.insert("REWRITE_REPLY".into(), reply.into());
18136 a
18137 }
18138
18139 const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
18140 const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
18141
18142 fn seed_public_summary(runner: &mut Runner) {
18144 let repo = &runner.state.repo;
18145 let g = |args: &[&str]| {
18146 let out = std::process::Command::new("git")
18147 .args(args)
18148 .current_dir(repo)
18149 .quiet()
18150 .output()
18151 .unwrap();
18152 assert!(out.status.success(), "git {args:?}");
18153 String::from_utf8_lossy(&out.stdout).trim().to_owned()
18154 };
18155 g(&["init", "-q", "-b", "main"]);
18156 g(&[
18157 "-c",
18158 "user.name=Test",
18159 "-c",
18160 "user.email=test@example.invalid",
18161 "commit",
18162 "-q",
18163 "--allow-empty",
18164 "-m",
18165 "feat: fixture",
18166 ]);
18167 let head = g(&["rev-parse", "HEAD"]);
18168 g(&["update-ref", "refs/remotes/origin/main", &head]);
18169 let evidence = crate::pr_outcome::summary_evidence(
18170 &runner.state,
18171 crate::pr_outcome::FinalHead::At(&head),
18172 );
18173 let generated = &runner.state.candidates[0].summary;
18174 runner.state.change_summary = Some(crate::run::ChangeSummary {
18175 base: head.clone(),
18176 head,
18177 evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
18178 title: summary_title(generated).unwrap(),
18179 body: summary_without_title(generated),
18180 });
18181 }
18182
18183 fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
18184 let mut runner = runner_at(RunStatus::Gating);
18185 runner.state = state_with_summary("add retries", JA_SUMMARY);
18186 if review_only {
18187 runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
18188 }
18189 runner.state.repo = dir.to_owned();
18190 runner.state.candidates[0].worktree = dir.to_owned();
18191 runner.state.config.agents = agents;
18192 seed_public_summary(&mut runner);
18193 runner
18194 }
18195
18196 fn gate_questions(runner: &Runner) -> usize {
18197 ask::Questions::open()
18198 .list()
18199 .into_iter()
18200 .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
18201 .count()
18202 }
18203
18204 #[tokio::test]
18205 async fn review_only_description_is_rewritten_by_the_language_judge_role() {
18206 crate::run::pin_test_home();
18207 let dir = tempfile::tempdir().unwrap();
18208 let mut runner = rewrite_runner(
18209 dir.path(),
18210 true,
18211 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18212 );
18213 runner.state.config.roles.language_judge =
18214 Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
18215 let winner = runner.state.candidates[0].clone();
18216 let msg = runner
18217 .guarded_pr_message(&winner, None, true)
18218 .await
18219 .unwrap()
18220 .expect("an English rewrite posts without asking");
18221 assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
18222 assert!(msg.body.contains("<details>kept</details>"));
18223 assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
18224 assert_eq!(gate_questions(&runner), 0);
18225 assert!(runner.state.github_text.is_none());
18226 assert!(!runner.state.seats.contains_key("impl-A"));
18227 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18228 }
18229
18230 #[tokio::test]
18231 async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
18232 crate::run::pin_test_home();
18233 let dir = tempfile::tempdir().unwrap();
18234 let mut runner = rewrite_runner(
18235 dir.path(),
18236 false,
18237 vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
18238 );
18239 runner.state.config.roles.language_judge =
18240 Some(crate::config::AgentChoice::One("beta".into()));
18241 runner.state.seats.insert(
18242 "impl-A".into(),
18243 SeatState::new("impl-A", "alpha", runner.state.seed),
18244 );
18245 let winner = runner.state.candidates[0].clone();
18246 let msg = runner
18247 .guarded_pr_message(&winner, None, true)
18248 .await
18249 .unwrap()
18250 .expect("rewritten");
18251 assert!(msg.body.contains("<details>kept</details>"));
18252 assert_eq!(runner.state.seats["impl-A"].turns, 0);
18253 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18254 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18255 }
18256
18257 #[tokio::test]
18258 async fn rewrite_reuses_a_resumable_implementer_session() {
18259 crate::run::pin_test_home();
18260 let dir = tempfile::tempdir().unwrap();
18261 let mut runner = rewrite_runner(
18262 dir.path(),
18263 false,
18264 vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
18265 );
18266 runner.state.config.roles.language_judge =
18267 Some(crate::config::AgentChoice::One("beta".into()));
18268 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18269 seat.turns = 1;
18270 runner.state.seats.insert("impl-A".into(), seat);
18271 let winner = runner.state.candidates[0].clone();
18272 runner
18273 .guarded_pr_message(&winner, None, true)
18274 .await
18275 .unwrap()
18276 .expect("rewritten");
18277 assert_eq!(runner.state.seats["impl-A"].turns, 2);
18278 assert!(!runner.state.seats.contains_key("github-text-rewrite"));
18279 }
18280
18281 #[tokio::test]
18282 async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
18283 crate::run::pin_test_home();
18286 let dir = tempfile::tempdir().unwrap();
18287 let mut exhausted = spec("alpha");
18288 exhausted.command = vec!["false".into()];
18289 let mut runner = rewrite_runner(
18290 dir.path(),
18291 false,
18292 vec![exhausted, scripted("beta", EN_REWRITE)],
18293 );
18294 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18296 "alpha".into(),
18297 "beta".into(),
18298 ]));
18299 let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
18300 seat.turns = 1;
18301 runner.state.seats.insert("impl-A".into(), seat);
18302 let winner = runner.state.candidates[0].clone();
18303 let msg = runner
18304 .guarded_pr_message(&winner, None, true)
18305 .await
18306 .unwrap()
18307 .expect("the role rewrites after the implementer fails");
18308 assert!(msg.body.contains("`src/net.rs`"));
18309 assert_eq!(
18310 runner.state.seats["impl-A"].turns, 2,
18311 "the implementer was tried once"
18312 );
18313 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
18314 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18315 }
18316
18317 #[tokio::test]
18318 async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
18319 crate::run::pin_test_home();
18320 let dir = tempfile::tempdir().unwrap();
18321 let mut broken = spec("bad");
18322 broken.command = vec!["false".into()];
18323 let mut runner = rewrite_runner(
18324 dir.path(),
18325 true,
18326 vec![
18327 broken,
18328 scripted("noformat", "Add retries."),
18329 scripted("good", EN_REWRITE),
18330 ],
18331 );
18332 runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
18333 "bad".into(),
18334 "noformat".into(),
18335 "good".into(),
18336 ]));
18337 let winner = runner.state.candidates[0].clone();
18338 let msg = runner
18339 .guarded_pr_message(&winner, None, true)
18340 .await
18341 .unwrap()
18342 .expect("the third agent rewrites");
18343 assert!(msg.body.contains("`src/net.rs`"));
18344 assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
18345 assert!(runner.state.handovers.is_empty(), "no handover is recorded");
18346 }
18347
18348 #[tokio::test]
18349 async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
18350 crate::run::pin_test_home();
18351 for (agents, role, saved) in [
18353 (
18354 vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
18355 Some("beta"),
18356 true,
18357 ),
18358 (vec![scripted("beta", "Add retries.")], Some("beta"), false),
18359 (Vec::new(), None, false),
18360 ] {
18361 let dir = tempfile::tempdir().unwrap();
18362 let mut runner = rewrite_runner(dir.path(), true, agents);
18363 runner.state.config.roles.language_judge =
18364 role.map(|r| crate::config::AgentChoice::One(r.into()));
18365 let winner = runner.state.candidates[0].clone();
18366 let first = runner
18367 .guarded_pr_message(&winner, None, true)
18368 .await
18369 .unwrap();
18370 assert!(first.is_none(), "withheld text parks the run");
18371 assert!(runner.state.parked);
18372 assert_eq!(gate_questions(&runner), 1);
18373 let g = runner.state.github_text.as_ref().unwrap();
18374 assert_eq!(g.rewrite.is_some(), saved);
18375 assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
18376 }
18377 }
18378
18379 #[tokio::test]
18380 async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
18381 crate::run::pin_test_home();
18382 let dir = tempfile::tempdir().unwrap();
18383 let mut runner = runner_at(RunStatus::Gating);
18384 runner.state =
18385 state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
18386 runner.state.repo = dir.path().to_owned();
18387 runner.state.candidates[0].worktree = dir.path().to_owned();
18388 let mut author = spec("alpha");
18389 author.command = vec![
18390 "sh".into(),
18391 "-c".into(),
18392 "printf '%s' \"$REWRITE_REPLY\"".into(),
18393 ];
18394 author.env.insert(
18395 "REWRITE_REPLY".into(),
18396 format!(
18397 "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
18398 "English sentence. ".repeat(600)
18399 ),
18400 );
18401 runner.state.config.agents = vec![author];
18402 seed_public_summary(&mut runner);
18403 let winner = runner.state.candidates[0].clone();
18404 assert!(
18405 runner
18406 .guarded_pr_message(&winner, None, true)
18407 .await
18408 .unwrap()
18409 .is_none()
18410 );
18411 let record = runner.state.github_text.clone().unwrap();
18412 let rewrite = record.rewrite.clone().unwrap();
18413 assert_eq!(rewrite.failed, ["body-language"]);
18414 assert!(
18415 rewrite
18416 .body
18417 .ends_with(&format!("magi:run/{}", runner.state.id))
18418 );
18419 let store = ask::Questions::open();
18420 let q = store.get(record.question.as_ref().unwrap()).unwrap();
18421 assert!(
18422 q.choices
18423 .contains(&crate::github_text::USE_REWRITE.to_owned())
18424 );
18425 for label in [
18426 "ORIGINAL",
18427 "FALLBACK",
18428 "REWRITE",
18429 "Rules still failed: body-language",
18430 "truncated",
18431 ] {
18432 assert!(q.detail.contains(label), "{label}");
18433 }
18434 let artifact = std::fs::read_to_string(crate::run::artifact_path(
18435 &runner.state,
18436 &crate::github_text::artifact_name(&record.fingerprint),
18437 ))
18438 .unwrap();
18439 assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
18440 runner.state =
18442 serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
18443 store
18444 .update(&q.id, |q| {
18445 q.say("まだ日本語")?;
18446 q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
18447 })
18448 .unwrap();
18449 assert!(
18450 runner
18451 .guarded_pr_message(&winner, None, true)
18452 .await
18453 .unwrap()
18454 .is_none()
18455 );
18456 let retry = runner.state.github_text.as_ref().unwrap();
18457 assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
18458 let q = store.get(retry.question.as_ref().unwrap()).unwrap();
18459 assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
18460 assert!(
18461 q.posting_comparison
18462 .as_ref()
18463 .unwrap()
18464 .matches_detail(&q.detail)
18465 );
18466 store
18467 .update(&q.id, |q| {
18468 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18469 })
18470 .unwrap();
18471 let message = runner
18472 .guarded_pr_message(&winner, None, true)
18473 .await
18474 .unwrap()
18475 .unwrap();
18476 assert_eq!(message.title, rewrite.title);
18477 assert_eq!(message.body, rewrite.body);
18478 assert_eq!(
18479 runner.state.github_text.as_ref().unwrap().choice,
18480 Some(GithubTextChoice::Rewrite)
18481 );
18482 let again = runner
18483 .guarded_pr_message(&winner, None, true)
18484 .await
18485 .unwrap()
18486 .unwrap();
18487 assert_eq!(again.title, message.title);
18488 assert_eq!(again.body, message.body);
18489 assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
18490 assert!(
18491 !runner
18492 .state
18493 .events
18494 .iter()
18495 .any(|e| e.message.contains(&rewrite.title)
18496 || e.message.contains("English sentence."))
18497 );
18498 }
18499
18500 #[tokio::test]
18501 async fn github_text_rewrite_choice_without_saved_text_falls_back() {
18502 crate::run::pin_test_home();
18503 let dir = tempfile::tempdir().unwrap();
18504 let mut runner = runner_at(RunStatus::Gating);
18505 runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
18506 runner.state.repo = dir.path().to_owned();
18507 runner.state.candidates[0].worktree = dir.path().to_owned();
18508 seed_public_summary(&mut runner);
18509 let winner = runner.state.candidates[0].clone();
18510 assert!(
18511 runner
18512 .guarded_pr_message(&winner, None, true)
18513 .await
18514 .unwrap()
18515 .is_none()
18516 );
18517 let record = runner.state.github_text.as_ref().unwrap();
18518 assert!(record.rewrite.is_none());
18519 let mut old = serde_json::to_value(record).unwrap();
18521 old.as_object_mut().unwrap().remove("rewrite");
18522 let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
18523 assert!(old.rewrite.is_none());
18524 let id = record.question.clone().unwrap();
18525 ask::Questions::open()
18526 .update(&id, |q| {
18527 q.choices.push(crate::github_text::USE_REWRITE.into());
18528 q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
18529 })
18530 .unwrap();
18531 let message = runner
18532 .guarded_pr_message(&winner, None, true)
18533 .await
18534 .unwrap()
18535 .unwrap();
18536 assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
18537 assert_eq!(
18538 runner.state.github_text.as_ref().unwrap().choice,
18539 Some(GithubTextChoice::Fallback)
18540 );
18541 }
18542
18543 #[tokio::test]
18544 async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
18545 crate::run::pin_test_home();
18546 let dir = tempfile::tempdir().unwrap();
18547 let mut runner = runner_at(RunStatus::Gating);
18548 runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
18549 runner.state.repo = dir.path().to_owned();
18550 runner.state.candidates[0].worktree = dir.path().to_owned();
18551 seed_public_summary(&mut runner);
18552 let winner = runner.state.candidates[0].clone();
18553 assert!(
18554 runner
18555 .guarded_pr_message(&winner, None, true)
18556 .await
18557 .unwrap()
18558 .is_none()
18559 );
18560 let store = ask::Questions::open();
18561 let find = |run: &str| {
18562 store
18563 .list()
18564 .into_iter()
18565 .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
18566 .unwrap()
18567 };
18568 let run = runner.state.id.clone();
18569 let q = find(&run);
18570 let fp = runner
18571 .state
18572 .github_text
18573 .as_ref()
18574 .unwrap()
18575 .fingerprint
18576 .clone();
18577 let name = crate::github_text::artifact_name(&fp);
18578 let path = crate::run::artifact_path(&runner.state, &name);
18579 assert!(q.detail.contains(&path.display().to_string()));
18580 assert!(q.detail.contains("title-language: title"));
18581 let saved = std::fs::read_to_string(&path).expect("artifact written");
18582 assert!(saved.contains("日本語のタイトル"));
18583 assert!(q.cwd.is_none());
18584 assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
18585 let say = |id: &str, text: &str, choice: &str| {
18587 store
18588 .update(id, |q| {
18589 q.thread.push(ask::Turn {
18590 who: ask::Who::Operator,
18591 body: text.to_owned(),
18592 at: jiff::Timestamp::now(),
18593 note: None,
18594 });
18595 q.answer(ask::Answer::Choice(choice.to_owned()))
18596 })
18597 .unwrap();
18598 };
18599 say(&q.id, "まだ日本語", "use my text");
18600 assert!(
18601 runner
18602 .guarded_pr_message(&winner, None, true)
18603 .await
18604 .unwrap()
18605 .is_none()
18606 );
18607 let second = find(&run);
18608 assert_ne!(second.id, q.id);
18609 assert!(second.detail.contains("did not pass"));
18610 assert!(second.detail.contains("日本語のタイトル"));
18613 assert!(!second.detail.contains("まだ日本語"));
18614 assert!(second.detail.contains(&path.display().to_string()));
18615 say(&second.id, "fix: retry failed requests", "use my text");
18616 let message = runner
18617 .guarded_pr_message(&winner, None, true)
18618 .await
18619 .unwrap()
18620 .expect("a vetted replacement is posted");
18621 assert_eq!(message.title, "fix: retry failed requests");
18622 assert_eq!(
18623 runner
18624 .state
18625 .github_text
18626 .as_ref()
18627 .unwrap()
18628 .chosen_title
18629 .as_deref(),
18630 Some("fix: retry failed requests")
18631 );
18632 assert!(runner.state.github_text.as_ref().unwrap().resolved);
18633 let again = runner
18635 .guarded_pr_message(&winner, None, true)
18636 .await
18637 .unwrap()
18638 .unwrap();
18639 assert_eq!(again.title, message.title);
18640 assert_eq!(
18641 store
18642 .list()
18643 .into_iter()
18644 .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
18645 .count(),
18646 2
18647 );
18648 }
18649
18650 #[test]
18651 fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
18652 let mut state = state_with_summary(
18656 "add retries",
18657 "TITLE: fix(web): batch reads\n- reads run.json once",
18658 );
18659 state.config.graph.language = "ja".to_owned();
18660 let m = pr_message(&state, 'A');
18661 assert!(crate::github_text::check(&m.title, &m.body).is_empty());
18662 assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
18663
18664 let task = "今回やってほしいこと: results projector を直す";
18667 let mut state = state_with_summary(task, "- no title line");
18668 state.config.graph.language = "ja".to_owned();
18669 let m = pr_message(&state, 'A');
18670 assert_eq!(
18671 m.title,
18672 format!("chore: land candidate A of run {}", state.id)
18673 );
18674 assert!(
18675 m.body.contains(&format!(
18676 "<summary>Original task</summary>\n\n{task}\n\n</details>"
18677 )),
18678 "{}",
18679 m.body
18680 );
18681 }
18682
18683 #[test]
18684 fn pr_message_scrubs_home_paths_and_addresses() {
18685 let state = state_with_summary(
18686 "fix it in /Users/someone/src/x",
18687 "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
18688 );
18689 let m = pr_message(&state, 'A');
18690 for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
18691 assert!(!m.body.contains(leak), "{}", m.body);
18692 }
18693 assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
18694 }
18695
18696 #[test]
18697 fn pr_message_survives_a_task_that_closes_details() {
18698 let state = state_with_summary("a </details> b", "TITLE: fix: x");
18699 let m = pr_message(&state, 'A');
18700 assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
18701 }
18702
18703 #[test]
18704 fn manual_squash_subject_cannot_break_out_of_its_quotes() {
18705 let cmd = manual_merge_command(
18706 MergeStyle::Squash,
18707 Path::new("/repo"),
18708 "b",
18709 "fix: \"quoted\" $(x) `y`\n\nbody",
18710 );
18711 assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
18712 }
18713
18714 #[test]
18715 fn manual_merge_command_matches_the_configured_style() {
18716 let repo = Path::new("/repo");
18717 let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
18718
18719 let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
18720 assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
18721
18722 let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
18723 assert_eq!(
18724 squash,
18725 "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
18726 \"Merge magi run 0832 (candidate A)\""
18727 );
18728
18729 let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
18730 assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
18731 }
18732
18733 #[test]
18734 fn a_nudge_gets_a_quarter_of_the_budget() {
18735 assert_eq!(retry_budget(secs(1200), true), secs(300));
18737 assert_eq!(retry_budget(secs(3600), true), secs(900));
18738 }
18739
18740 #[test]
18741 fn a_resent_prompt_keeps_the_whole_budget() {
18742 assert_eq!(retry_budget(secs(1200), false), secs(1200));
18745 assert_eq!(retry_budget(secs(60), false), secs(60));
18746 }
18747
18748 #[test]
18749 fn the_floor_never_exceeds_the_original_budget() {
18750 assert_eq!(retry_budget(secs(60), true), secs(60));
18754 assert_eq!(retry_budget(secs(480), true), secs(120));
18755 assert_eq!(retry_budget(secs(0), true), secs(0));
18756 }
18757
18758 fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
18759 agent::CommandEvidence {
18760 id: "item1".to_owned(),
18761 description: "cargo test".to_owned(),
18762 exit_code,
18763 result_summary: String::new(),
18764 source: "codex".to_owned(),
18765 }
18766 }
18767
18768 #[test]
18769 fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
18770 assert!(!has_unconfirmed_command(&[]));
18774 }
18775
18776 #[test]
18777 fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
18778 assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
18782 assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
18783 assert!(!has_unconfirmed_command(&[
18784 evidence(Some(0)),
18785 evidence(Some(101))
18786 ]));
18787 }
18788
18789 #[test]
18790 fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
18791 assert!(has_unconfirmed_command(&[
18792 evidence(Some(0)),
18793 evidence(None)
18794 ]));
18795 }
18796
18797 #[test]
18798 fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
18799 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18800 assert_eq!(
18801 verified_noop_claim(true, &[], text).as_deref(),
18802 Some("already fixed by b32cfc4, on main.")
18803 );
18804 }
18805
18806 #[test]
18807 fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
18808 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18811 assert!(verified_noop_claim(false, &[], text).is_none());
18812 }
18813
18814 #[test]
18815 fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
18816 let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
18817 assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
18818 assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
18820 }
18821
18822 #[test]
18823 fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
18824 assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
18825 }
18826
18827 fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
18830 runner.state.candidates = shape
18831 .iter()
18832 .enumerate()
18833 .map(|(i, &(empty, verified))| Candidate {
18834 index: i,
18835 label: (b'A' + i as u8) as char,
18836 agent: "sonnet".to_owned(),
18837 branch: format!("magi/x/{}", (b'A' + i as u8) as char),
18838 worktree: PathBuf::from(format!("/wt/{i}")),
18839 summary: String::new(),
18840 stat: String::new(),
18841 files: 0,
18842 commits: 0,
18843 empty,
18844 failed: None,
18845 verified_noop: verified.map(str::to_owned),
18846 duration_ms: 0,
18847 folded: false,
18848 })
18849 .collect();
18850 }
18851
18852 #[test]
18853 fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
18854 ask_test_home();
18855 let mut runner = runner_at(RunStatus::Implementing);
18856 set_candidates(
18857 &mut runner,
18858 &[
18859 (true, Some("already on main at b32cfc4")),
18860 (true, Some("same fix, see the existing test")),
18861 ],
18862 );
18863
18864 runner
18865 .after_implement()
18866 .expect("a verified no-op is not an error");
18867
18868 assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
18869 }
18870
18871 #[test]
18872 fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
18873 ask_test_home();
18874 let mut runner = runner_at(RunStatus::Implementing);
18875 set_candidates(
18879 &mut runner,
18880 &[(true, Some("already on main at b32cfc4")), (true, None)],
18881 );
18882
18883 let err = runner
18884 .after_implement()
18885 .expect_err("an unverified empty candidate must still fail the run");
18886
18887 assert!(
18888 err.to_string().contains("no candidate produced a change"),
18889 "{err}"
18890 );
18891 assert_eq!(runner.state.status, RunStatus::Failed);
18892 }
18893
18894 #[test]
18895 fn after_implement_still_fails_an_ordinary_all_empty_run() {
18896 ask_test_home();
18897 let mut runner = runner_at(RunStatus::Implementing);
18898 set_candidates(&mut runner, &[(true, None), (true, None)]);
18899
18900 let err = runner
18901 .after_implement()
18902 .expect_err("no candidate declared anything; this is an ordinary failure");
18903
18904 assert!(
18905 err.to_string().contains("no candidate produced a change"),
18906 "{err}"
18907 );
18908 assert_eq!(runner.state.status, RunStatus::Failed);
18909 }
18910}
18911
18912#[derive(Debug)]
18915struct GraftRestoreFailed(String);
18916
18917impl std::fmt::Display for GraftRestoreFailed {
18918 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
18919 write!(
18920 f,
18921 "could not restore the winner to its pre-graft commit: {}",
18922 self.0
18923 )
18924 }
18925}
18926
18927impl std::error::Error for GraftRestoreFailed {}
18928
18929#[cfg(test)]
18933mod resource_wait_tests {
18934 use super::*;
18935 use std::collections::BTreeMap;
18936
18937 fn sh(id: &str, script: &str) -> AgentSpec {
18938 AgentSpec {
18939 id: id.to_owned(),
18940 kind: crate::config::AgentKind::Command,
18941 model: None,
18942 command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
18943 extra_args: Vec::new(),
18944 env: BTreeMap::new(),
18945 prompt_delivery: None,
18946 max_turn_secs: None,
18947 }
18948 }
18949
18950 fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
18951 SeatJob {
18952 seat: SeatState::new("impl-A", &spec.id, 1),
18953 spec,
18954 cwd: dir.to_path_buf(),
18955 prompt: "do it".to_owned(),
18956 timeout,
18957 allow_write: true,
18958 sessions: false,
18959 artifacts: dir.join("art"),
18960 stem: "impl-A".to_owned(),
18961 handover: None,
18962 }
18963 }
18964
18965 fn state(resource_wait_secs: u64) -> RunState {
18966 let mut config = Config::default();
18967 config.graph.timeout_resource_wait = resource_wait_secs;
18968 RunState::new(
18969 PathBuf::from("/repo"),
18970 "main".to_owned(),
18971 "abc1234".to_owned(),
18972 "task".to_owned(),
18973 config,
18974 )
18975 }
18976
18977 fn hold(cache: &Path) -> crate::cache::Guard {
18978 let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
18979 match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
18980 crate::cache::AcquireOutcome::Acquired(g) => g,
18981 crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
18982 }
18983 }
18984
18985 async fn run(
18986 jobs: Vec<SeatJob>,
18987 sem: Arc<Semaphore>,
18988 cache: Option<&Path>,
18989 st: &mut RunState,
18990 ) -> Vec<(usize, SeatState, AgentOutcome)> {
18991 let prompts = Prompts::default();
18992 let ctx = WaveCtx {
18993 run: "r",
18994 node: "implement",
18995 prompts: &prompts,
18996 cache,
18997 round: None,
18998 carry_seats: false,
18999 pauses: None,
19000 };
19001 wave(jobs, sem, &ctx, st, 0).await
19002 }
19003
19004 #[tokio::test]
19005 async fn verification_cache_wait_preserves_the_command_budget() {
19006 crate::run::pin_test_home();
19007 let tmp = tempfile::tempdir().unwrap();
19008 let cache = tmp.path().join("target");
19009 let guard = hold(&cache);
19010 let release = tokio::spawn(async move {
19011 tokio::time::sleep(Duration::from_millis(100)).await;
19012 guard.release();
19013 });
19014 let mut st = state(10);
19015 let budget = Duration::from_millis(50);
19016 let wait = repository_wait(&st, None);
19017 let (outcomes, _) = with_cache_lease(
19018 &mut st,
19019 Some(&cache),
19020 "gate",
19021 "gate",
19022 tmp.path(),
19023 "abc1234",
19024 budget,
19025 "test gate",
19026 &wait,
19027 |_, available| async move {
19028 assert_eq!(
19029 available, budget,
19030 "contention must not shorten verification"
19031 );
19032 (Vec::new(), false, Vec::new())
19033 },
19034 )
19035 .await
19036 .unwrap();
19037 release.await.unwrap();
19038 assert!(outcomes.is_empty(), "{outcomes:?}");
19039 assert!(
19040 st.events
19041 .iter()
19042 .any(|event| event.message.contains("waiting for"))
19043 );
19044 assert!(st.resource_wait.is_none());
19045 hold(&cache).release();
19047 }
19048
19049 #[tokio::test]
19050 async fn verification_cache_wait_expires_without_starting_commands() {
19051 crate::run::pin_test_home();
19052 let tmp = tempfile::tempdir().unwrap();
19053 let cache = tmp.path().join("target");
19054 let _held = hold(&cache);
19055 let mut st = state(0);
19056 let wait = repository_wait(&st, None);
19057 let (outcomes, _) = with_cache_lease(
19058 &mut st,
19059 Some(&cache),
19060 "gate",
19061 "gate",
19062 tmp.path(),
19063 "abc1234",
19064 Duration::from_secs(60),
19065 "test gate",
19066 &wait,
19067 |_, _| async { panic!("commands must not start without the cache lease") },
19068 )
19069 .await
19070 .unwrap();
19071 assert_eq!(outcomes.len(), 1);
19072 assert!(outcomes[0].resource_blocked);
19073 assert!(st.resource_wait.is_none());
19074 }
19075
19076 #[tokio::test]
19077 async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
19078 crate::run::pin_test_home();
19079 let tmp = tempfile::tempdir().unwrap();
19080 let cache = tmp.path().join("target-a");
19081 let guard = hold(&cache);
19082 tokio::spawn(async move {
19084 tokio::time::sleep(Duration::from_millis(1500)).await;
19085 guard.release();
19086 });
19087 let mut st = state(60);
19088 let results = run(
19089 vec![job(
19090 sh("alpha", "echo ok"),
19091 tmp.path(),
19092 Duration::from_secs(1),
19093 )],
19094 Arc::new(Semaphore::new(1)),
19095 Some(&cache),
19096 &mut st,
19097 )
19098 .await;
19099 assert!(
19100 matches!(results[0].2, AgentOutcome::Ok(_)),
19101 "the wait must not be charged to the agent"
19102 );
19103 assert!(st.resource_block.is_none());
19104 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19105 assert!(st.resource_wait.is_none() && st.active.is_empty());
19106 }
19107
19108 #[tokio::test]
19109 #[cfg(unix)]
19110 async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
19111 use std::os::fd::AsRawFd as _;
19112 crate::run::pin_test_home();
19113 let tmp = tempfile::tempdir().unwrap();
19114 crate::git::git(tmp.path(), &["init"]).await.unwrap();
19115 let common = tmp.path().join(".git");
19116 std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19117 let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19118 assert_eq!(
19119 unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19120 0
19121 );
19122 let mut st = state(1);
19123 st.config.coordination = crate::config::test_coordination();
19124 let mut reader = job(
19125 sh("reader", "echo must-not-run"),
19126 tmp.path(),
19127 Duration::from_secs(30),
19128 );
19129 reader.allow_write = false;
19130 let results = run(
19131 vec![reader.clone()],
19132 Arc::new(Semaphore::new(1)),
19133 None,
19134 &mut st,
19135 )
19136 .await;
19137 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19138 assert_eq!(results[0].1.turns, 0);
19139 assert!(FailClass::of(&results[0].2).is_none());
19140 assert_eq!(
19141 st.resource_block.as_ref().unwrap().kind,
19142 "repository coordination"
19143 );
19144 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19145 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19146 drop(external);
19147 let editor = crate::repo_lock::SeatLease::for_checkout(
19148 tmp.path(),
19149 true,
19150 crate::config::test_coordination().as_ref(),
19151 )
19152 .await
19153 .unwrap();
19154 let pause = Pause::new();
19155 let interrupt = Pause::new();
19156 let prompts = Prompts::default();
19157 let ctx = WaveCtx {
19158 run: "r",
19159 node: "review",
19160 prompts: &prompts,
19161 cache: None,
19162 round: None,
19163 carry_seats: false,
19164 pauses: Some((&pause, &interrupt)),
19165 };
19166 let (results, ()) = tokio::join!(
19167 wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
19168 async {
19169 tokio::time::sleep(Duration::from_millis(50)).await;
19170 pause.park();
19171 }
19172 );
19173 assert!(
19174 matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
19175 );
19176 assert_eq!(results[0].1.turns, 0);
19177 assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
19178 assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
19179 drop(editor);
19180 }
19181
19182 #[tokio::test]
19183 async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
19184 crate::run::pin_test_home();
19185 let tmp = tempfile::tempdir().unwrap();
19186 let cache = tmp.path().join("target-b");
19187 let _held = hold(&cache);
19188 let mut st = state(1);
19189 let results = run(
19190 vec![job(
19191 sh("alpha", "echo ok"),
19192 tmp.path(),
19193 Duration::from_secs(1),
19194 )],
19195 Arc::new(Semaphore::new(1)),
19196 Some(&cache),
19197 &mut st,
19198 )
19199 .await;
19200 let (_, seat, out) = &results[0];
19201 assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
19202 assert!(wave_blocked(&results));
19203 assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
19204 assert_eq!(seat.turns, 0, "nothing was asked");
19205 assert!(
19206 !tmp.path().join("art").join("impl-A.prompt.md").exists(),
19207 "no agent was spawned"
19208 );
19209 assert!(FailClass::of(out).is_none(), "not a failure class");
19210 let block = st.resource_block.as_ref().expect("recorded for the resume");
19211 assert_eq!(block.kind, "shared build cache");
19212 assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
19213 assert!(st.active.is_empty());
19214 }
19215
19216 #[tokio::test]
19217 async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
19218 crate::run::pin_test_home();
19219 let tmp = tempfile::tempdir().unwrap();
19220 let mut st = state(1);
19221 let results = run(
19222 vec![job(
19223 sh("alpha", "echo ok"),
19224 tmp.path(),
19225 Duration::from_secs(1),
19226 )],
19227 Arc::new(Semaphore::new(0)),
19228 None,
19229 &mut st,
19230 )
19231 .await;
19232 assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
19233 assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
19234 assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
19235 }
19236
19237 #[tokio::test]
19238 async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
19239 crate::run::pin_test_home();
19240 let tmp = tempfile::tempdir().unwrap();
19241 let mut st = state(60);
19242 let results = run(
19243 vec![job(
19244 sh("alpha", "sleep 30"),
19245 tmp.path(),
19246 Duration::from_secs(1),
19247 )],
19248 Arc::new(Semaphore::new(1)),
19249 None,
19250 &mut st,
19251 )
19252 .await;
19253 let out = &results[0].2;
19254 assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
19255 let class = FailClass::of(out).expect("a failure");
19256 assert_eq!(class, FailClass::Timeout);
19257 assert!(should_hand_over(None, &class), "existing handover policy");
19258 assert!(st.resource_block.is_none());
19259 }
19260}